Move Python proto package into submodule (#1393)

* Move Python proto package into subdirectory

Signed-off-by: Willem Pienaar <git@willem.co>

* Enable CI dep installation

Signed-off-by: Willem Pienaar <git@willem.co>

* Add tensorflow types back

Signed-off-by: Willem Pienaar <git@willem.co>

Makefile

@@ -45,8 +45,8 @@ package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
 
 compile-protos-python: install-python-ci-dependencies
-	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;)
-	@$(foreach dir,$(PROTO_SERVICE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;)
+	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
+	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
 	cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto
 
 install-python: compile-protos-python

sdk/python/feast/client.py

@@ -26,7 +26,23 @@
 
 from feast.config import Config
 from feast.constants import ConfigOptions as opt
-from feast.core.CoreService_pb2 import (
+from feast.data_format import ParquetFormat
+from feast.data_source import BigQuerySource, FileSource
+from feast.entity import Entity
+from feast.feature import Feature, FeatureRef, _build_feature_references
+from feast.feature_table import FeatureTable
+from feast.grpc import auth as feast_auth
+from feast.grpc.grpc import create_grpc_channel
+from feast.loaders.ingest import (
+    _check_field_mappings,
+    _read_table_from_source,
+    _upload_to_bq_source,
+    _upload_to_file_source,
+    _write_non_partitioned_table_from_source,
+    _write_partitioned_table_from_source,
+)
+from feast.online_response import OnlineResponse, _infer_online_entity_rows
+from feast.protos.feast.core.CoreService_pb2 import (
     ApplyEntityRequest,
     ApplyEntityResponse,
     ApplyFeatureTableRequest,
@@ -50,29 +66,13 @@
     ListProjectsRequest,
     ListProjectsResponse,
 )
-from feast.core.CoreService_pb2_grpc import CoreServiceStub
-from feast.data_format import ParquetFormat
-from feast.data_source import BigQuerySource, FileSource
-from feast.entity import Entity
-from feast.feature import Feature, FeatureRef, _build_feature_references
-from feast.feature_table import FeatureTable
-from feast.grpc import auth as feast_auth
-from feast.grpc.grpc import create_grpc_channel
-from feast.loaders.ingest import (
-    _check_field_mappings,
-    _read_table_from_source,
-    _upload_to_bq_source,
-    _upload_to_file_source,
-    _write_non_partitioned_table_from_source,
-    _write_partitioned_table_from_source,
-)
-from feast.online_response import OnlineResponse, _infer_online_entity_rows
-from feast.registry import Registry
-from feast.serving.ServingService_pb2 import (
+from feast.protos.feast.core.CoreService_pb2_grpc import CoreServiceStub
+from feast.protos.feast.serving.ServingService_pb2 import (
     GetFeastServingInfoRequest,
     GetOnlineFeaturesRequestV2,
 )
-from feast.serving.ServingService_pb2_grpc import ServingServiceStub
+from feast.protos.feast.serving.ServingService_pb2_grpc import ServingServiceStub
+from feast.registry import Registry
 from feast.telemetry import log_usage
 
 _logger = logging.getLogger(__name__)

sdk/python/feast/data_format.py

@@ -15,8 +15,8 @@
 
 from abc import ABC, abstractmethod
 
-from feast.core.DataFormat_pb2 import FileFormat as FileFormatProto
-from feast.core.DataFormat_pb2 import StreamFormat as StreamFormatProto
+from feast.protos.feast.core.DataFormat_pb2 import FileFormat as FileFormatProto
+from feast.protos.feast.core.DataFormat_pb2 import StreamFormat as StreamFormatProto
 
 
 class FileFormat(ABC):

sdk/python/feast/data_source.py

@@ -16,8 +16,8 @@
 import enum
 from typing import Dict, Optional
 
-from feast.core.DataSource_pb2 import DataSource as DataSourceProto
 from feast.data_format import FileFormat, StreamFormat
+from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
 
 
 class SourceType(enum.Enum):

sdk/python/feast/entity.py

@@ -19,10 +19,10 @@
 from google.protobuf.json_format import MessageToDict, MessageToJson
 from google.protobuf.timestamp_pb2 import Timestamp
 
-from feast.core.Entity_pb2 import Entity as EntityV2Proto
-from feast.core.Entity_pb2 import EntityMeta as EntityMetaProto
-from feast.core.Entity_pb2 import EntitySpecV2 as EntitySpecProto
 from feast.loaders import yaml as feast_yaml
+from feast.protos.feast.core.Entity_pb2 import Entity as EntityV2Proto
+from feast.protos.feast.core.Entity_pb2 import EntityMeta as EntityMetaProto
+from feast.protos.feast.core.Entity_pb2 import EntitySpecV2 as EntitySpecProto
 from feast.value_type import ValueType
 
 

sdk/python/feast/feature.py

@@ -14,9 +14,11 @@
 
 from typing import List, MutableMapping, Optional
 
-from feast.core.Feature_pb2 import FeatureSpecV2 as FeatureSpecProto
-from feast.serving.ServingService_pb2 import FeatureReferenceV2 as FeatureRefProto
-from feast.types import Value_pb2 as ValueTypeProto
+from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FeatureSpecProto
+from feast.protos.feast.serving.ServingService_pb2 import (
+    FeatureReferenceV2 as FeatureRefProto,
+)
+from feast.protos.feast.types import Value_pb2 as ValueTypeProto
 from feast.value_type import ValueType
 
 

sdk/python/feast/feature_table.py

@@ -20,9 +20,6 @@
 from google.protobuf.json_format import MessageToDict, MessageToJson
 from google.protobuf.timestamp_pb2 import Timestamp
 
-from feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto
-from feast.core.FeatureTable_pb2 import FeatureTableMeta as FeatureTableMetaProto
-from feast.core.FeatureTable_pb2 import FeatureTableSpec as FeatureTableSpecProto
 from feast.data_source import (
     BigQuerySource,
     DataSource,
@@ -32,6 +29,13 @@
 )
 from feast.feature import Feature
 from feast.loaders import yaml as feast_yaml
+from feast.protos.feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto
+from feast.protos.feast.core.FeatureTable_pb2 import (
+    FeatureTableMeta as FeatureTableMetaProto,
+)
+from feast.protos.feast.core.FeatureTable_pb2 import (
+    FeatureTableSpec as FeatureTableSpecProto,
+)
 from feast.value_type import ValueType
 
 

sdk/python/feast/feature_view.py

@@ -17,11 +17,15 @@
 from google.protobuf.duration_pb2 import Duration
 from google.protobuf.timestamp_pb2 import Timestamp
 
-from feast.core.FeatureView_pb2 import FeatureView as FeatureViewProto
-from feast.core.FeatureView_pb2 import FeatureViewMeta as FeatureViewMetaProto
-from feast.core.FeatureView_pb2 import FeatureViewSpec as FeatureViewSpecProto
 from feast.data_source import BigQuerySource, DataSource, FileSource
 from feast.feature import Feature
+from feast.protos.feast.core.FeatureView_pb2 import FeatureView as FeatureViewProto
+from feast.protos.feast.core.FeatureView_pb2 import (
+    FeatureViewMeta as FeatureViewMetaProto,
+)
+from feast.protos.feast.core.FeatureView_pb2 import (
+    FeatureViewSpec as FeatureViewSpecProto,
+)
 from feast.value_type import ValueType
 
 

sdk/python/feast/infra/gcp.py

@@ -6,9 +6,9 @@
 
 from feast import FeatureTable, FeatureView
 from feast.infra.provider import Provider
+from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
+from feast.protos.feast.types.Value_pb2 import Value as ValueProto
 from feast.repo_config import DatastoreOnlineStoreConfig
-from feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
-from feast.types.Value_pb2 import Value as ValueProto
 
 from .key_encoding_utils import serialize_entity_key
 

sdk/python/feast/infra/key_encoding_utils.py

@@ -1,9 +1,9 @@
 import struct
 from typing import List, Tuple
 
-from feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
-from feast.types.Value_pb2 import Value as ValueProto
-from feast.types.Value_pb2 import ValueType
+from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
+from feast.protos.feast.types.Value_pb2 import Value as ValueProto
+from feast.protos.feast.types.Value_pb2 import ValueType
 
 
 def _serialize_val(value_type, v: ValueProto) -> Tuple[bytes, int]:

sdk/python/feast/infra/local_sqlite.py

@@ -6,9 +6,9 @@
 from feast import FeatureTable, FeatureView
 from feast.infra.key_encoding_utils import serialize_entity_key
 from feast.infra.provider import Provider
+from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
+from feast.protos.feast.types.Value_pb2 import Value as ValueProto
 from feast.repo_config import LocalOnlineStoreConfig
-from feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
-from feast.types.Value_pb2 import Value as ValueProto
 
 
 def _table_id(project: str, table: Union[FeatureTable, FeatureView]) -> str:

sdk/python/feast/infra/provider.py

@@ -3,9 +3,9 @@
 from typing import Dict, List, Optional, Tuple, Union
 
 from feast import FeatureTable, FeatureView
+from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
+from feast.protos.feast.types.Value_pb2 import Value as ValueProto
 from feast.repo_config import RepoConfig
-from feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
-from feast.types.Value_pb2 import Value as ValueProto
 
 
 class Provider(abc.ABC):

sdk/python/feast/online_response.py

@@ -14,16 +14,16 @@
 
 from typing import Any, Dict, List, cast
 
-from feast.serving.ServingService_pb2 import (
+from feast.protos.feast.serving.ServingService_pb2 import (
     GetOnlineFeaturesRequestV2,
     GetOnlineFeaturesResponse,
 )
+from feast.protos.feast.types.Value_pb2 import Value as Value
 from feast.type_map import (
     _python_value_to_proto_value,
     feast_value_type_to_python_type,
     python_type_to_feast_value_type,
 )
-from feast.types.Value_pb2 import Value as Value
 
 
 class OnlineResponse:

sdk/python/feast/registry.py

@@ -22,10 +22,10 @@
 
 from google.auth.exceptions import DefaultCredentialsError
 
-from feast.core.Registry_pb2 import Registry as RegistryProto
 from feast.entity import Entity
 from feast.feature_table import FeatureTable
 from feast.feature_view import FeatureView
+from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
 
 REGISTRY_SCHEMA_VERSION = "1"
 

sdk/python/feast/type_map.py

@@ -22,7 +22,7 @@
 from google.protobuf.timestamp_pb2 import Timestamp
 from pyarrow.lib import TimestampType
 
-from feast.types.Value_pb2 import (
+from feast.protos.feast.types.Value_pb2 import (
     BoolList,
     BytesList,
     DoubleList,
@@ -31,8 +31,8 @@
     Int64List,
     StringList,
 )
-from feast.types.Value_pb2 import Value as ProtoValue
-from feast.types.Value_pb2 import ValueType as ProtoValueType
+from feast.protos.feast.types.Value_pb2 import Value as ProtoValue
+from feast.protos.feast.types.Value_pb2 import ValueType as ProtoValueType
 from feast.value_type import ValueType
 
 

sdk/python/pyproject.toml

@@ -17,11 +17,7 @@ exclude = '''
     | dist
     | pb2.py
     | \.pyi
-    | core
-    | serving
-    | storage
-    | types
-    | third_party
+    | protos
   )/
 )
 '''

sdk/python/setup.cfg

@@ -4,7 +4,7 @@ include_trailing_comma=True
 force_grid_wrap=0
 use_parentheses=True
 line_length=88
-skip=feast/types,feast/core,feast/serving,feast/storage,feast/third_party
+skip=feast/protos
 known_first_party=feast,feast_serving_server,feast_core_server
 default_section=THIRDPARTY
 
@@ -13,7 +13,7 @@ ignore = E203, E266, E501, W503
 max-line-length = 88
 max-complexity = 20
 select = B,C,E,F,W,T4
-exclude = .git,__pycache__,docs/conf.py,dist,feast/core,feast/serving,feast/types,feast/storage,feast/third_party
+exclude = .git,__pycache__,docs/conf.py,dist,feast/protos
 
 [mypy]
 files=feast,test

sdk/python/tests/cli/online_read_write_test.py

@@ -2,8 +2,8 @@
 from pathlib import Path
 
 from feast.feature_store import FeatureStore
-from feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
-from feast.types.Value_pb2 import Value as ValueProto
+from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
+from feast.protos.feast.types.Value_pb2 import Value as ValueProto
 
 
 def basic_rw_test(repo_path: Path, project_name: str) -> None:

sdk/python/tests/feast_core_server.py

@@ -5,8 +5,8 @@
 import grpc
 from google.protobuf.timestamp_pb2 import Timestamp
 
-from feast.core import CoreService_pb2_grpc as Core
-from feast.core.CoreService_pb2 import (
+from feast.protos.feast.core import CoreService_pb2_grpc as Core
+from feast.protos.feast.core.CoreService_pb2 import (
     ApplyEntityRequest,
     ApplyEntityResponse,
     ApplyFeatureTableRequest,
@@ -24,10 +24,10 @@
     ListFeatureTablesResponse,
     ListProjectsResponse,
 )
-from feast.core.Entity_pb2 import Entity as EntityProto
-from feast.core.Entity_pb2 import EntityMeta
-from feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto
-from feast.core.FeatureTable_pb2 import FeatureTableMeta
+from feast.protos.feast.core.Entity_pb2 import Entity as EntityProto
+from feast.protos.feast.core.Entity_pb2 import EntityMeta
+from feast.protos.feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto
+from feast.protos.feast.core.FeatureTable_pb2 import FeatureTableMeta
 
 _logger = logging.getLogger(__name__)
 

sdk/python/tests/feast_serving_server.py

@@ -5,11 +5,11 @@
 
 import grpc
 
-from feast.core import FeatureTable_pb2 as FeatureTableProto
-from feast.core.CoreService_pb2 import ListFeatureTablesResponse
-from feast.core.CoreService_pb2_grpc import CoreServiceStub
-from feast.serving import ServingService_pb2_grpc as Serving
-from feast.serving.ServingService_pb2 import GetFeastServingInfoResponse
+from feast.protos.feast.core import FeatureTable_pb2 as FeatureTableProto
+from feast.protos.feast.core.CoreService_pb2 import ListFeatureTablesResponse
+from feast.protos.feast.core.CoreService_pb2_grpc import CoreServiceStub
+from feast.protos.feast.serving import ServingService_pb2_grpc as Serving
+from feast.protos.feast.serving.ServingService_pb2 import GetFeastServingInfoResponse
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 

sdk/python/tests/test_client.py

@@ -32,30 +32,36 @@
 from pytest_lazyfixture import lazy_fixture
 
 from feast.client import Client
-from feast.core import CoreService_pb2_grpc as Core
-from feast.core.CoreService_pb2 import (
-    GetFeastCoreVersionResponse,
-    GetFeatureTableResponse,
-    ListFeaturesResponse,
-)
-from feast.core.DataSource_pb2 import DataSource as DataSourceProto
-from feast.core.Feature_pb2 import FeatureSpecV2 as FeatureSpecProto
-from feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto
-from feast.core.FeatureTable_pb2 import FeatureTableMeta as FeatureTableMetaProto
-from feast.core.FeatureTable_pb2 import FeatureTableSpec as FeatureTableSpecProto
 from feast.data_format import ParquetFormat, ProtoFormat
 from feast.data_source import FileSource, KafkaSource
 from feast.entity import Entity
 from feast.feature import Feature
 from feast.feature_table import FeatureTable
-from feast.serving import ServingService_pb2_grpc as Serving
-from feast.serving.ServingService_pb2 import FeatureReferenceV2 as FeatureRefProto
-from feast.serving.ServingService_pb2 import (
+from feast.protos.feast.core import CoreService_pb2_grpc as Core
+from feast.protos.feast.core.CoreService_pb2 import (
+    GetFeastCoreVersionResponse,
+    GetFeatureTableResponse,
+    ListFeaturesResponse,
+)
+from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
+from feast.protos.feast.core.Feature_pb2 import FeatureSpecV2 as FeatureSpecProto
+from feast.protos.feast.core.FeatureTable_pb2 import FeatureTable as FeatureTableProto
+from feast.protos.feast.core.FeatureTable_pb2 import (
+    FeatureTableMeta as FeatureTableMetaProto,
+)
+from feast.protos.feast.core.FeatureTable_pb2 import (
+    FeatureTableSpec as FeatureTableSpecProto,
+)
+from feast.protos.feast.serving import ServingService_pb2_grpc as Serving
+from feast.protos.feast.serving.ServingService_pb2 import (
+    FeatureReferenceV2 as FeatureRefProto,
+)
+from feast.protos.feast.serving.ServingService_pb2 import (
     GetFeastServingInfoResponse,
     GetOnlineFeaturesRequestV2,
     GetOnlineFeaturesResponse,
 )
-from feast.types import Value_pb2 as ValueProto
+from feast.protos.feast.types import Value_pb2 as ValueProto
 from feast.value_type import ValueType
 from feast_core_server import (
     AllowAuthInterceptor,