Add custom data sources

protos/feast/core/DataSource.proto

@@ -26,6 +26,10 @@ import "feast/core/DataFormat.proto";
 
 // Defines a Data Source that can be used source Feature data
 message DataSource {
+  // Field indexes should *not* be reused. Not sure if fields 6-10 were used previously or not,
+  // but they are going to be reserved for backwards compatibility.
+  reserved 6 to 10;
+
   // Type of Data Source.
   enum SourceType {
     INVALID = 0;
@@ -34,6 +38,7 @@ message DataSource {
     STREAM_KAFKA = 3;
     STREAM_KINESIS = 4;
     BATCH_REDSHIFT = 5;
+    CUSTOM_SOURCE = 6;
   }
   SourceType type = 1;
 
@@ -51,6 +56,10 @@ message DataSource {
   // Must specify creation timestamp column name
   string created_timestamp_column = 5;
 
+  // This is an internal field that is represents the python class for the data source object a proto object represents.
+  // This should be set by feast, and not by users.
+  string data_source_class_type = 17;
+
   // Defines options for DataSource that sources features from a file
   message FileOptions {
     FileFormat file_format = 1;
@@ -111,12 +120,20 @@ message DataSource {
     string query = 2;
   }
 
+  // Defines configuration for custom third-party data sources.
+  message CustomSourceOptions {
+    // Serialized configuration information for the data source. The implementer of the custom data source is
+    // responsible for serializing and deserializing data from bytes
+    bytes configuration = 1;
+  }
+
   // DataSource options.
   oneof options {
     FileOptions file_options = 11;
     BigQueryOptions bigquery_options = 12;
     KafkaOptions kafka_options = 13;
     KinesisOptions kinesis_options = 14;
     RedshiftOptions redshift_options = 15;
+    CustomSourceOptions custom_options = 16;
   }
 }

sdk/python/feast/__init__.py

@@ -2,15 +2,12 @@
 
 from pkg_resources import DistributionNotFound, get_distribution
 
+from feast.infra.offline_stores.bigquery import BigQuerySource
+from feast.infra.offline_stores.file import FileSource
+from feast.infra.offline_stores.redshift import RedshiftSource
+
 from .client import Client
-from .data_source import (
-    BigQuerySource,
-    FileSource,
-    KafkaSource,
-    KinesisSource,
-    RedshiftSource,
-    SourceType,
-)
+from .data_source import KafkaSource, KinesisSource, SourceType
 from .entity import Entity
 from .feature import Feature
 from .feature_store import FeatureStore
@@ -32,10 +29,9 @@
     pass
 
 __all__ = [
+    "BigQuerySource",
     "Client",
     "Entity",
-    "BigQuerySource",
-    "FileSource",
     "KafkaSource",
     "KinesisSource",
     "RedshiftSource",
@@ -46,4 +42,5 @@
     "RepoConfig",
     "SourceType",
     "ValueType",
+    "FileSource",
 ]

sdk/python/feast/client.py

@@ -20,10 +20,10 @@
 import grpc
 import pandas as pd
 
+from feast import BigQuerySource, FileSource
 from feast.config import Config
 from feast.constants import ConfigOptions as opt
 from feast.data_format import ParquetFormat
-from feast.data_source import BigQuerySource, FileSource
 from feast.entity import Entity
 from feast.feature import Feature, FeatureRef, _build_feature_references
 from feast.feature_table import FeatureTable