feast-dev
@@ -3,7 +3,7 @@
 ## Motivation
 
 Feast uses an internal type system to provide guarantees on training and serving data.
-Feast supports primitive types, array types, and map types for feature values.
+Feast supports primitive types, array types, set types, and map types for feature values.
 Null types are not supported, although the `UNIX_TIMESTAMP` type is nullable.
 The type system is controlled by [`Value.proto`](https://github.com/feast-dev/feast/blob/master/protos/feast/types/Value.proto) in protobuf and by [`types.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/types.py) in Python.
 Type conversion logic can be found in [`type_map.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/type_map.py).
@@ -40,6 +40,23 @@ All primitive types have corresponding array (list) types:
 | `Array(Bool)` | `List[bool]` | List of booleans |
 | `Array(UnixTimestamp)` | `List[datetime]` | List of timestamps |
 
+### Set Types
+
+All primitive types (except Map) have corresponding set types for storing unique values:
+
+| Feast Type | Python Type | Description |
+|------------|-------------|-------------|
+| `Set(Int32)` | `Set[int]` | Set of unique 32-bit integers |
+| `Set(Int64)` | `Set[int]` | Set of unique 64-bit integers |
+| `Set(Float32)` | `Set[float]` | Set of unique 32-bit floats |
+| `Set(Float64)` | `Set[float]` | Set of unique 64-bit floats |
+| `Set(String)` | `Set[str]` | Set of unique strings |
+| `Set(Bytes)` | `Set[bytes]` | Set of unique binary data |
+| `Set(Bool)` | `Set[bool]` | Set of unique booleans |
+| `Set(UnixTimestamp)` | `Set[datetime]` | Set of unique timestamps |
+
+**Note:** Set types automatically remove duplicate values. When converting from lists or other iterables to sets, duplicates are eliminated.
+
 ### Map Types
 
 Map types allow storing dictionary-like data structures:
@@ -60,7 +77,7 @@ from datetime import timedelta
 from feast import Entity, FeatureView, Field, FileSource
 from feast.types import (
     Int32, Int64, Float32, Float64, String, Bytes, Bool, UnixTimestamp,
-    Array, Map
+    Array, Set, Map
 )
 
 # Define a data source
@@ -101,6 +118,12 @@ user_features = FeatureView(
         Field(name="notification_settings", dtype=Array(Bool)),
         Field(name="login_timestamps", dtype=Array(UnixTimestamp)),
 
+        # Set types (unique values only)
+        Field(name="visited_pages", dtype=Set(String)),
+        Field(name="unique_categories", dtype=Set(Int32)),
+        Field(name="tag_ids", dtype=Set(Int64)),
+        Field(name="preferred_languages", dtype=Set(String)),
+
         # Map types
         Field(name="user_preferences", dtype=Map),
         Field(name="metadata", dtype=Map),
@@ -110,6 +133,24 @@ user_features = FeatureView(
 )
 ```
 
+### Set Type Usage Examples
+
+Sets store unique values and automatically remove duplicates:
+
+```python
+# Simple set
+visited_pages = {"home", "products", "checkout", "products"}  # "products" appears twice
+# Feast will store this as: {"home", "products", "checkout"}
+
+# Integer set
+unique_categories = {1, 2, 3, 2, 1}  # duplicates will be removed
+# Feast will store this as: {1, 2, 3}
+
+# Converting a list with duplicates to a set
+tag_list = [100, 200, 300, 100, 200]
+tag_ids = set(tag_list)  # {100, 200, 300}
+```
+
 ### Map Type Usage Examples
 
 Maps can store complex nested data structures:

@@ -45,6 +45,14 @@ message ValueType {
     NULL = 19;
     MAP = 20;
     MAP_LIST = 21;
+    BYTES_SET = 22;
+    STRING_SET = 23;
+    INT32_SET = 24;
+    INT64_SET = 25;
+    DOUBLE_SET = 26;
+    FLOAT_SET = 27;
+    BOOL_SET = 28;
+    UNIX_TIMESTAMP_SET = 29;
   }
 }
 
@@ -72,6 +80,14 @@ message Value {
     Null null_val = 19;
     Map map_val = 20;
     MapList map_list_val = 21;
+    BytesSet bytes_set_val = 22;
+    StringSet string_set_val = 23;
+    Int32Set int32_set_val = 24;
+    Int64Set int64_set_val = 25;
+    DoubleSet double_set_val = 26;
+    FloatSet float_set_val = 27;
+    BoolSet bool_set_val = 28;
+    Int64Set unix_timestamp_set_val = 29;
   }
 }
 
@@ -107,6 +123,34 @@ message BoolList {
   repeated bool val = 1;
 }
 
+message BytesSet {
+  repeated bytes val = 1;
+}
+
+message StringSet {
+  repeated string val = 1;
+}
+
+message Int32Set {
+  repeated int32 val = 1;
+}
+
+message Int64Set {
+  repeated int64 val = 1;
+}
+
+message DoubleSet {
+  repeated double val = 1;
+}
+
+message FloatSet {
+  repeated float val = 1;
+}
+
+message BoolSet {
+  repeated bool val = 1;
+}
+
 message Map {
   map<string, Value> val = 1;
 }

@@ -167,6 +167,8 @@ def _get_or_create_collection(
             fields_to_exclude = [
                 "event_ts",
                 "created_ts",
+                "event_timestamp",
+                "created_timestamp",
             ]
             fields_to_add = [f for f in table.schema if f.name not in fields_to_exclude]
             for field in fields_to_add:
@@ -202,6 +204,7 @@ def _get_or_create_collection(
                     schema=schema,
                 )
                 index_params = self.client.prepare_index_params()
+                indices_added = False
                 for vector_field in schema.fields:
                     if (
                         vector_field.dtype
@@ -222,7 +225,8 @@ def _get_or_create_collection(
                             index_name=f"vector_index_{vector_field.name}",
                             params={"nlist": config.online_store.nlist},
                         )
-                if len(index_params) > 0:
+                        indices_added = True
+                if indices_added:
                     self.client.create_index(
                         collection_name=collection_name,
                         index_params=index_params,
@@ -281,6 +285,16 @@ def online_write_batch(
                 serialize_to_string=True,
             )
 
+            # Remove timestamp fields that are handled separately to avoid conflicts
+            timestamp_fields = [
+                "event_timestamp",
+                "created_timestamp",
+                "event_ts",
+                "created_ts",
+            ]
+            for field in timestamp_fields:
+                values_dict.pop(field, None)
+
             single_entity_record = {
                 composite_key_name: entity_key_str,
                 "event_ts": timestamp_int,
@@ -722,7 +736,7 @@ def _extract_proto_values_to_dict(
     numeric_vector_list_types = [
         k
         for k in PROTO_VALUE_TO_VALUE_TYPE_MAP.keys()
-        if k is not None and "list" in k and "string" not in k
+        if k is not None and ("list" in k or "set" in k) and "string" not in k
     ]
     numeric_types = [
         "double_val",
@@ -747,9 +761,13 @@ def _extract_proto_values_to_dict(
                         if (
                             serialize_to_string
                             and proto_val_type
-                            not in ["string_val", "bytes_val"] + numeric_types
+                            not in ["string_val", "bytes_val", "unix_timestamp_val"]
+                            + numeric_types
                         ):
-                            vector_values = feature_values.SerializeToString().decode()
+                            # For complex types, use base64 encoding instead of decode
+                            vector_values = base64.b64encode(
+                                feature_values.SerializeToString()
+                            ).decode("utf-8")
                         elif proto_val_type == "bytes_val":
                             byte_data = getattr(feature_values, proto_val_type)
                             vector_values = base64.b64encode(byte_data).decode("utf-8")

@@ -1,19 +1,19 @@
 """
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
-
-* Copyright 2021 The Feast Authors
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     https://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
+
+* Copyright 2021 The Feast Authors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
 * limitations under the License.
 """
 import builtins

@@ -1,19 +1,19 @@
 """
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
-
-* Copyright 2020 The Feast Authors
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     https://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
+
+* Copyright 2020 The Feast Authors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
 * limitations under the License.
 """
 import builtins

@@ -19,7 +19,7 @@ else:
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
 class FeatureViewProjection(google.protobuf.message.Message):
-    """A projection to be applied on top of a FeatureView. 
+    """A projection to be applied on top of a FeatureView. 
     Contains the modifications to a FeatureView such as the features subset to use.
     """
 

@@ -1,19 +1,19 @@
 """
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
-
-* Copyright 2020 The Feast Authors
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     https://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
+
+* Copyright 2020 The Feast Authors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
 * limitations under the License.
 """
 import builtins

@@ -1,19 +1,19 @@
 """
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
-
-* Copyright 2020 The Feast Authors
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     https://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
+
+* Copyright 2020 The Feast Authors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
 * limitations under the License.
 """
 import builtins