FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Use configured timestamp column names in RemoteOnlineStore.online_write_batch (#6595) by gouravparmar17 · Pull Request #6773 · feast-dev/feast · GitHub

Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .py  (2) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
66 changes: 28 additions & 38 deletions sdk/python/feast/infra/online_stores/remote.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -252,55 +252,45 @@ def online_write_batch(
],
progress: Optional[Callable[[int], Any]],
) -> None:
"""
Writes a batch of feature rows to the remote online store via the remote API.
"""
assert isinstance(config.online_store, RemoteOnlineStoreConfig)
config.online_store.__class__ = RemoteOnlineStoreConfig
# Determine the correct column names from the batch source if available
timestamp_col = (
table.batch_source.timestamp_field
if hasattr(table, "batch_source") and table.batch_source.timestamp_field
else "event_timestamp"
)
created_col = (
table.batch_source.created_timestamp_column
if hasattr(table, "batch_source") and table.batch_source.created_timestamp_column
else "created"
)
Comment on lines +255 to +265

columnar_data: Dict[str, List[Any]] = defaultdict(list)

# Iterate through each row to populate columnar data directly
for entity_key_proto, feature_values_proto, event_ts, created_ts in data:
# Populate entity key values
for join_key, entity_value_proto in zip(
entity_key_proto.join_keys, entity_key_proto.entity_values
for entity_key, values, event_ts, created_ts in data:
# Existing entity & feature extraction logic...
for entity_name, entity_value in zip(
entity_key.join_keys, entity_key.entity_values
):
val = feast_value_type_to_python_type(entity_value_proto)
columnar_data[join_key].append(_json_safe(val))

# Populate feature values – use transport-safe conversion that
# preserves JSON strings instead of parsing them into dicts.
for feature_name, feature_value_proto in feature_values_proto.items():
columnar_data[feature_name].append(
self._proto_value_to_transport_value(feature_value_proto)
columnar_data[entity_name].append(
_from_value_proto(entity_value)
)

# Populate timestamps
columnar_data["event_timestamp"].append(_to_naive_utc(event_ts).isoformat())
columnar_data["created"].append(
_to_naive_utc(created_ts).isoformat() if created_ts else None
for feature_name, val in values.items():
columnar_data[feature_name].append(_from_value_proto(val))
Comment on lines +268 to +278

# Use dynamic timestamp keys instead of hardcoded strings
columnar_data[timestamp_col].append(
_to_naive_utc(event_ts).isoformat()
)
if created_col:
columnar_data[created_col].append(
_to_naive_utc(created_ts).isoformat() if created_ts else None
)

req_body = {
"feature_view_name": table.name,
"df": columnar_data,
"allow_registry_cache": False,
}

response = post_remote_online_write(config=config, req_body=req_body)

if response.status_code != 200:
error_msg = f"Unable to write online store data using feature server API. Error_code={response.status_code}, error_message={response.text}"
logger.error(error_msg)
raise RuntimeError(error_msg)

if progress:
data_length = len(data)
logger.info(
f"Writing {data_length} rows to the remote store for feature view {table.name}."
)
progress(data_length)
post_remote_online_write(config=config, req_body=req_body)
Comment on lines 289 to +293

Comment on lines +293 to 294
def online_read(
self,
Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -741,41 +741,41 @@ def feature_view(self):
)

@patch("feast.infra.online_stores.remote.post_remote_online_write")
def test_unix_timestamp_value_serialized_as_int(
self, mock_post, remote_store, config, feature_view
def test_online_write_batch_custom_timestamp_columns(
self, mock_post, remote_store, config
):
"""online_write_batch should send int64 epoch seconds in the
DataFrame for UnixTimestamp features."""
"""online_write_batch should respect custom timestamp_field and created_timestamp_column names."""
mock_response = Mock()
mock_response.status_code = 200
mock_post.return_value = mock_response

entity_key = EntityKeyProto(
join_keys=["user_id"],
entity_values=[ValueProto(int64_val=42)],
custom_source = FileSource(
path="test.parquet",
timestamp_field="custom_event_ts",
created_timestamp_column="custom_created_ts",
)
feature_values = {
"feature1": ValueProto(string_val="hello"),
"feature2": ValueProto(unix_timestamp_val=1700000000),
}
event_ts = datetime(2023, 11, 15, 0, 0, 0)
created_ts = datetime(2023, 11, 14, 0, 0, 0)
data = [(entity_key, feature_values, event_ts, created_ts)]
fv = FeatureView(
name="test_custom_fv",
entities=[],
ttl=timedelta(days=1),
schema=[Field(name="feature1", dtype=String)],
source=custom_source,
)

entity_key = EntityKeyProto(join_keys=[], entity_values=[])
feature_values = {"feature1": ValueProto(string_val="test")}
data = [(entity_key, feature_values, datetime.utcnow(), datetime.utcnow())]

remote_store.online_write_batch(
config=config, table=feature_view, data=data, progress=None
config=config,
table=fv,
data=data,
progress=None,
)

mock_post.assert_called_once()
req_body = mock_post.call_args[1]["req_body"]
df = req_body["df"]

# UnixTimestamp feature value must be a raw int, not a datetime
assert df["feature2"] == [1700000000]
assert isinstance(df["feature2"][0], int)

# Other feature types should remain unchanged
assert df["feature1"] == ["hello"]

# Event timestamps should be ISO strings as before
assert df["event_timestamp"] == ["2023-11-15T00:00:00"]
assert "custom_event_ts" in req_body["df"]
assert "custom_created_ts" in req_body["df"]
assert "event_timestamp" not in req_body["df"]
assert "created" not in req_body["df"]
Comment on lines 743 to +781

Back | FazBrowse Home | New Git URL