| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -3064,18 +3064,13 @@ def _retrieve_from_online_store_v2( | |||
| 3064 | 3064 | entity_key_dict, | |
| 3065 | 3065 | ) | |
| 3066 | 3066 | ||
| 3067 | - feature_data = utils._convert_rows_to_protobuf( | ||
| 3068 | - requested_features=features_to_request, | ||
| 3069 | - read_rows=list(zip(datevals, list_of_feature_dicts)), | ||
| 3070 | - ) | ||
| 3071 | - | ||
| 3072 | 3067 | online_features_response = GetOnlineFeaturesResponse(results=[]) | |
| 3073 | 3068 | utils._populate_response_from_feature_data( | |
| 3074 | - feature_data=feature_data, | ||
| 3069 | + requested_features=features_to_request, | ||
| 3070 | + read_rows=list(zip(datevals, list_of_feature_dicts)), | ||
| 3075 | 3071 | indexes=idxs, | |
| 3076 | 3072 | online_features_response=online_features_response, | |
| 3077 | 3073 | full_feature_names=False, | |
| 3078 | - requested_features=features_to_request, | ||
| 3079 | 3074 | table=table, | |
| 3080 | 3075 | output_len=output_len, | |
| 3081 | 3076 | include_feature_view_version_metadata=include_feature_view_version_metadata, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -220,17 +220,12 @@ def get_online_features( | |||
| 220 | 220 | requested_features=requested_features, | |
| 221 | 221 | ) | |
| 222 | 222 | ||
| 223 | - feature_data = utils._convert_rows_to_protobuf( | ||
| 224 | - requested_features, read_rows | ||
| 225 | - ) | ||
| 226 | - | ||
| 227 | - # Populate the result_rows with the Features from the OnlineStore inplace. | ||
| 228 | 223 | utils._populate_response_from_feature_data( | |
| 229 | - feature_data, | ||
| 224 | + requested_features, | ||
| 225 | + read_rows, | ||
| 230 | 226 | idxs, | |
| 231 | 227 | online_features_response, | |
| 232 | 228 | full_feature_names, | |
| 233 | - requested_features, | ||
| 234 | 229 | table, | |
| 235 | 230 | output_len, | |
| 236 | 231 | include_feature_view_version_metadata, | |
@@ -356,17 +351,12 @@ async def query_table(table, requested_features): | |||
| 356 | 351 | for (idxs, read_rows, output_len), (table, requested_features) in zip( | |
| 357 | 352 | all_responses, grouped_refs | |
| 358 | 353 | ): | |
| 359 | - feature_data = utils._convert_rows_to_protobuf( | ||
| 360 | - requested_features, read_rows | ||
| 361 | - ) | ||
| 362 | - | ||
| 363 | - # Populate the result_rows with the Features from the OnlineStore inplace. | ||
| 364 | 354 | utils._populate_response_from_feature_data( | |
| 365 | - feature_data, | ||
| 355 | + requested_features, | ||
| 356 | + read_rows, | ||
| 366 | 357 | idxs, | |
| 367 | 358 | online_features_response, | |
| 368 | 359 | full_feature_names, | |
| 369 | - requested_features, | ||
| 370 | 360 | table, | |
| 371 | 361 | output_len, | |
| 372 | 362 | include_feature_view_version_metadata, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -666,30 +666,6 @@ def _group_feature_refs( | |||
| 666 | 666 | return fvs_result, odfvs_result | |
| 667 | 667 | ||
| 668 | 668 | ||
| 669 | - def construct_response_feature_vector( | ||
| 670 | - values_vector: Iterable[Any], | ||
| 671 | - statuses_vector: Iterable[Any], | ||
| 672 | - timestamp_vector: Iterable[Any], | ||
| 673 | - mapping_indexes: Iterable[List[int]], | ||
| 674 | - output_len: int, | ||
| 675 | - ) -> GetOnlineFeaturesResponse.FeatureVector: | ||
| 676 | - values_output: Iterable[Any] = [None] * output_len | ||
| 677 | - statuses_output: Iterable[Any] = [None] * output_len | ||
| 678 | - timestamp_output: Iterable[Any] = [None] * output_len | ||
| 679 | - | ||
| 680 | - for i, destinations in enumerate(mapping_indexes): | ||
| 681 | - for idx in destinations: | ||
| 682 | - values_output[idx] = values_vector[i] # type: ignore[index] | ||
| 683 | - statuses_output[idx] = statuses_vector[i] # type: ignore[index] | ||
| 684 | - timestamp_output[idx] = timestamp_vector[i] # type: ignore[index] | ||
| 685 | - | ||
| 686 | - return GetOnlineFeaturesResponse.FeatureVector( | ||
| 687 | - values=values_output, | ||
| 688 | - statuses=statuses_output, | ||
| 689 | - event_timestamps=timestamp_output, | ||
| 690 | - ) | ||
| 691 | - | ||
| 692 | - | ||
| 693 | 669 | def _apply_aggregations_to_response( | |
| 694 | 670 | response_data: Union[pyarrow.Table, Dict[str, List[Any]]], | |
| 695 | 671 | aggregations, | |
@@ -1130,115 +1106,6 @@ def ensure_request_data_values_exist( | |||
| 1130 | 1106 | raise RequestDataNotFoundInEntityRowsException(feature_names=missing_features) | |
| 1131 | 1107 | ||
| 1132 | 1108 | ||
| 1133 | - def _populate_response_from_feature_data( | ||
| 1134 | - feature_data: Iterable[ | ||
| 1135 | - Tuple[ | ||
| 1136 | - Iterable[Timestamp], Iterable["FieldStatus.ValueType"], Iterable[ValueProto] | ||
| 1137 | - ] | ||
| 1138 | - ], | ||
| 1139 | - indexes: Iterable[List[int]], | ||
| 1140 | - online_features_response: GetOnlineFeaturesResponse, | ||
| 1141 | - full_feature_names: bool, | ||
| 1142 | - requested_features: Iterable[str], | ||
| 1143 | - table: "FeatureView", | ||
| 1144 | - output_len: int, | ||
| 1145 | - include_feature_view_version_metadata: bool = False, | ||
| 1146 | - ): | ||
| 1147 | - """Populate the GetOnlineFeaturesResponse with feature data. | ||
| 1148 | - | ||
| 1149 | - This method assumes that `_read_from_online_store` returns data for each | ||
| 1150 | - combination of Entities in `entity_rows` in the same order as they | ||
| 1151 | - are provided. | ||
| 1152 | - | ||
| 1153 | - Args: | ||
| 1154 | - feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. | ||
| 1155 | - indexes: A list of indexes which should be the same length as `feature_data`. Each list | ||
| 1156 | - of indexes corresponds to a set of result rows in `online_features_response`. | ||
| 1157 | - online_features_response: The object to populate. | ||
| 1158 | - full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, | ||
| 1159 | - changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to | ||
| 1160 | - "customer_fv__daily_transactions"). | ||
| 1161 | - requested_features: The names of the features in `feature_data`. This should be ordered in the same way as the | ||
| 1162 | - data in `feature_data`. | ||
| 1163 | - table: The FeatureView that `feature_data` was retrieved from. | ||
| 1164 | - output_len: The number of result rows in `online_features_response`. | ||
| 1165 | - """ | ||
| 1166 | - # Add the feature names to the response. | ||
| 1167 | - # Use name_to_use() which includes version tag (e.g. "fv@v2") when a | ||
| 1168 | - # version-qualified ref was used, so multi-version queries produce | ||
| 1169 | - # distinct column names like "fv@v1__feat" and "fv@v2__feat". | ||
| 1170 | - table_name = table.projection.name_to_use() | ||
| 1171 | - clean_table_name = table.projection.name_alias or table.projection.name | ||
| 1172 | - requested_feature_refs = [ | ||
| 1173 | - f"{table_name}__{feature_name}" if full_feature_names else feature_name | ||
| 1174 | - for feature_name in requested_features | ||
| 1175 | - ] | ||
| 1176 | - online_features_response.metadata.feature_names.val.extend(requested_feature_refs) | ||
| 1177 | - | ||
| 1178 | - # Add version metadata if requested | ||
| 1179 | - if include_feature_view_version_metadata: | ||
| 1180 | - # Check if this feature view already exists in metadata to avoid duplicates | ||
| 1181 | - existing_names = [ | ||
| 1182 | - fvm.name for fvm in online_features_response.metadata.feature_view_metadata | ||
| 1183 | - ] | ||
| 1184 | - if clean_table_name not in existing_names: | ||
| 1185 | - fv_metadata = online_features_response.metadata.feature_view_metadata.add() | ||
| 1186 | - fv_metadata.name = clean_table_name | ||
| 1187 | - # Extract version from the table's current_version_number attribute | ||
| 1188 | - fv_metadata.version = getattr(table, "current_version_number", 0) or 0 | ||
| 1189 | - | ||
| 1190 | - # Process each feature vector in a single pass | ||
| 1191 | - for timestamp_vector, statuses_vector, values_vector in feature_data: | ||
| 1192 | - response_vector = construct_response_feature_vector( | ||
| 1193 | - values_vector, statuses_vector, timestamp_vector, indexes, output_len | ||
| 1194 | - ) | ||
| 1195 | - online_features_response.results.append(response_vector) | ||
| 1196 | - | ||
| 1197 | - | ||
| 1198 | - def _populate_response_from_feature_data_v2( | ||
| 1199 | - feature_data: Iterable[ | ||
| 1200 | - Tuple[ | ||
| 1201 | - Iterable[Timestamp], Iterable["FieldStatus.ValueType"], Iterable[ValueProto] | ||
| 1202 | - ] | ||
| 1203 | - ], | ||
| 1204 | - indexes: Iterable[List[int]], | ||
| 1205 | - online_features_response: GetOnlineFeaturesResponse, | ||
| 1206 | - requested_features: Iterable[str], | ||
| 1207 | - output_len: int, | ||
| 1208 | - ): | ||
| 1209 | - """Populate the GetOnlineFeaturesResponse with feature data. | ||
| 1210 | - | ||
| 1211 | - This method assumes that `_read_from_online_store` returns data for each | ||
| 1212 | - combination of Entities in `entity_rows` in the same order as they | ||
| 1213 | - are provided. | ||
| 1214 | - | ||
| 1215 | - Args: | ||
| 1216 | - feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. | ||
| 1217 | - indexes: A list of indexes which should be the same length as `feature_data`. Each list | ||
| 1218 | - of indexes corresponds to a set of result rows in `online_features_response`. | ||
| 1219 | - online_features_response: The object to populate. | ||
| 1220 | - full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, | ||
| 1221 | - changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to | ||
| 1222 | - "customer_fv__daily_transactions"). | ||
| 1223 | - requested_features: The names of the features in `feature_data`. This should be ordered in the same way as the | ||
| 1224 | - data in `feature_data`. | ||
| 1225 | - output_len: The number of result rows in `online_features_response`. | ||
| 1226 | - """ | ||
| 1227 | - # Add the feature names to the response. | ||
| 1228 | - requested_feature_refs = [(feature_name) for feature_name in requested_features] | ||
| 1229 | - online_features_response.metadata.feature_names.val.extend(requested_feature_refs) | ||
| 1230 | - | ||
| 1231 | - timestamps, statuses, values = zip(*feature_data) | ||
| 1232 | - | ||
| 1233 | - # Populate the result with data fetched from the OnlineStore | ||
| 1234 | - # which is guaranteed to be aligned with `requested_features`. | ||
| 1235 | - for timestamp_vector, statuses_vector, values_vector in feature_data: | ||
| 1236 | - response_vector = construct_response_feature_vector( | ||
| 1237 | - values_vector, statuses_vector, timestamp_vector, indexes, output_len | ||
| 1238 | - ) | ||
| 1239 | - online_features_response.results.append(response_vector) | ||
| 1240 | - | ||
| 1241 | - | ||
| 1242 | 1109 | def _convert_entity_key_to_proto_to_dict( | |
| 1243 | 1110 | entity_key_vals: List[EntityKeyProto], | |
| 1244 | 1111 | ) -> Dict[str, List[ValueProto]]: | |
@@ -1612,36 +1479,99 @@ def _get_entity_key_protos( | |||
| 1612 | 1479 | return entity_key_protos | |
| 1613 | 1480 | ||
| 1614 | 1481 | ||
| 1615 | - def _convert_rows_to_protobuf( | ||
| 1482 | + def _populate_response_from_feature_data( | ||
| 1616 | 1483 | requested_features: List[str], | |
| 1617 | 1484 | read_rows: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]], | |
| 1618 | - ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[ValueProto]]]: | ||
| 1619 | - n_rows = len(read_rows) | ||
| 1485 | + indexes: Iterable[List[int]], | ||
| 1486 | + online_features_response: GetOnlineFeaturesResponse, | ||
| 1487 | + full_feature_names: bool, | ||
| 1488 | + table: "FeatureView", | ||
| 1489 | + output_len: int, | ||
| 1490 | + include_feature_view_version_metadata: bool = False, | ||
| 1491 | + ): | ||
| 1492 | + """Populate the GetOnlineFeaturesResponse from raw online_read rows. | ||
| 1493 | + | ||
| 1494 | + Converts raw rows from the OnlineStore into protobuf FeatureVectors and | ||
| 1495 | + appends them to the response. This method assumes that ``online_read`` | ||
| 1496 | + returns data for each unique entity in the same order as ``indexes``. | ||
| 1497 | + | ||
| 1498 | + Args: | ||
| 1499 | + requested_features: The names of the features to extract from | ||
| 1500 | + each row. Determines the order of FeatureVectors in the response. | ||
| 1501 | + read_rows: Raw output from ``OnlineStore.online_read`` — a list of | ||
| 1502 | + ``(event_timestamp, feature_dict)`` tuples, one per unique entity. | ||
| 1503 | + ``feature_dict`` may be ``None`` when the entity is not found. | ||
| 1504 | + indexes: A tuple of lists that maps each unique entity (by position | ||
| 1505 | + in ``read_rows``) to one or more output positions in the response. | ||
| 1506 | + Used to fan-out deduplicated reads back to the original request rows. | ||
| 1507 | + online_features_response: The protobuf response object to populate. | ||
| 1508 | + full_feature_names: If True, feature names are prefixed with the | ||
| 1509 | + feature view name (e.g. ``"driver_fv__trips_today"``). | ||
| 1510 | + table: The FeatureView that ``read_rows`` was retrieved from. | ||
| 1511 | + output_len: Total number of result rows in the response. | ||
| 1512 | + include_feature_view_version_metadata: If True, version metadata | ||
| 1513 | + for the feature view is added to the response. | ||
| 1514 | + """ | ||
| 1515 | + n_features = len(requested_features) | ||
| 1516 | + | ||
| 1517 | + table_name = table.projection.name_to_use() | ||
| 1518 | + clean_table_name = table.projection.name_alias or table.projection.name | ||
| 1519 | + feature_refs = [ | ||
| 1520 | + f"{table_name}__{fn}" if full_feature_names else fn for fn in requested_features | ||
| 1521 | + ] | ||
| 1522 | + online_features_response.metadata.feature_names.val.extend(feature_refs) | ||
| 1523 | + | ||
| 1524 | + if include_feature_view_version_metadata: | ||
| 1525 | + existing_names = [ | ||
| 1526 | + fvm.name for fvm in online_features_response.metadata.feature_view_metadata | ||
| 1527 | + ] | ||
| 1528 | + if clean_table_name not in existing_names: | ||
| 1529 | + fv_metadata = online_features_response.metadata.feature_view_metadata.add() | ||
| 1530 | + fv_metadata.name = clean_table_name | ||
| 1531 | + fv_metadata.version = getattr(table, "current_version_number", 0) or 0 | ||
| 1620 | 1532 | ||
| 1621 | 1533 | null_value = ValueProto() | |
| 1622 | - null_status = FieldStatus.NOT_FOUND | ||
| 1623 | - present_status = FieldStatus.PRESENT | ||
| 1534 | + null_ts = Timestamp() | ||
| 1535 | + PRESENT = FieldStatus.PRESENT | ||
| 1536 | + NOT_FOUND = FieldStatus.NOT_FOUND | ||
| 1624 | 1537 | ||
| 1625 | - # Pre-compute timestamps once per entity (not per feature) | ||
| 1626 | - # This reduces O(features * entities) to O(entities) for timestamp conversion | ||
| 1627 | - row_timestamps = [] | ||
| 1538 | + row_ts_protos = [] | ||
| 1628 | 1539 | for row_ts, _ in read_rows: | |
| 1629 | - ts_proto = Timestamp() | ||
| 1540 | + ts = Timestamp() | ||
| 1630 | 1541 | if row_ts is not None: | |
| 1631 | - ts_proto.FromDatetime(row_ts) | ||
| 1632 | - row_timestamps.append(ts_proto) | ||
| 1633 | - | ||
| 1634 | - requested_features_vectors = [] | ||
| 1635 | - for feature_name in requested_features: | ||
| 1636 | - ts_vector = list(row_timestamps) # Shallow copy of pre-computed timestamps | ||
| 1637 | - status_vector = [null_status] * n_rows | ||
| 1638 | - value_vector = [null_value] * n_rows | ||
| 1639 | - for idx, (_, feature_data) in enumerate(read_rows): | ||
| 1640 | - if (feature_data is not None) and (feature_name in feature_data): | ||
| 1641 | - status_vector[idx] = present_status | ||
| 1642 | - value_vector[idx] = feature_data[feature_name] | ||
| 1643 | - requested_features_vectors.append((ts_vector, status_vector, value_vector)) | ||
| 1644 | - return requested_features_vectors | ||
| 1542 | + ts.FromDatetime(row_ts) | ||
| 1543 | + row_ts_protos.append(ts) | ||
| 1544 | + | ||
| 1545 | + ts_template = [null_ts] * output_len | ||
| 1546 | + indexes_tuple = tuple(indexes) | ||
| 1547 | + for row_idx, destinations in enumerate(indexes_tuple): | ||
| 1548 | + ts = row_ts_protos[row_idx] | ||
| 1549 | + for out_idx in destinations: | ||
| 1550 | + ts_template[out_idx] = ts | ||
| 1551 | + | ||
| 1552 | + feat_values = [[null_value] * output_len for _ in range(n_features)] | ||
| 1553 | + feat_statuses = [[NOT_FOUND] * output_len for _ in range(n_features)] | ||
| 1554 | + | ||
| 1555 | + feat_idx_map = {name: i for i, name in enumerate(requested_features)} | ||
| 1556 | + for row_idx, destinations in enumerate(indexes_tuple): | ||
| 1557 | + _, feature_data = read_rows[row_idx] | ||
| 1558 | + if feature_data is None: | ||
| 1559 | + continue | ||
| 1560 | + for feat_name, feat_val in feature_data.items(): | ||
| 1561 | + f_idx = feat_idx_map.get(feat_name) | ||
| 1562 | + if f_idx is not None: | ||
| 1563 | + for out_idx in destinations: | ||
| 1564 | + feat_values[f_idx][out_idx] = feat_val | ||
| 1565 | + feat_statuses[f_idx][out_idx] = PRESENT | ||
| 1566 | + | ||
| 1567 | + for f_idx in range(n_features): | ||
| 1568 | + online_features_response.results.append( | ||
| 1569 | + GetOnlineFeaturesResponse.FeatureVector( | ||
| 1570 | + values=feat_values[f_idx], | ||
| 1571 | + statuses=feat_statuses[f_idx], | ||
| 1572 | + event_timestamps=list(ts_template), | ||
| 1573 | + ) | ||
| 1574 | + ) | ||
| 1645 | 1575 | ||
| 1646 | 1576 | ||
| 1647 | 1577 | def has_all_tags( | |
| Back | FazBrowse Home | New Git URL |
0 commit comments