FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fix: Add spark to lambda dockerfile by achals · Pull Request #2480 · feast-dev/feast · GitHub

/ feast Public

fix: Add spark to lambda dockerfile #2480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 13 commits into from
Apr 4, 2022
Conversation 8 Commits 13 Checks 0 Files changed
Merged

fix: Add spark to lambda dockerfile #2480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
92caab0
add spark to lambda dockerfile
achals Apr 2, 2022
4aef576
add *args
achals Apr 2, 2022
cba63fb
Add *args
felixwang9817 Apr 2, 2022
01a5680
Pin protobuf==3.19.4
felixwang9817 Apr 2, 2022
e534b16
Remove *args
felixwang9817 Apr 2, 2022
886f450
Add a range
achals Apr 3, 2022
b8b94e1
Add a todo
achals Apr 4, 2022
47eb5a1
cleanup prints
achals Apr 4, 2022
4c0d78e
lock deps
achals Apr 4, 2022
997165e
lock deps correctly
achals Apr 4, 2022
992c3b8
fix lint
achals Apr 4, 2022
6badd47
fix lint take 2
achals Apr 4, 2022
dfe125e
Undo general updates
achals Apr 4, 2022
File filter

Filter by extension

Filter by extension .py  (3) .txt  (1) No extension  (1) All 3 file types selected
Only manifest files
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
3 changes: 2 additions & 1 deletion sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ COPY go go
COPY README.md README.md

# Install Feast for AWS with Lambda dependencies
RUN pip3 install -e 'sdk/python[aws,redis]'
# TODO(achals): The additional spark deps should be removed. Details at https://github.com/feast-dev/feast/pull/2480.
RUN pip3 install -e 'sdk/python[aws,redis,spark]'
RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}"

# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
Expand Down
8 changes: 5 additions & 3 deletions sdk/python/feast/infra/offline_stores/bigquery.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -338,12 +338,14 @@ def to_bigquery(

def _to_arrow_internal(self) -> pyarrow.Table:
with self._query_generator() as query:
return self._execute_query(query).to_arrow()
q = self._execute_query(query=query)
assert q
return q.to_arrow()

@log_exceptions_and_usage
def _execute_query(
self, query, job_config=None, timeout: int = 1800
) -> bigquery.job.query.QueryJob:
) -> Optional[bigquery.job.query.QueryJob]:
bq_job = self.client.query(query, job_config=job_config)

if job_config and job_config.dry_run:
Expand Down Expand Up @@ -426,7 +428,7 @@ def _get_table_reference_for_new_entity(
dataset.location = dataset_location if dataset_location else "US"

try:
client.get_dataset(dataset)
client.get_dataset(dataset.reference)
except NotFound:
# Only create the dataset if it does not exist
client.create_dataset(dataset, exists_ok=True)
Expand Down
8 changes: 5 additions & 3 deletions sdk/python/feast/proto_json.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
JsonObject = Any


# TODO: These methods need to be updated when bumping the version of protobuf.
# https://github.com/feast-dev/feast/issues/2484
def _patch_proto_json_encoding(
proto_type: Type[ProtoMessage],
to_json_object: Callable[[_Printer, ProtoMessage], JsonObject],
Expand Down Expand Up @@ -68,7 +70,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
return value

def from_json_object(
parser: _Parser, value: JsonObject, message: ProtoMessage
parser: _Parser, value: JsonObject, message: ProtoMessage,
) -> None:
if value is None:
message.null_val = 0
Expand Down Expand Up @@ -140,7 +142,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
return [printer._MessageToJsonObject(item) for item in message.val]

def from_json_object(
parser: _Parser, value: JsonObject, message: ProtoMessage
parser: _Parser, value: JsonObject, message: ProtoMessage,
) -> None:
array = value if isinstance(value, list) else value["val"]
for item in array:
Expand Down Expand Up @@ -181,7 +183,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
return list(message.val)

def from_json_object(
parser: _Parser, value: JsonObject, message: ProtoMessage
parser: _Parser, value: JsonObject, message: ProtoMessage,
) -> None:
array = value if isinstance(value, list) else value["val"]
message.val.extend(array)
Expand Down
4 changes: 2 additions & 2 deletions sdk/python/requirements/py3.7-ci-requirements.txt
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ mypy==0.931
# via feast (setup.py)
mypy-extensions==0.4.3
# via mypy
mypy-protobuf==3.1.0
mypy-protobuf==3.1
# via feast (setup.py)
nbclient==0.5.11
# via nbconvert
Expand Down Expand Up @@ -854,4 +854,4 @@ zipp==3.7.0

# The following packages are considered to be unsafe in a requirements file:
# pip
# setuptools
# setuptools
8 changes: 4 additions & 4 deletions sdk/python/setup.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"mmh3",
"pandas>=1.0.0",
"pandavro==1.5.*",
"protobuf>=3.10",
"protobuf>=3.10,<3.20",

felixwang9817 Apr 4, 2022

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

nit: do we want to also update py3.7-requirements.txt and other files?

achals Apr 4, 2022

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

oh yeah good point

achals Apr 4, 2022

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

just read through what the make lock-python-*dependencies command does and existing requirements file - I don't think any changes are needed here.
"proto-plus<1.19.7",
"pyarrow>=4.0.0",
"pydantic>=1.0.0",
Expand Down Expand Up @@ -111,7 +111,7 @@
"mock==2.0.0",
"moto",
"mypy==0.931",
"mypy-protobuf==3.1.0",
"mypy-protobuf==3.1",
"avro==1.10.0",
"gcsfs",
"urllib3>=1.25.4",
Expand Down Expand Up @@ -150,7 +150,7 @@
+ GE_REQUIRED
)

DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED
DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED

# Get git repo root directory
repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent)
Expand Down Expand Up @@ -422,7 +422,7 @@ def run(self):
"setuptools_scm",
"grpcio",
"grpcio-tools==1.34.0",
"mypy-protobuf==3.1.0",
"mypy-protobuf==3.1",
"sphinx!=4.0.0",
],
package_data={
Expand Down
Back | FazBrowse Home | New Git URL