FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
feast/sdk/python/feast/dataframe.py at v0.62.0 · feast-dev/feast · GitHub
Uh oh!
There was an error while loading.
Please reload this page
.
feast-dev
/
feast
Public
Notifications
You must be signed in to change notification settings
Fork
1.4k
Star
7.2k
Code
Issues
215
Pull requests
190
Discussions
Actions
Security and quality
1
Insights
Additional navigation options
Code
Issues
Pull requests
Discussions
Actions
Security and quality
Insights
Expand file tree
Breadcrumbs
feast
/
sdk
/
python
/
feast
/
dataframe.py
Copy path
More file actions
More file actions
Latest commit
History
History
History
96 lines (79 loc) · 2.95 KB
Breadcrumbs
feast
/
sdk
/
python
/
feast
/
dataframe.py
Copy path
File metadata and controls
96 lines (79 loc) · 2.95 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
"""FeastDataFrame: A lightweight container for DataFrame-like objects in Feast."""
from
enum
import
Enum
from
typing
import
Any
,
Dict
,
Optional
import
pandas
as
pd
import
pyarrow
as
pa
class
DataFrameEngine
(
str
,
Enum
):
"""Supported DataFrame engines."""
PANDAS
=
"pandas"
SPARK
=
"spark"
DASK
=
"dask"
RAY
=
"ray"
ARROW
=
"arrow"
POLARS
=
"polars"
UNKNOWN
=
"unknown"
class
FeastDataFrame
:
"""
A lightweight container for DataFrame-like objects in Feast.
This class wraps any DataFrame implementation and provides metadata
about the engine type for proper routing in Feast's processing pipeline.
"""
def
__init__
(
self
,
data
:
Any
,
engine
:
Optional
[
DataFrameEngine
]
=
None
,
metadata
:
Optional
[
Dict
[
str
,
Any
]]
=
None
,
):
"""
Initialize a FeastDataFrame.
Args:
data: The wrapped DataFrame object (pandas, Spark, Dask, etc.)
engine: Explicitly specify the engine type (auto-detected if None)
metadata: Additional metadata (schema hints, etc.)
"""
self
.
data
=
data
self
.
metadata
=
metadata
or
{}
# Detect the actual engine from the data
detected_engine
=
self
.
_detect_engine
()
if
engine
is
not
None
:
# Validate that the provided engine matches the detected engine
if
engine
!=
detected_engine
:
raise
ValueError
(
f"Provided engine '
{
engine
.
value
}
' does not match detected engine '
{
detected_engine
.
value
}
' "
f"for data type
{
type
(
data
).
__name__
}
"
)
self
.
_engine
=
engine
else
:
self
.
_engine
=
detected_engine
def
_detect_engine
(
self
)
->
DataFrameEngine
:
"""Auto-detect the DataFrame engine based on type."""
if
isinstance
(
self
.
data
,
pd
.
DataFrame
):
return
DataFrameEngine
.
PANDAS
elif
isinstance
(
self
.
data
,
pa
.
Table
):
return
DataFrameEngine
.
ARROW
# For optional dependencies, check module name to avoid import errors
module
=
type
(
self
.
data
).
__module__
if
"pyspark"
in
module
:
return
DataFrameEngine
.
SPARK
elif
"dask"
in
module
:
return
DataFrameEngine
.
DASK
elif
"ray"
in
module
:
return
DataFrameEngine
.
RAY
elif
"polars"
in
module
:
return
DataFrameEngine
.
POLARS
else
:
return
DataFrameEngine
.
UNKNOWN
@
property
def
engine
(
self
)
->
DataFrameEngine
:
"""Get the detected or specified engine type."""
return
self
.
_engine
def
__repr__
(
self
):
return
f"FeastDataFrame(engine=
{
self
.
engine
.
value
}
, type=
{
type
(
self
.
data
).
__name__
}
)"
@
property
def
is_lazy
(
self
)
->
bool
:
"""Check if the underlying DataFrame is lazy (Spark, Dask, Ray)."""
return
self
.
engine
in
[
DataFrameEngine
.
SPARK
,
DataFrameEngine
.
DASK
,
DataFrameEngine
.
RAY
,
]
Back
|
FazBrowse Home
|
New Git URL