FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat: Add PostgreSQL online store support for Go feature server by samuelkim7 · Pull Request #5963 · feast-dev/feast · GitHub

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .go  (3) .mod  (1) .sum  (1) All 3 file types selected
Only manifest files
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
4 changes: 4 additions & 0 deletions go.mod
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.8.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
Expand Down
14 changes: 14 additions & 0 deletions go.sum
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls=
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
Expand Down Expand Up @@ -137,6 +138,14 @@ github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
Expand Down Expand Up @@ -165,6 +174,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
Expand All @@ -180,8 +190,11 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
Expand Down Expand Up @@ -265,5 +278,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
5 changes: 4 additions & 1 deletion go/internal/feast/onlinestore/onlinestore.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,11 @@ func NewOnlineStore(config *registry.RepoConfig) (OnlineStore, error) {
} else if onlineStoreType == "dynamodb" {
onlineStore, err := NewDynamodbOnlineStore(config.Project, config, config.OnlineStore)
return onlineStore, err
} else if onlineStoreType == "postgres" {
onlineStore, err := NewPostgresOnlineStore(config.Project, config, config.OnlineStore)
return onlineStore, err
} else {
return nil, fmt.Errorf("%s online store type is currently not supported; only redis and sqlite are supported", onlineStoreType)
return nil, fmt.Errorf("%s online store type is currently not supported; only redis, sqlite, dynamodb, and postgres are supported", onlineStoreType)
}
}

Expand Down
197 changes: 197 additions & 0 deletions go/internal/feast/onlinestore/postgresonlinestore.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
package onlinestore

import (
"context"
"fmt"
"net/url"
"strings"
"time"

"github.com/feast-dev/feast/go/internal/feast/registry"
"github.com/feast-dev/feast/go/protos/feast/serving"
"github.com/feast-dev/feast/go/protos/feast/types"
"github.com/jackc/pgx/v5/pgxpool"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)

type PostgresOnlineStore struct {
project string
pool *pgxpool.Pool
config *registry.RepoConfig
}

func NewPostgresOnlineStore(project string, config *registry.RepoConfig, onlineStoreConfig map[string]interface{}) (*PostgresOnlineStore, error) {
connStr := buildPostgresConnString(onlineStoreConfig)
poolConfig, err := pgxpool.ParseConfig(connStr)
if err != nil {
return nil, fmt.Errorf("failed to parse postgres config: %w", err)
}

if schema, ok := onlineStoreConfig["db_schema"].(string); ok && schema != "" {
poolConfig.ConnConfig.RuntimeParams["search_path"] = schema
}

pool, err := pgxpool.NewWithConfig(context.Background(), poolConfig)
if err != nil {
return nil, fmt.Errorf("failed to create postgres pool: %w", err)
}

return &PostgresOnlineStore{
project: project,
pool: pool,
config: config,
}, nil
}

func (p *PostgresOnlineStore) Destruct() {
if p.pool != nil {
p.pool.Close()
}
}

func (p *PostgresOnlineStore) OnlineRead(ctx context.Context, entityKeys []*types.EntityKey, featureViewNames []string, featureNames []string) ([][]FeatureData, error) {
featureCount := len(featureNames)
results := make([][]FeatureData, len(entityKeys))

serializedKeys := make([][]byte, len(entityKeys))
entityKeyMap := make(map[string]int, len(entityKeys))

for i, entityKey := range entityKeys {
serKey, err := serializeEntityKey(entityKey, p.config.EntityKeySerializationVersion)
if err != nil {
return nil, err
}
serializedKeys[i] = *serKey
entityKeyMap[string(*serKey)] = i
}

type featureRef struct {
name string
index int
}
featuresByView := make(map[string][]featureRef)
for i, viewName := range featureViewNames {
featuresByView[viewName] = append(featuresByView[viewName], featureRef{
name: featureNames[i],
index: i,
})
}

for viewName, features := range featuresByView {
featureNamesToIdx := make(map[string]int, len(features))
for _, f := range features {
featureNamesToIdx[f.name] = f.index
}

tableName := fmt.Sprintf(`"%s"`, strings.ReplaceAll(tableId(p.project, viewName), `"`, `""`))
query := fmt.Sprintf(
`SELECT entity_key, feature_name, value, event_ts FROM %s WHERE entity_key = ANY($1)`,
tableName,
)

rows, err := p.pool.Query(ctx, query, serializedKeys)
if err != nil {
return nil, fmt.Errorf("failed to query postgres: %w", err)
}

for rows.Next() {
var entityKeyBytes []byte
var featureName string
var valueBytes []byte
var eventTs time.Time

if err := rows.Scan(&entityKeyBytes, &featureName, &valueBytes, &eventTs); err != nil {
rows.Close()
return nil, fmt.Errorf("failed to scan postgres row: %w", err)
}

rowIdx, ok := entityKeyMap[string(entityKeyBytes)]
if !ok {
continue
}

if results[rowIdx] == nil {
results[rowIdx] = make([]FeatureData, featureCount)
}

if featureIdx, ok := featureNamesToIdx[featureName]; ok {
var value types.Value
if err := proto.Unmarshal(valueBytes, &value); err != nil {
rows.Close()
return nil, fmt.Errorf("failed to unmarshal feature value: %w", err)
}

results[rowIdx][featureIdx] = FeatureData{
Reference: serving.FeatureReferenceV2{FeatureViewName: viewName, FeatureName: featureName},
Timestamp: *timestamppb.New(eventTs),
Value: types.Value{Val: value.Val},
}
}
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating postgres rows: %w", err)
}
}

return results, nil
}

func buildPostgresConnString(config map[string]interface{}) string {
host, _ := config["host"].(string)
if host == "" {
host = "localhost"
}

port := 5432
if p, ok := config["port"].(float64); ok {
port = int(p)
}

database, _ := config["database"].(string)
user, _ := config["user"].(string)
password, _ := config["password"].(string)

var userInfo *url.Userinfo
if user != "" {
if password != "" {
userInfo = url.UserPassword(user, password)
} else {
userInfo = url.User(user)
}
}

query := url.Values{}
if sslMode, ok := config["sslmode"].(string); ok && sslMode != "" {
query.Set("sslmode", sslMode)
} else {
query.Set("sslmode", "disable")
}

if v, ok := config["sslcert_path"].(string); ok && v != "" {
query.Set("sslcert", v)
}
if v, ok := config["sslkey_path"].(string); ok && v != "" {
query.Set("sslkey", v)
}
if v, ok := config["sslrootcert_path"].(string); ok && v != "" {
query.Set("sslrootcert", v)
}
if v, ok := config["min_conn"].(float64); ok {
query.Set("pool_min_conns", fmt.Sprintf("%d", int(v)))
}
if v, ok := config["max_conn"].(float64); ok {
query.Set("pool_max_conns", fmt.Sprintf("%d", int(v)))
}

connURL := url.URL{
Scheme: "postgres",
User: userInfo,
Host: fmt.Sprintf("%s:%d", host, port),
Path: database,
RawQuery: query.Encode(),
}

return connURL.String()
}
Loading
Loading

Back | FazBrowse Home | New Git URL