[ Web Proxy ]
URL:
Viewing: https://raw.githubusercontent.com/cloud-orbitor/feast/master/sdk/python/feast/embedder.py [Back]  [Original]

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    List,
    Optional,
    Protocol,
    runtime_checkable,
)

import pandas as pd

if TYPE_CHECKING:
    import numpy as np

    from feast.repo_config import EmbeddingModelConfig


@runtime_checkable
class EmbeddingProvider(Protocol):
    """Protocol for query-time embedding providers.

    Implement this to plug in any embedding backend (OpenAI, Cohere,
    SentenceTransformers, a local model, etc.) for use in
    ``openai_search`` and the OpenAI-compatible
    feature server endpoints.

    Example::

        class MyEmbeddingProvider:
            def embed(self, texts: List[str]) -> List[List[float]]:
                return my_model.encode(texts)

            async def aembed(self, texts: List[str]) -> List[List[float]]:
                return await my_model.aencode(texts)
    """

    def embed(self, texts: List[str]) -> List[List[float]]:
        """Return one embedding vector per input text."""
        ...

    async def aembed(self, texts: List[str]) -> List[List[float]]:
        """Async variant of :meth:`embed`."""
        ...


class SentenceTransformersEmbeddingProvider:
    """EmbeddingProvider backed by Sentence Transformers (local inference).

    Runs entirely on-device  no external API key required.  Ideal for
    air-gapped deployments, cost-sensitive workloads, or environments where
    outbound API calls are restricted.

    Requires the ``sentence-transformers`` package::

        pip install sentence-transformers

    Configuration in ``feature_store.yaml``::

        embedding_model:
          provider: sentence_transformers
          model: all-MiniLM-L6-v2   # any HuggingFace sentence-transformers model

    The model is loaded lazily on the first call to :meth:`embed` or
    :meth:`aembed` so that import cost is paid only when the provider is
    actually used.
    """

    def __init__(self, model: str = "all-MiniLM-L6-v2"):
        self.model_name = model
        self._model = None

    @classmethod
    def from_config(
        cls, config: "EmbeddingModelConfig"
    ) -> "SentenceTransformersEmbeddingProvider":
        """Build a provider from a :class:`~feast.repo_config.EmbeddingModelConfig`."""
        return cls(model=config.model)

    def _load_model(self):
        if self._model is None:
            try:
                from sentence_transformers import SentenceTransformer
            except ImportError:
                raise ImportError(
                    "sentence-transformers is required for the "
                    "'sentence_transformers' embedding provider. "
                    "Install with: pip install sentence-transformers"
                )
            self._model = SentenceTransformer(self.model_name)
        return self._model

    def embed(self, texts: List[str]) -> List[List[float]]:
        model = self._load_model()
        embeddings = model.encode(texts, convert_to_numpy=True)
        return [emb.tolist() for emb in embeddings]

    async def aembed(self, texts: List[str]) -> List[List[float]]:
        import asyncio

        loop = asyncio.get_event_loop()
        return await loop.run_in_executor(None, self.embed, texts)


def get_embedding_provider(
    config: "EmbeddingModelConfig",
) -> "EmbeddingProvider":
    """Factory that returns the appropriate :class:`EmbeddingProvider` for *config*.

    Dispatches on ``config.provider``:

    * ``"sentence_transformers"`` (default)  :class:`SentenceTransformersEmbeddingProvider`

    Raises:
        ValueError: If ``config.provider`` is not a recognised value.
    """
    provider = (config.provider or "sentence_transformers").lower()
    if provider == "sentence_transformers":
        return SentenceTransformersEmbeddingProvider.from_config(config)
    raise ValueError(
        f"Unknown embedding provider: '{config.provider}'. "
        "Supported values: 'sentence_transformers'."
    )


@dataclass
class EmbeddingConfig:
    batch_size: int = 64
    show_progress: bool = True


class BaseEmbedder(ABC):
    """
    Abstract base class for embedding generation.

    Supports multiple modalities via routing.
    Users can register custom modality handlers.
    """

    def __init__(self, config: Optional[EmbeddingConfig] = None):
        self.config = config or EmbeddingConfig()

        # Registry: modality -> embedding function
        self._modality_handlers: dict[str, Callable[[List[Any]], "np.ndarray"]] = {}

        # Register default modalities (subclass can override)
        self._register_default_modalities()

    def _register_default_modalities(self) -> None:
        """Override in subclass to register default modality handlers."""
        pass

    def register_modality(
        self,
        modality: str,
        handler: Callable[[List[Any]], "np.ndarray"],
    ) -> None:
        """
        Register a handler for a modality.

        Args:
            modality: Name of modality ("text", "image", "video", etc.)
            handler: Function that takes list of inputs and returns embeddings.
        """
        self._modality_handlers[modality] = handler

    @property
    def supported_modalities(self) -> List[str]:
        """Return list of supported modalities."""
        return list(self._modality_handlers.keys())

    def get_embedding_dim(self, modality: str) -> Optional[int]:
        """
        Return the embedding dimension for a given modality.

        Subclasses should override this to return the actual dimension
        so that auto-generated FeatureView schemas use the correct vector_length.

        Args:
            modality: The modality to query (e.g. "text", "image").

        Returns:
            The embedding dimension, or None if unknown.
        """
        return None

    @abstractmethod
    def embed(self, inputs: List[Any], modality: str) -> "np.ndarray":
        """
        Generate embeddings for inputs of a given modality.

        Args:
            inputs: List of inputs.
            modality: Type of content ("text", "image", "video", etc.)

        Returns:
            numpy array of shape (len(inputs), embedding_dim)
        """
        pass

    def embed_dataframe(
        self,
        df: pd.DataFrame,
        column_mapping: dict[str, tuple[str, str]],
    ) -> pd.DataFrame:
        """
        Add embeddings for multiple columns with modality routing.

        Args:
            df: Input DataFrame.
            column_mapping: Dict mapping source_column -> (modality, output_column).
                Example: {
                    "text": ("text", "text_embedding"),
                    "image_path": ("image", "image_embedding"),
                    "video_path": ("video", "video_embedding"),
                }
        """
        df = df.copy()

        for source_column, (modality, output_column) in column_mapping.items():
            inputs = df[source_column].tolist()
            embeddings = self.embed(inputs, modality)
            df[output_column] = pd.Series(
                [emb.tolist() for emb in embeddings], dtype=object, index=df.index
            )

        return df


class MultiModalEmbedder(BaseEmbedder):
    """
    Multi-modal embedder with built-in support for common modalities.

    Supports: text, image, video (extensible)
    """

    def __init__(
        self,
        text_model: str = "all-MiniLM-L6-v2",
        image_model: str = "openai/clip-vit-base-patch32",
        config: Optional[EmbeddingConfig] = None,
    ):
        self.text_model_name = text_model
        self.image_model_name = image_model

        # Lazy-loaded models
        self._text_model = None
        self._image_model = None
        self._image_processor = None

        super().__init__(config)

    def _register_default_modalities(self) -> None:
        """Register built-in modality handlers."""
        self.register_modality("text", self._embed_text)
        self.register_modality("image", self._embed_image)
        # Future: add more as needed
        # self.register_modality("video", self._embed_video)
        # self.register_modality("audio", self._embed_audio)

    def embed(self, inputs: List[Any], modality: str) -> "np.ndarray":
        """Route to appropriate handler based on modality."""
        if modality not in self._modality_handlers:
            raise ValueError(
                f"Unsupported modality: '{modality}'. "
                f"Supported: {self.supported_modalities}"
            )

        handler = self._modality_handlers[modality]
        return handler(inputs)

    def get_embedding_dim(self, modality: str) -> Optional[int]:
        """
        Return the embedding dimension for a given modality.

        For "text", this queries the SentenceTransformer model's dimension
        (which triggers lazy model loading).

        Args:
            modality: The modality to query (e.g. "text", "image").

        Returns:
            The embedding dimension, or None if unknown.
        """
        if modality == "text":
            return self.text_model.get_sentence_embedding_dimension()
        elif modality == "image":
            return self.image_model.config.vision_config.hidden_size
        return None

    # Text Embedding
    @property
    def text_model(self):
        if self._text_model is None:
            from sentence_transformers import SentenceTransformer

            self._text_model = SentenceTransformer(self.text_model_name)
        return self._text_model

    def _embed_text(self, inputs: List[str]) -> "np.ndarray":
        return self.text_model.encode(
            inputs,
            batch_size=self.config.batch_size,
            show_progress_bar=self.config.show_progress,
        )

    # Image Embedding
    @property
    def image_model(self):
        if self._image_model is None:
            from transformers import CLIPModel

            self._image_model = CLIPModel.from_pretrained(self.image_model_name)
        return self._image_model

    @property
    def image_processor(self):
        if self._image_processor is None:
            from transformers import CLIPProcessor

            self._image_processor = CLIPProcessor.from_pretrained(self.image_model_name)
        return self._image_processor

    def _embed_image(self, inputs: List[Any]) -> "np.ndarray":
        from pathlib import Path

        import numpy as np
        from PIL import Image

        all_embeddings: List["np.ndarray"] = []
        batch_size = self.config.batch_size

        for start in range(0, len(inputs), batch_size):
            batch = inputs[start : start + batch_size]
            images = []
            opened: List[Image.Image] = []
            try:
                for inp in batch:
                    if isinstance(
                        inp, (str, Path)
                    ):  # If the input string path is too large that It gives error and we could not open the image.
                        img = Image.open(inp)
                        opened.append(img)
                        images.append(img)
                    else:
                        images.append(inp)

                processed = self.image_processor(images=images, return_tensors="pt")
            finally:
                for opened_img in opened:
                    opened_img.close()

            embeddings = self.image_model.get_image_features(**processed)
            embeddings = embeddings / embeddings.norm(p=2, dim=-1, keepdim=True)
            all_embeddings.append(embeddings.detach().numpy())

        return np.concatenate(all_embeddings, axis=0)

Web Proxy Viewer  |  New URL  |  Original Page