[ Web Proxy ]
URL:
Viewing: https://raw.githubusercontent.com/feast-dev/feast/codex/flink-doc-fix/sdk/python/feast/chunker.py [Back]  [Original]

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, Optional

import pandas as pd


@dataclass
class ChunkingConfig:
    chunk_size: int = 100
    chunk_overlap: int = 20
    min_chunk_size: int = 20
    max_chunk_chars: Optional[int] = 500


class BaseChunker(ABC):
    """
    Abstract base class for document chunking.

    Subclasses implement load_parse_and_chunk() with their own:
    - Loading logic
    - Parsing logic
    - Chunking strategy
    """

    def __init__(self, config: Optional[ChunkingConfig] = None):
        self.config = config or ChunkingConfig()

    @abstractmethod
    def load_parse_and_chunk(
        self,
        source: Any,
        source_id: str,
        source_column: str,
        source_type: Optional[str] = None,
    ) -> list[dict]:
        """
        Load, parse, and chunk a document.

        Args:
            source: File path, raw text, bytes, etc.
            source_id: Document identifier.
            source_type: Optional type hint.
            source_column: The column containing the document sources.

        Returns:
            List of chunk dicts with keys:
                - chunk_id: str
                - original_id: str
                - text: str
                - chunk_index: int
                - (any additional metadata)
        """
        pass

    def chunk_dataframe(
        self,
        df: pd.DataFrame,
        id_column: str,
        source_column: str,
        type_column: Optional[str] = None,
    ) -> pd.DataFrame:
        """
        Chunk all documents in a DataFrame.

        Args:
            df: The DataFrame containing the documents to chunk.
            id_column: The column containing the document IDs.
            source_column: The column containing the document sources.
            type_column: The column containing the document types.
        """

        all_chunks = []
        for row in df.itertuples(index=False):
            chunks = self.load_parse_and_chunk(
                getattr(row, source_column),
                str(getattr(row, id_column)),
                source_column,
                getattr(row, type_column) if type_column else None,
            )
            all_chunks.extend(chunks)

        if not all_chunks:
            return pd.DataFrame(
                columns=["chunk_id", "original_id", source_column, "chunk_index"]
            )
        return pd.DataFrame(all_chunks)


class TextChunker(BaseChunker):
    """Default chunker for plain text. Chunks by word count."""

    def load_parse_and_chunk(
        self,
        source: Any,
        source_id: str,
        source_column: str,
        source_type: Optional[str] = None,
    ) -> list[dict]:
        # Load
        text = self._load(source)

        # Chunk by words
        return self._chunk_by_words(text, source_id, source_column)

    def _load(self, source: Any) -> str:
        from pathlib import Path

        if isinstance(source, Path) and source.exists():
            return Path(source).read_text()
        if isinstance(source, str):
            if source.endswith(".txt") and Path(source).exists():
                return Path(source).read_text()
        return str(source)

    def _chunk_by_words(
        self, text: str, source_id: str, source_column: str
    ) -> list[dict]:
        words = text.split()
        chunks = []

        step = self.config.chunk_size - self.config.chunk_overlap
        if step 

Web Proxy Viewer  |  New URL  |  Original Page