# syntax = docker/dockerfile:experimental

# Multistage Dockerfile to build the Splitgraph engine with debugging capabilities.

# For now does not include MySQL and Mongo FDWs

# The underlying splitgraph/pg_debug:development image is built via

# Dockerfile.pg_debug, and the reason for this separation is so that we don't

# need to rebuild PG on every change in Multicorn/FDW instances.

#####

##### cstore_fdw and multicorn

#####

FROM splitgraph/pg_debug:development AS builder_cstore_multicorn

COPY ./engine/src/cstore_fdw /src/cstore_fdw

COPY ./engine/src/Multicorn /src/Multicorn

COPY ./engine/build_scripts/fdws/multicorn /build

COPY ./engine/build_scripts/fdws/cstore_fdw /build

RUN /build/build_cstore_fdw.sh

RUN /build/build_multicorn.sh

#####

##### splitgraph/engine

#####

FROM splitgraph/pg_debug:development

# We still have to install some runtime libraries here, but no dev.

RUN apt-get update -qq && \

apt-get install -y \

libprotobuf-c1 \

libpython3.7 \

postgresql-plpython3-12 && \

curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \

python3.7 get-pip.py && \

rm get-pip.py && \

rm -rf /var/lib/apt/lists/*

# Set locale to C instead of en-US, Postgres initdb breaks with default

# locale otherwise + we get build warnings for FDWs.

ENV LANG C.UTF-8

ENV LC_ALL C.UTF-8

# Install Splitgraph itself (required for layered querying)

RUN mkdir -p /var/lib/splitgraph/objects && chmod 777 /var/lib/splitgraph/objects

# Do the installation in two phases to use Docker caching: first,

# install all the dependencies and then copy the Splitgraph code over (way later in the build).

# That way, if we just change the code, we only need to rebuild one last layer rather than

# reinstall all dependencies from scratch. We also do dependency installation before

# copying over various FDW SOs, since those are cheaper to copy and can get rebuilt more often.

# However, we need to make Poetry think that we actually have a genuine package

# that we want to install in editable mode in here, so we create and copy some files to fool it.

RUN mkdir -p /splitgraph/splitgraph && touch /splitgraph/splitgraph/__init__.py

COPY poetry.lock pyproject.toml README.md /splitgraph/

COPY ./engine/build_scripts/build_splitgraph.sh /build/

RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip /build/build_splitgraph.sh

# Extract and copy over all binaries from the builder stages

# This is slightly sad: whilst multicorn's make install respects DESTDIR,

# python setup.py doesn't and installs it to /usr/local.

# Since nothing else in here uses easy_install, we grab the .pth file and the unpacked egg.

COPY --from=builder_cstore_multicorn /output/root /

COPY --from=builder_cstore_multicorn \

/usr/local/lib/python3.7/dist-packages/easy-install.pth \

/usr/local/lib/python3.7/dist-packages/easy-install.pth

COPY --from=builder_cstore_multicorn \

/usr/local/lib/python3.7/dist-packages/multicorn-1.4.0.dev0-py3.7-linux-x86_64.egg \

/usr/local/lib/python3.7/dist-packages/multicorn-1.4.0.dev0-py3.7-linux-x86_64.egg

# Also copy over the source code for GDB to be able to print it while

# traversing through code, or e.g. executing the `list` command

COPY --from=builder_cstore_multicorn /src/Multicorn /src/Multicorn

COPY --from=builder_cstore_multicorn /src/cstore_fdw /src/cstore_fdw

# NB important: the plpython installation upgrades postgresql and overwrites our binaries,

# so we have to get them back.

COPY --from=splitgraph/pg_debug:development /usr/lib/postgresql/12/bin /usr/lib/postgresql/12/bin

COPY --from=splitgraph/pg_debug:development /usr/lib/postgresql/12/lib /usr/lib/postgresql/12/lib

RUN ldconfig

# Set up Postgres config files/envvars/init scripts

EXPOSE 5432

ENV POSTGRES_DB splitgraph

ENV POSTGRES_USER sgr

COPY ./engine/etc /etc/

COPY ./engine/init_scripts /docker-entrypoint-initdb.d/

# We don't install Mongo/MySQL extensions in the debug image

RUN sed -i '/\(mongo_fdw\|mysql_fdw\)/d' /docker-entrypoint-initdb.d/000_create_extensions.sql

# Copy the actual Splitgraph code over at this point.

COPY ./splitgraph /splitgraph/splitgraph

COPY ./bin /splitgraph/bin

# "Install" elasticsearch_fdw

RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip \

mkdir /pg_es_fdw && \

pip install "elasticsearch>=7.7.0,<8.0"

COPY ./engine/src/postgres-elasticsearch-fdw/pg_es_fdw /pg_es_fdw/pg_es_fdw

# Install the Snowflake SQLAlchemy connector

# Use our fork that supports server-side cursors

RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip \

pip install "git+https://github.com/splitgraph/snowflake-sqlalchemy.git@14e64cc0ef7374df0cecc91923ff6901b0d721b7"

# Install PyAthena for Amazon Athena SQLAlchemy-based FDW, as well as pandas

RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip \

pip install "PyAthena>=2.4.1" && \

pip install "pandas>=1.0.0"

# Install Google's Big Query SQLAlchemy dialect lib

RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip \

pip install "sqlalchemy-bigquery"

ENV PATH "${PATH}:/splitgraph/bin"

ENV PYTHONPATH "${PYTHONPATH}:/splitgraph:/pg_es_fdw"

# https://github.com/postgis/docker-postgis/blob/master/12-3.2/Dockerfile

ARG with_postgis

RUN test -z "${with_postgis}" || (\

export POSTGIS_MAJOR=3 && \

export POSTGIS_VERSION=3.2.3+dfsg-1.pgdg100+1 && \

apt-get update \

&& apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \

&& apt-get install -y --no-install-recommends \

postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \

postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts=$POSTGIS_VERSION \

&& rm -rf /var/lib/apt/lists/* && \

echo "CREATE EXTENSION postgis;" >> /docker-entrypoint-initdb.d/000_create_extensions.sql)

CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql_debug.conf"]