ARG BASE_IMAGE_REPO

ARG BASE_IMAGE_TAG

ARG CPU_BASE_IMAGE_NAME

ARG GPU_BASE_IMAGE_NAME

ARG LIGHTGBM_VERSION

ARG TORCH_VERSION

ARG TORCHAUDIO_VERSION

ARG TORCHTEXT_VERSION

ARG TORCHVISION_VERSION

ARG JAX_VERSION

{{ if eq .Accelerator "gpu" }}

FROM gcr.io/kaggle-images/python-lightgbm-whl:${GPU_BASE_IMAGE_NAME}-${BASE_IMAGE_TAG}-${LIGHTGBM_VERSION} AS lightgbm_whl

FROM gcr.io/kaggle-images/python-torch-whl:${GPU_BASE_IMAGE_NAME}-${BASE_IMAGE_TAG}-${TORCH_VERSION} AS torch_whl

FROM gcr.io/kaggle-images/python-jaxlib-whl:${GPU_BASE_IMAGE_NAME}-${BASE_IMAGE_TAG}-${JAX_VERSION} AS jaxlib_whl

FROM ${BASE_IMAGE_REPO}/${GPU_BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}

{{ else }}

FROM ${BASE_IMAGE_REPO}/${CPU_BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}

{{ end }}

# Ensures shared libraries installed with conda can be found by the dynamic link loader.

ENV LIBRARY_PATH="$LIBRARY_PATH:/opt/conda/lib"

ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"

{{ if eq .Accelerator "gpu" }}

ARG CUDA_MAJOR_VERSION

ARG CUDA_MINOR_VERSION

ENV CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION}

ENV CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION}

# Make sure we are on the right version of CUDA

RUN update-alternatives --set cuda /usr/local/cuda-$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION

# NVIDIA binaries from the host are mounted to /opt/bin.

ENV PATH=/opt/bin:${PATH}

# Add CUDA stubs to LD_LIBRARY_PATH to support building the GPU image on a CPU machine.

ENV LD_LIBRARY_PATH_NO_STUBS="$LD_LIBRARY_PATH"

ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs"

RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1

{{ end }}

# Keep these variables in sync if base image is updated.

ENV TENSORFLOW_VERSION=2.15.0

# See https://github.com/tensorflow/io#tensorflow-version-compatibility

ENV TENSORFLOW_IO_VERSION=0.35.0

# We need to redefine the ARG here to get the ARG value defined above the FROM instruction.

# See: https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact

ARG LIGHTGBM_VERSION

ARG TORCH_VERSION

ARG TORCHAUDIO_VERSION

ARG TORCHTEXT_VERSION

ARG TORCHVISION_VERSION

ARG JAX_VERSION

# Disable pesky logs like: KMP_AFFINITY: pid 6121 tid 6121 thread 0 bound to OS proc set 0

# See: https://stackoverflow.com/questions/57385766/disable-tensorflow-log-information

ENV KMP_WARNINGS=0

# Also make the KMP logs noverbose.

# https://stackoverflow.com/questions/70250304/stop-tensorflow-from-printing-warning-message

ENV KMP_SETTINGS=false

# Remove the pip as the root user warning.

ENV PIP_ROOT_USER_ACTION=ignore

ADD clean-layer.sh /tmp/clean-layer.sh

ADD patches/keras_patch.sh /tmp/keras_patch.sh

ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl

ADD patches/template_conf.json /opt/kaggle/conf.json

# b/276344496: Install specific version of boto3, because 1.26.103 is broken.

RUN pip install boto3==1.26.100 && \

/tmp/clean-layer.sh

{{ if eq .Accelerator "gpu" }}

# b/200968891 Keeps horovod once torch is upgraded.

RUN pip uninstall -y horovod && \

/tmp/clean-layer.sh

{{ end }}

# Update GPG key per documentation at https://cloud.google.com/compute/docs/troubleshooting/known-issues

RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -

# Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections,

# as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346

RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \

apt-get update --allow-releaseinfo-change && \

# Needed by lightGBM (GPU build)

# https://lightgbm.readthedocs.io/en/latest/GPU-Tutorial.html#build-lightgbm

apt-get install -y build-essential unzip cmake libboost-dev libboost-system-dev libboost-filesystem-dev p7zip-full && \

# b/182601974: ssh client was removed from the base image but is required for packages such as stable-baselines.

apt-get install -y openssh-client && \

/tmp/clean-layer.sh

# b/128333086: Set PROJ_LIB to points to the proj4 cartographic library.

ENV PROJ_LIB=/opt/conda/share/proj

# Install conda packages not available on pip.

# When using pip in a conda environment, conda commands should be ran first and then

# the remaining pip commands: https://www.anaconda.com/using-pip-in-a-conda-environment/

RUN conda config --add channels nvidia && \

conda config --add channels rapidsai && \

# b/299991198 remove curl/libcurl install once DLVM base image includes version >= 7.86

conda install -c conda-forge mamba curl libcurl && \

# Base image channel order: conda-forge (highest priority), defaults.

# End state: rapidsai (highest priority), nvidia, conda-forge, defaults.

mamba install -y mkl cartopy imagemagick pyproj "shapely<2" && \

/tmp/clean-layer.sh

# Install spacy

{{ if eq .Accelerator "gpu" }}

RUN mamba install -y -c conda-forge spacy cupy cuda-version=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION && \

/tmp/clean-layer.sh

{{ else }}

RUN pip install spacy && \

/tmp/clean-layer.sh

{{ end}}

{{ if eq .Accelerator "gpu" }}

# b/232247930: uninstall pyarrow to avoid double installation with the GPU specific version.

RUN pip uninstall -y pyarrow && \

mamba install -y cudf cuml && \

/tmp/clean-layer.sh

# TODO: b/296444923 - Resolve pandas dependency another way

RUN sed -i 's/^is_extension_type/# is_extension_type/g' /opt/conda/lib/python3.10/site-packages/cudf/api/types.py \

&& sed -i 's/^is_categorical/# is_categorical/g' /opt/conda/lib/python3.10/site-packages/cudf/api/types.py

{{ end }}

# Install PyTorch

{{ if eq .Accelerator "gpu" }}

COPY --from=torch_whl /tmp/whl/*.whl /tmp/torch/

RUN rm /opt/conda/lib/python3.10/site-packages/google*/direct_url.json

RUN rm /opt/conda/lib/python3.10/site-packages/google*/REQUESTED

# dlib has a libmkl incompatibility:

# test_dlib_face_detector (test_dlib.TestDLib) ... INTEL MKL ERROR: /opt/conda/bin/../lib/libmkl_avx512.so.2: undefined symbol: mkl_sparse_optimize_bsr_trsm_i8.

# Intel MKL FATAL ERROR: Cannot load libmkl_avx512.so.2 or libmkl_def.so.2.

# nnabla breaks protobuf compatibiilty:

RUN pip install flashtext \

wandb \

# b/214080882 blake3 0.3.0 is not compatible with vaex.

blake3==0.2.1 \

vaex \

pyemd \

pyupset \

pympler \

featuretools \

#-e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper \

git+https://github.com/Kaggle/learntools \

ray \

gym \

pyarabic \

pandasql \

# b/302136621 Fix eli5 import for learntools

scikit-learn==1.2.2 \

hpsklearn \

kmapper \

# b/329869023 shap 0.45.0 breaks learntools

shap==0.44.1 \

cesium \

rgf_python \

jieba \

# ggplot is broken and main repo does not merge and release https://github.com/yhat/ggpy/pull/668

https://github.com/hbasria/ggpy/archive/0.11.5.zip \

tsfresh \

pykalman \

optuna \

plotly_express \

albumentations \

accelerate \

# b/290207097 switch back to the pip catalyst package when bug fixed

# https://github.com/catalyst-team/catalyst/issues/1440

git+https://github.com/Philmod/catalyst.git@fix-fp16#egg=catalyst \

osmnx && \

apt-get -y install libspatialindex-dev

RUN pip install pytorch-ignite \

qgrid \

bqplot \

earthengine-api \

transformers \

datasets \

s3fs \

gcsfs \

kaggle-environments \

geopandas \

"shapely<2" \

vowpalwabbit \

pydub \

pydegensac \

torchmetrics \

pytorch-lightning \

sympy \

# flask is used by agents in the simulation competitions.

flask \

# pycrypto is used by competitions team.

pycryptodome \

easyocr \

# ipympl adds interactive widget support for matplotlib

ipympl==0.7.0 \

onnx \

tables \

openpyxl \

timm \

torchinfo && \

pip install git+https://github.com/facebookresearch/segment-anything.git && \

/tmp/clean-layer.sh

# Download base easyocr models.

# https://github.com/JaidedAI/EasyOCR#usage

RUN mkdir -p /root/.EasyOCR/model && \

wget --no-verbose "https://github.com/JaidedAI/EasyOCR/releases/download/v1.3/latin_g2.zip" -O /root/.EasyOCR/model/latin.zip && \

unzip /root/.EasyOCR/model/latin.zip -d /root/.EasyOCR/model/ && \

rm /root/.EasyOCR/model/latin.zip && \

wget --no-verbose "https://github.com/JaidedAI/EasyOCR/releases/download/v1.3/english_g2.zip" -O /root/.EasyOCR/model/english.zip && \

unzip /root/.EasyOCR/model/english.zip -d /root/.EasyOCR/model/ && \

rm /root/.EasyOCR/model/english.zip && \

wget --no-verbose "https://github.com/JaidedAI/EasyOCR/releases/download/pre-v1.1.6/craft_mlt_25k.zip" -O /root/.EasyOCR/model/craft_mlt_25k.zip && \

unzip /root/.EasyOCR/model/craft_mlt_25k.zip -d /root/.EasyOCR/model/ && \

rm /root/.EasyOCR/model/craft_mlt_25k.zip && \

/tmp/clean-layer.sh

# Tesseract and some associated utility packages

RUN apt-get install tesseract-ocr -y && \

pip install pytesseract \

wand \

pdf2image \

PyPDF && \

/tmp/clean-layer.sh

ENV TESSERACT_PATH=/usr/bin/tesseract

# For Facets

ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/

# For Theano with MKL

ENV MKL_THREADING_LAYER=GNU

# Temporary fixes and patches

# Temporary patch for Dask getting downgraded, which breaks Keras

RUN pip install --upgrade dask && \

# Stop jupyter nbconvert trying to rewrite its folder hierarchy

mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \

mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \

# Stop Matplotlib printing junk to the console on first load

sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.10/site-packages/matplotlib/font_manager.py && \

# Make matplotlib output in Jupyter notebooks display correctly

mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \

# Temporary patch for broken libpixman 0.38 in conda-forge, symlink to system libpixman 0.34 untile conda package gets updated to 0.38.5 or higher.

ln -sf /usr/lib/x86_64-linux-gnu/libpixman-1.so.0.34.0 /opt/conda/lib/libpixman-1.so.0.38.0 && \

# pin jupyter-server to version 2.12.5; later versions break LSP (b/333854354)

pip install --force-reinstall --no-deps jupyter_server==2.12.5 && \

/tmp/clean-layer.sh

# Fix to import bq_helper library without downgrading setuptools

RUN mkdir -p ~/src && git clone https://github.com/SohierDane/BigQuery_Helper ~/src/BigQuery_Helper && \

mkdir -p ~/src/BigQuery_Helper/bq_helper && \

mv ~/src/BigQuery_Helper/bq_helper.py ~/src/BigQuery_Helper/bq_helper/__init__.py && \

mv ~/src/BigQuery_Helper/test_helper.py ~/src/BigQuery_Helper/bq_helper/ && \

sed -i 's/)/packages=["bq_helper"])/g' ~/src/BigQuery_Helper/setup.py && \

pip install -e ~/src/BigQuery_Helper && \

/tmp/clean-layer.sh

# Add BigQuery client proxy settings

ENV PYTHONUSERBASE "/root/.local"

ADD patches/kaggle_gcp.py /root/.local/lib/python3.10/site-packages/kaggle_gcp.py

ADD patches/kaggle_secrets.py /root/.local/lib/python3.10/site-packages/kaggle_secrets.py

ADD patches/kaggle_session.py /root/.local/lib/python3.10/site-packages/kaggle_session.py

ADD patches/kaggle_web_client.py /root/.local/lib/python3.10/site-packages/kaggle_web_client.py

ADD patches/kaggle_datasets.py /root/.local/lib/python3.10/site-packages/kaggle_datasets.py

ADD patches/log.py /root/.local/lib/python3.10/site-packages/log.py

ADD patches/sitecustomize.py /root/.local/lib/python3.10/site-packages/sitecustomize.py

# Override default imagemagick policies

ADD patches/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml

# Add Kaggle module resolver

ADD patches/kaggle_module_resolver.py /opt/conda/lib/python3.10/site-packages/tensorflow_hub/kaggle_module_resolver.py

RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tensorflow_hub import kaggle_module_resolver' /opt/conda/lib/python3.10/site-packages/tensorflow_hub/config.py && \

sed -i '/_install_default_resolvers()/a \ \ registry.resolver.add_implementation(kaggle_module_resolver.KaggleFileResolver())' /opt/conda/lib/python3.10/site-packages/tensorflow_hub/config.py

# TensorBoard Jupyter extension. Should be replaced with TensorBoard's provided magic once we have

# worker tunneling support in place.

# b/139212522 re-enable TensorBoard once solution for slowdown is implemented.

# ENV JUPYTER_CONFIG_DIR "/root/.jupyter/"

# RUN pip install jupyter_tensorboard && \

# jupyter serverextension enable jupyter_tensorboard && \

# jupyter tensorboard enable

# ADD patches/tensorboard/notebook.py /opt/conda/lib/python3.10/site-packages/tensorboard/notebook.py

# Disable unnecessary jupyter extensions

#RUN jupyter-nbextension disable nb_conda --py --sys-prefix && \

# jupyter-serverextension disable nb_conda --py --sys-prefix && \

# python -m nb_conda_kernels.install --disable

# Disable preloaded jupyter modules (they add to startup, and break when they are missing)

RUN sed -i /bq_stats/d /etc/ipython/ipython_kernel_config.py && \

sed -i /beatrix/d /etc/ipython/ipython_kernel_config.py && \

sed -i /bigquery/d /etc/ipython/ipython_kernel_config.py && \

sed -i /sql/d /etc/ipython/ipython_kernel_config.py

# Force only one libcusolver

{{ if eq .Accelerator "gpu" }}

RUN rm /opt/conda/bin/../lib/libcusolver.so.11 && ln -s /usr/local/cuda/lib64/libcusolver.so.11 /opt/conda/bin/../lib/libcusolver.so.11

{{ else }}

RUN ln -s /usr/local/cuda/lib64/libcusolver.so.11 /opt/conda/bin/../lib/libcusolver.so.11

{{ end }}

# b/270147159 conda ships with a version of libtinfo which is missing version info causing warnings, replace it with a good version.

RUN rm /opt/conda/lib/libtinfo.so.6 && ln -s /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /opt/conda/lib/libtinfo.so.6

# b/276358430 fix Jupyter lsp freezing up the jupyter server

RUN pip install "jupyter-lsp==1.5.1"

# Set backend for matplotlib

ENV MPLBACKEND "agg"

# Set LC_ALL

# https://github.com/explosion/spaCy/issues/12872#issuecomment-1661847588

ENV LC_ALL "POSIX"

ARG GIT_COMMIT=unknown

ARG BUILD_DATE=unknown

LABEL git-commit=$GIT_COMMIT

LABEL build-date=$BUILD_DATE

ENV GIT_COMMIT=${GIT_COMMIT}

ENV BUILD_DATE=${BUILD_DATE}

LABEL tensorflow-version=$TENSORFLOW_VERSION

# Used in the Jenkins `Docker GPU Build` step to restrict the images being pruned.

LABEL kaggle-lang=python

# Correlate current release with the git hash inside the kernel editor by running `!cat /etc/git_commit`.

RUN echo "$GIT_COMMIT" > /etc/git_commit && echo "$BUILD_DATE" > /etc/build_date

{{ if eq .Accelerator "gpu" }}

# Remove the CUDA stubs.

ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH_NO_STUBS"

# Add the CUDA home.

ENV CUDA_HOME=/usr/local/cuda

{{ end }}