GitHub Viewer

#!/bin/bash set -e IMAGE_TAG='kaggle/python-build' IMAGE_TAG_OVERRIDE='' ADDITONAL_OPTS='--runtime runc ' # Use the CPU runtime by default PATTERN='test*.py' usage() { cat &2 exit fi IMAGE_TAG_OVERRIDE=$2 shift # skip the flag value ;; -p|--pattern) if [[ -z $2 ]]; then usage printf 'ERROR: No PATTERN specified after the %s flag.\n' "$1" >&2 exit fi PATTERN="$2" shift # skip the flag value ;; -?*) usage printf 'ERROR: Unknown option: %s\n' "$1" >&2 exit ;; *) break esac shift done if [[ -n "$IMAGE_TAG_OVERRIDE" ]]; then IMAGE_TAG="$IMAGE_TAG_OVERRIDE" fi readonly IMAGE_TAG readonly ADDITONAL_OPTS readonly PATTERN set -x mkdir -p /tmp/python-build/tmp mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working mkdir -p /tmp/python-build/kaggle # Only run Jupyter server test if no specific test pattern is specified. if [ $PATTERN == 'test*.py' ]; then # Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm "$IMAGE_TAG" jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test fi # Note about `TF_FORCE_GPU_ALLOW_GROWTH`. This allocate memory at runtime as needed. # By default, TensorFlow maps nearly all of the GPU memory visible to the process. # This is causing issue when other libraries are trying to run tests using a GPU. # See: https://www.tensorflow.org/guide/gpu#allowing_gpu_memory_growth # # Note about `XLA_PYTHON_CLIENT_PREALLOCATE`. By default, JAX preallocates 90% # of the GPU memory which is causing issues when other libraries are trying to run # tests using a GPU. # See: https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html # # Note about `--hostname localhost` (b/158137436) # hostname defaults to the container name which fails DNS name # resolution with --net=none (required to keep tests hermetic). See details in bug. # # Note about CLOUDSDK_CONFIG=/tmp/.config/gcloud # We use the /tmp dir since the filesystem is --read-only and we need writable space for gcloud configs. docker run --rm -t --read-only --net=none \ -e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key \ -e KAGGLE_USER_SECRETS_TOKEN_KEY=test-secrets-key \ -e KAGGLE_URL_BASE=http://127.0.0.1:0 \ -e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 \ -e KAGGLE_DATA_PROXY_PROJECT=test \ -e TF_FORCE_GPU_ALLOW_GROWTH=true \ -e XLA_PYTHON_CLIENT_PREALLOCATE=false \ -e CLOUDSDK_CONFIG=/tmp/.config/gcloud \ --hostname localhost \ --shm-size=2g \ -v $PWD:/input:ro -v /tmp/python-build/working:/working \ -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm \ -v /tmp/python-build/kaggle:/kaggle \ -w=/working \ $ADDITONAL_OPTS \ "$IMAGE_TAG" \ /bin/bash -c "python -m unittest discover -s /input/tests -p $PATTERN -v"