FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Merge branch 'main' into py3.10 · JasonYan99/docker-python@9552e2c · GitHub

  1. JasonYan99
  2. docker-python

Repository navigation

Commit 9552e2c

Browse files
djherbis
authored
Merge branch 'main' into py3.10
2 parents 13ca309 + f897864 commit 9552e2c

5 files changed

Lines changed: 42 additions & 14 deletions

File tree

‎Dockerfile.tmpl‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ ADD clean-layer.sh /tmp/clean-layer.sh
5555
ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl
5656
ADD patches/template_conf.json /opt/kaggle/conf.json
5757

58+
# b/276344496: Install specific version of boto3, because 1.26.103 is broken.
59+
RUN pip install boto3==1.26.100 && \
60+
/tmp/clean-layer.sh
61+
5862
{{ if eq .Accelerator "gpu" }}
5963
# b/200968891 Keeps horovod once torch is upgraded.
6064
RUN pip uninstall -y horovod && \
@@ -423,6 +427,7 @@ RUN pip install annoy \
423427
/tmp/clean-layer.sh
424428

425429
# b/273059949 The pre-installed nbconvert is slow on html conversions and has to be force-uninstalled.
430+
# b/274619697 learntools also requires a specific nbconvert right now
426431
RUN rm -rf /opt/conda/lib/python3.7/site-packages/nbconvert*
427432
RUN rm -rf /opt/conda/lib/python3.7/site-packages/mistune*
428433

@@ -449,12 +454,11 @@ RUN pip install bleach \
449454
mistune \
450455
nbformat \
451456
notebook \
452-
"nbconvert<7" \
457+
"nbconvert==6.4.5" \
453458
papermill \
454459
python-lsp-server[all] \
455460
olefile \
456-
# b/198300835 kornia 0.5.10 is not compatible with our version of numpy.
457-
kornia==0.5.8 \
461+
kornia \
458462
pandas_summary \
459463
pandocfilters \
460464
pexpect \

‎patches/kaggle_datasets.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
import os
2+
import sys
3+
from os import listdir
4+
from os.path import isdir, join
25
from kaggle_web_client import KaggleWebClient
36

47
_KAGGLE_TPU_NAME_ENV_VAR_NAME = 'TPU_NAME'
8+
_KAGGLE_TPUVM_NAME_ENV_VAR_NAME = 'ISTPUVM'
9+
_KAGGLE_INPUT_DIR = '/kaggle/input'
510

611
class KaggleDatasets:
712
GET_GCS_PATH_ENDPOINT = '/requests/CopyDatasetVersionToKnownGcsBucketRequest'
@@ -14,8 +19,20 @@ class KaggleDatasets:
1419
def __init__(self):
1520
self.web_client = KaggleWebClient()
1621
self.has_tpu = os.getenv(_KAGGLE_TPU_NAME_ENV_VAR_NAME) is not None
22+
self.has_tpuvm = os.getenv(_KAGGLE_TPUVM_NAME_ENV_VAR_NAME) is not None
1723

1824
def get_gcs_path(self, dataset_dir: str = None) -> str:
25+
if self.has_tpuvm:
26+
if dataset_dir is None:
27+
onlydirs = [f for f in listdir(_KAGGLE_INPUT_DIR) if isdir(join(_KAGGLE_INPUT_DIR, f))]
28+
if len(onlydirs) == 1:
29+
dataset_dir = onlydirs[0]
30+
else:
31+
raise Exception("Could not infer dataset_dir. dataset_dir can only be inferred if there is exactly 1 Kaggle dataset attached.")
32+
dataset = join(_KAGGLE_INPUT_DIR, dataset_dir)
33+
print("get_gcs_path is not required on TPU VMs which can directly use Kaggle datasets, using path: " + dataset, file=sys.stderr)
34+
return dataset
35+
1936
integration_type = self.TPU if self.has_tpu else self.AUTO_ML
2037
data = {
2138
'MountSlug': dataset_dir,

‎tests/test_kornia.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@
77
class TestKornia(unittest.TestCase):
88
def test_imread_opencv(self):
99
img = cv2.imread('/input/tests/data/dot.png')
10-
img_t = kornia.image_to_tensor(img)
10+
img_t = kornia.utils.image_to_tensor(img)
1111

1212
self.assertEqual(img.shape, (1, 1, 3))
1313
self.assertEqual(img_t.shape, (3, 1, 1))
1414

1515
def test_grayscale_torch(self):
1616
img_rgb = torch.rand(2, 3, 4, 5)
17-
img_gray = kornia.rgb_to_grayscale(img_rgb)
17+
img_gray = kornia.color.rgb_to_grayscale(img_rgb)
1818

1919
self.assertEqual(img_gray.shape, (2, 1, 4, 5))

‎tpu/Dockerfile‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,22 +31,24 @@ ADD patches/kaggle_datasets.py /root/.local/lib/python3.8/site-packages/kaggle_d
3131
# sitecustomize adds significant latency to ipython kernel startup and should only be added if needed
3232
# ADD patches/sitecustomize.py /root/.local/lib/python3.8/site-packages/sitecustomize.py
3333

34+
# LIBTPU installed here:
35+
ENV PIP_LIBTPU=/usr/local/lib/python3.8/site-packages/libtpu/libtpu.so
36+
ENV PYTORCH_LIBTPU=/lib/torch-libtpu.so
37+
3438
# Tensorflow wheel:
3539
RUN pip install tensorflow_hub https://storage.googleapis.com/cloud-tpu-tpuvm-artifacts/tensorflow/tf-${TENSORFLOW_VERSION}/tensorflow-${TENSORFLOW_VERSION}-cp38-cp38-linux_x86_64.whl
3640
RUN curl --output /lib/libtpu.so https://storage.googleapis.com/cloud-tpu-tpuvm-artifacts/libtpu/${TF_LIBTPU_VERSION}/libtpu.so
3741
ADD patches/kaggle_module_resolver.py /usr/local/lib/python3.8/site-packages/tensorflow_hub/kaggle_module_resolver.py
3842
RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tensorflow_hub import kaggle_module_resolver' /usr/local/lib/python3.8/site-packages/tensorflow_hub/config.py
3943
RUN sed -i '/_install_default_resolvers()/a \ \ registry.resolver.add_implementation(kaggle_module_resolver.KaggleFileResolver())' /usr/local/lib/python3.8/site-packages/tensorflow_hub/config.py
4044

41-
# LIBTPU installed here:
42-
ENV PIP_LIBTPU=/usr/local/lib/python3.8/site-packages/libtpu/libtpu.so
43-
ENV PYTORCH_LIBTPU=/lib/torch-libtpu.so
44-
4545
# Install Pytorch & related packages
4646
# https://cloud.google.com/tpu/docs/pytorch-xla-ug-tpu-vm#changing_pytorch_version
4747
# The URL doesn't include patch version. i.e. must use 1.11 instead of 1.11.0
4848
# We need to keep the numpy version the same as the installed tf one but compatible with other installs.
49-
RUN pip install numpy==1.24.2 torch==${TORCH_VERSION} torch_xla[tpuvm] -f https://storage.googleapis.com/tpu-pytorch/wheels/tpuvm/torch_xla-${TORCH_VERSION%.*}-cp38-cp38-linux_x86_64.whl torchvision==${TORCHVISION_VERSION} torchtext==${TORCHTEXT_VERSION} torchaudio==${TORCHAUDIO_VERSION}
49+
RUN pip install torch==${TORCH_VERSION} torch_xla[tpuvm] -f https://storage.googleapis.com/tpu-pytorch/wheels/tpuvm/torch_xla-${TORCH_VERSION%.*}-cp38-cp38-linux_x86_64.whl torchvision==${TORCHVISION_VERSION} torchtext==${TORCHTEXT_VERSION} torchaudio==${TORCHAUDIO_VERSION} \
50+
https://storage.googleapis.com/cloud-tpu-tpuvm-artifacts/tensorflow/tf-${TENSORFLOW_VERSION}/tensorflow-${TENSORFLOW_VERSION}-cp38-cp38-linux_x86_64.whl \
51+
numpy==1.23.5
5052

5153
RUN cp $PIP_LIBTPU $PYTORCH_LIBTPU
5254

@@ -56,13 +58,18 @@ RUN sed -i "s|^\(\(.*\)libtpu.configure_library_path.*\)|\1\n\2os.environ['TPU_L
5658
# Install JAX & related packages
5759
# https://cloud.google.com/tpu/docs/jax-quickstart-tpu-vm#install_jax_on_your_cloud_tpu_vm
5860
# We need to keep the numpy version the same as the installed tf one but compatible with other installs.
59-
RUN pip install numpy==1.24.2 jax[tpu]==${JAX_VERSION} -f https://storage.googleapis.com/jax-releases/libtpu_releases.html trax flax optax elegy git+https://github.com/deepmind/dm-haiku jraph distrax
61+
RUN pip install jax[tpu]==${JAX_VERSION} -f https://storage.googleapis.com/jax-releases/libtpu_releases.html trax flax optax elegy git+https://github.com/deepmind/dm-haiku jraph distrax \
62+
https://storage.googleapis.com/cloud-tpu-tpuvm-artifacts/tensorflow/tf-${TENSORFLOW_VERSION}/tensorflow-${TENSORFLOW_VERSION}-cp38-cp38-linux_x86_64.whl \
63+
numpy==1.23.5
6064

6165
# Packages needed by the Notebook editor:
6266
RUN pip install papermill jupyterlab python-lsp-server[all] jupyterlab-lsp
6367

6468
# Additional useful packages should be added here:
65-
RUN pip install pandas
69+
RUN pip install pandas matplotlib
70+
71+
# Monkey-patch the default TPU to the local (TPU VM).
72+
RUN sed -i 's/tpu=None,/tpu="local",/' /usr/local/lib/python3.8/site-packages/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py
6673

6774
# Set these env vars so that they don't produce errs calling the metadata server to load them:
6875
ENV TPU_ACCELERATOR_TYPE=v3-8

‎tpu/config.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
TENSORFLOW_VERSION=2.11.0
2-
TF_LIBTPU_VERSION=1.5.0
1+
TENSORFLOW_VERSION=2.12.0
2+
TF_LIBTPU_VERSION=1.6.0
33
JAX_VERSION=0.4.6
44
# Supports nightly
55
TORCH_VERSION=2.0.0

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL