FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Cleanup useless files for each image layers · feibyte/docker-python@eb3f09f · GitHub

/ docker-python Public
forked from Kaggle/docker-python

Commit eb3f09f

Browse files
rosbo
committed
Cleanup useless files for each image layers
1 parent 604efce commit eb3f09f

3 files changed

Lines changed: 65 additions & 43 deletions

File tree

‎Dockerfile‎

Lines changed: 35 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,33 @@
11
FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl
22
FROM continuumio/anaconda3:5.2.0
33

4-
# This is necessary for apt to access HTTPS sources
5-
RUN apt-get update && \
6-
apt-get install apt-transport-https
7-
4+
ADD clean-layer.sh /tmp/clean-layer.sh
85
ADD patches/ /tmp/patches/
96
ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl
107

8+
# This is necessary for apt to access HTTPS sources
9+
RUN apt-get update && \
10+
apt-get install apt-transport-https && \
11+
/tmp/clean-layer.sh
12+
1113
# Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections,
1214
# as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346
1315
RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \
1416
apt-get update && apt-get install -y build-essential unzip && \
1517
# https://stackoverflow.com/a/46498173
1618
conda update -y conda && conda update -y python && \
1719
pip install --upgrade pip && \
18-
apt-get -y install cmake
20+
apt-get -y install cmake && \
21+
/tmp/clean-layer.sh
1922

2023
# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517
2124
# Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2
2225
RUN conda install -y python=3.6.6 && \
2326
pip install pandas==0.23.2 && \
2427
# Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518
2528
pip install keras_applications==1.0.4 --no-deps && \
26-
pip install keras_preprocessing==1.0.2 --no-deps
29+
pip install keras_preprocessing==1.0.2 --no-deps && \
30+
/tmp/clean-layer.sh
2731

2832
# The anaconda base image includes outdated versions of these packages. Update them to include the latest version.
2933
RUN pip install --upgrade seaborn python-dateutil dask && \
@@ -43,13 +47,13 @@ RUN pip install --upgrade seaborn python-dateutil dask && \
4347
wget https://imagemagick.org/download/ImageMagick.tar.gz && \
4448
tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \
4549
make -j $(nproc) && make install && \
46-
# clean up ImageMagick source files
47-
cd ../ && rm -rf ImageMagick*
50+
/tmp/clean-layer.sh
4851

4952
# Install tensorflow from a pre-built wheel
5053
COPY --from=tensorflow_whl /tmp/tensorflow_cpu/*.whl /tmp/tensorflow_cpu/
5154
RUN pip install /tmp/tensorflow_cpu/tensorflow*.whl && \
52-
rm -rf /tmp/tensorflow_cpu
55+
rm -rf /tmp/tensorflow_cpu && \
56+
/tmp/clean-layer.sh
5357

5458
RUN apt-get install -y libfreetype6-dev && \
5559
apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \
@@ -112,10 +116,7 @@ RUN apt-get install -y libfreetype6-dev && \
112116
vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \
113117
# Stop-words
114118
pip install stop-words && \
115-
# clean up
116-
rm -rf /root/.cache/pip/* && \
117-
apt-get autoremove -y && apt-get clean && \
118-
rm -rf /usr/local/src/*
119+
/tmp/clean-layer.sh
119120

120121
# Make sure the dynamic linker finds the right libstdc++
121122
ENV LD_LIBRARY_PATH=/opt/conda/lib
@@ -128,10 +129,9 @@ RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \
128129
cd basemap && \
129130
git checkout v1.1.0 && \
130131
python setup.py install && \
131-
pip install basemap --no-binary basemap
132-
133-
# sasl is apparently an ibis dependency
134-
RUN apt-get -y install libsasl2-dev && \
132+
pip install basemap --no-binary basemap && \
133+
# sasl is apparently an ibis dependency
134+
apt-get -y install libsasl2-dev && \
135135
# ...as is psycopg2
136136
apt-get install -y libpq-dev && \
137137
pip install ibis-framework && \
@@ -162,7 +162,8 @@ RUN apt-get -y install libsasl2-dev && \
162162
# Re-run it to flush any more disk writes
163163
python -c "from keras.models import Sequential; from keras import backend; print(backend._BACKEND)" && \
164164
# Keras reverts to /tmp from ~ when it detects a read-only file system
165-
mkdir -p /tmp/.keras && cp /root/.keras/keras.json /tmp/.keras
165+
mkdir -p /tmp/.keras && cp /root/.keras/keras.json /tmp/.keras && \
166+
/tmp/clean-layer.sh
166167

167168
# scikit-learn dependencies
168169
RUN pip install scipy && \
@@ -204,21 +205,15 @@ RUN pip install scipy && \
204205
apt-get install -y sox libsox-dev libsox-fmt-all && \
205206
pip install cffi && \
206207
cd /usr/local/src && git clone https://github.com/pytorch/audio && cd audio && python setup.py install && \
207-
# ~~~~ CLEAN UP ~~~~
208-
rm -rf /root/.cache/pip/* && \
209-
apt-get autoremove -y && apt-get clean && \
210-
conda clean -i -l -t -y && \
211-
rm -rf /usr/local/src/*
208+
/tmp/clean-layer.sh
212209

213210
# vtk with dependencies
214211
RUN apt-get install -y libgl1-mesa-glx && \
215212
pip install vtk && \
216213
# xvfbwrapper with dependencies
217214
apt-get install -y xvfb && \
218215
pip install xvfbwrapper && \
219-
# ~~~~ CLEAN UP ~~~~
220-
rm -rf /root/.cache/pip/* && \
221-
apt-get autoremove -y && apt-get clean
216+
/tmp/clean-layer.sh
222217

223218
RUN pip install --upgrade mpld3 && \
224219
pip install mplleaflet && \
@@ -268,7 +263,8 @@ RUN pip install --upgrade mpld3 && \
268263
pip install pystan && \
269264
pip install ImageHash && \
270265
conda install -y ecos && \
271-
conda install -y CVXcanon
266+
conda install -y CVXcanon && \
267+
/tmp/clean-layer.sh
272268

273269
RUN pip install fancyimpute && \
274270
pip install git+https://github.com/pymc-devs/pymc3 && \
@@ -324,7 +320,8 @@ RUN pip install fancyimpute && \
324320
pip install geoplot && \
325321
pip install eli5 && \
326322
pip install implicit && \
327-
pip install dask-ml[xgboost]
323+
pip install dask-ml[xgboost] && \
324+
/tmp/clean-layer.sh
328325

329326
RUN pip install kmeans-smote --no-dependencies && \
330327
# Add google PAIR-code Facets
@@ -343,7 +340,8 @@ RUN pip install kmeans-smote --no-dependencies && \
343340
pip install cufflinks && \
344341
pip install glmnet_py && \
345342
pip install lime && \
346-
pip install memory_profiler
343+
pip install memory_profiler && \
344+
/tmp/clean-layer.sh
347345

348346
# install cython & cysignals before pyfasttext
349347
RUN pip install --upgrade cython && \
@@ -374,9 +372,8 @@ RUN pip install --upgrade cython && \
374372
pip install mlcrate && \
375373
# Required to display Altair charts in Jupyter notebook
376374
pip install vega3 && \
377-
jupyter nbextension install --sys-prefix --py vega3 && \
378-
# clean up pip cache
379-
rm -rf /root/.cache/pip/*
375+
jupyter nbextension install --sys-prefix --py vega3 && \
376+
/tmp/clean-layer.sh
380377

381378
# Fast.ai and dependencies
382379
RUN pip install bcolz && \
@@ -438,9 +435,7 @@ RUN pip install bcolz && \
438435
# which downgrades pytorch. fastai does work with pytorch 0.4.
439436
pip install fastai==0.7.0 --no-deps && \
440437
pip install torchtext && \
441-
# clean up pip cache
442-
rm -rf /root/.cache/pip/* && \
443-
cd && rm -rf /usr/local/src/*
438+
/tmp/clean-layer.sh
444439

445440
###########
446441
#
@@ -476,16 +471,15 @@ RUN pip install flashtext && \
476471
pip install ggplot && \
477472
pip install cesium && \
478473
pip install rgf_python && \
479-
##### ^^^^ Add new contributions above here ^^^^ #####
480-
# clean up pip cache
481-
rm -rf /root/.cache/pip/*
474+
/tmp/clean-layer.sh
482475

483476
# Pin Vowpal Wabbit v8.6.0 because 8.6.1 does not build or install successfully
484477
RUN cd /usr/local/src && \
485478
git clone -b 8.6.0 https://github.com/JohnLangford/vowpal_wabbit.git && \
486479
./vowpal_wabbit/python/conda_install.sh && \
487480
# Reinstall in non-editable mode (without the -e flag)
488-
pip install vowpal_wabbit/python
481+
pip install vowpal_wabbit/python && \
482+
/tmp/clean-layer.sh
489483

490484
# For Facets
491485
ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/
@@ -501,7 +495,8 @@ RUN pip install --upgrade dask && \
501495
# Stop Matplotlib printing junk to the console on first load
502496
sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.6/site-packages/matplotlib/font_manager.py && \
503497
# Make matplotlib output in Jupyter notebooks display correctly
504-
mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py
498+
mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \
499+
/tmp/clean-layer.sh
505500

506501
# Add BigQuery client proxy settings
507502
ENV PYTHONUSERBASE "/root/.local"

‎clean-layer.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
#
3+
# This scripts should be called at the end of each RUN command
4+
# in the Dockerfiles.
5+
#
6+
# Each RUN command creates a new layer that is stored separately.
7+
# At the end of each command, we should ensure we clean up downloaded
8+
# archives and source files used to produce binary to reduce the size
9+
# of the layer.
10+
set -e
11+
set -x
12+
13+
# Delete files that pip caches when installing a package.
14+
rm -rf /root/.cache/pip/*
15+
# Delete old downloaded archive files
16+
apt-get autoremove -y
17+
# Delete downloaded archive files
18+
apt-get clean
19+
# Delete source files used for building binaries
20+
rm -rf /usr/local/src/*
21+
# Delete conda downloaded tarballs
22+
conda clean -y --tarballs

‎gpu.Dockerfile‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia
22
FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl
33
FROM gcr.io/kaggle-images/python:staging
44

5+
ADD clean-layer.sh /tmp/clean-layer.sh
6+
57
# Cuda support
68
COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/
79
COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/
@@ -36,7 +38,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
3638
libnccl2=2.2.12-1+cuda9.1 \
3739
libnccl-dev=2.2.12-1+cuda9.1 && \
3840
ln -s /usr/local/cuda-9.1 /usr/local/cuda && \
39-
ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
41+
ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
42+
/tmp/clean-layer.sh
4043

4144
# Reinstall packages with a separate version for GPU support
4245
# Tensorflow
@@ -45,7 +48,9 @@ RUN pip uninstall -y tensorflow && \
4548
pip install /tmp/tensorflow_gpu/tensorflow*.whl && \
4649
rm -rf /tmp/tensorflow_gpu && \
4750
conda uninstall -y pytorch-cpu torchvision-cpu && \
48-
conda install -y pytorch torchvision -c pytorch
51+
conda install -y pytorch torchvision -c pytorch && \
52+
/tmp/clean-layer.sh
4953

5054
# Install GPU-only packages
51-
RUN pip install pycuda
55+
RUN pip install pycuda && \
56+
/tmp/clean-layer.sh

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL