FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Package updates and minor refactoring · Dhusanth/docker-python@77fb4c0 · GitHub

/ docker-python Public
forked from Kaggle/docker-python

Commit 77fb4c0

Browse files
crawforc3
authored
Package updates and minor refactoring
1 parent f7e40bf commit 77fb4c0

2 files changed

Lines changed: 83 additions & 52 deletions

File tree

‎Dockerfile‎

Lines changed: 61 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia
22

33
FROM continuumio/anaconda3:5.0.1
44

5+
# This is necessary to for apt to access HTTPS sources
6+
RUN apt-get update && \
7+
apt-get install apt-transport-https
8+
59
COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/
610
COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/
711
COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg
@@ -40,6 +44,8 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list &
4044
apt-get update && apt-get install -y build-essential && \
4145
# https://stackoverflow.com/a/46498173
4246
conda update -y conda && conda update -y python && \
47+
pip install --upgrade pip && \
48+
apt-get -y install cmake && \
4349
# Vowpal Rabbit
4450
#apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \
4551
#cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \
@@ -65,27 +71,24 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \
6571
tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \
6672
make -j $(nproc) && make install && \
6773
# clean up ImageMagick source files
68-
cd ../ && rm -rf ImageMagick* && \
69-
apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \
70-
apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \
71-
apt-get remove cmake && \
72-
apt-get purge --auto-remove cmake && \
73-
export CMAKE_VERSION=3.11 CMAKE_BUILD=1 && \
74-
cd /usr/local/src && wget https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.${CMAKE_BUILD}.tar.gz && \
75-
tar -xzvf cmake-${CMAKE_VERSION}.${CMAKE_BUILD}.tar.gz && \
76-
cd cmake-${CMAKE_VERSION}.${CMAKE_BUILD}/ && \
77-
./bootstrap && make -j4 && make install && \
78-
cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \
79-
cd opencv && \
80-
mkdir build && cd build && \
81-
cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \
82-
make -j $(nproc) && make install && \
83-
echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \
84-
cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \
74+
cd ../ && rm -rf ImageMagick*
75+
76+
#OpenCV install (from pip or source)
77+
RUN pip install opencv-python
78+
#apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \
79+
#apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \
80+
#cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \
81+
#cd opencv && \
82+
#mkdir build && cd build && \
83+
#cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \
84+
#make -j $(nproc) && make install && \
85+
#echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \
86+
#cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \
8587
# Clean up install cruft
86-
rm -rf /usr/local/src/opencv && \
87-
rm -rf /root/.cache/pip/* && \
88-
apt-get autoremove -y && apt-get clean
88+
#rm -rf /usr/local/src/opencv && \
89+
#rm -rf /root/.cache/pip/* && \
90+
#apt-get autoremove -y && apt-get clean
91+
8992

9093
# Tensorflow source build
9194
ENV TF_NEED_CUDA=1
@@ -104,8 +107,9 @@ RUN apt-get update && \
104107
apt-get install -y oracle-java8-installer && \
105108
echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
106109
curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
107-
apt-get update && apt-get install -y bazel && apt-get upgrade -y bazel && \
108-
cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \
110+
apt-get update && apt-get install -y bazel && \
111+
apt-get upgrade -y bazel
112+
RUN cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \
109113
cd tensorflow && cat /dev/null | ./configure && \
110114
echo "/usr/local/cuda-${TF_CUDA_VERSION}/targets/x86_64-linux/lib/stubs" > /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \
111115
bazel build --config=opt \
@@ -196,18 +200,24 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \
196200
# Make sure the dynamic linker finds the right libstdc++
197201
ENV LD_LIBRARY_PATH="/opt/conda/lib:${LD_LIBRARY_PATH}"
198202

203+
# Install Basemap via conda temporarily
199204
RUN apt-get update && \
200-
# Libgeos, for mapping libraries
201-
apt-get -y install libgeos-dev && \
202-
# pyshp and pyproj are now external dependencies of Basemap
203-
pip install pyshp pyproj && \
204-
cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \
205-
export GEOS_DIR=/usr/local && \
206-
cd basemap && python setup.py install && \
205+
#apt-get -y install libgeos-dev && \
206+
#pip install matplotlib && \
207+
#pip install pyshp && \
208+
#pip install pyproj && \
209+
#cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \
210+
#cd basemap/geos-3.3.3 && \
211+
#export GEOS_DIR=/usr/local && \
212+
#./configure --prefix=$GEOS_DIR && \
213+
#make && make install && \
214+
#cd .. && python setup.py install && \
215+
conda install basemap && \
207216
# Pillow (PIL)
208217
apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev && \
209-
pip install Pillow && \
210-
cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \
218+
pip install Pillow
219+
220+
RUN cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \
211221
cd opendeep && python setup.py develop && \
212222
# sasl is apparently an ibis dependency
213223
apt-get -y install libsasl2-dev && \
@@ -224,21 +234,20 @@ RUN apt-get update && \
224234
# MXNet
225235
pip install mxnet && \
226236
# h2o
227-
# Temporary sync of conda's numpy with pip's, needed to avoid an install error
228-
conda upgrade -y numpy && \
229237
# This requires python-software-properties and Java, which were installed above.
230238
cd /usr/local/src && mkdir h2o && cd h2o && \
231239
wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \
232240
wget --no-check-certificate -i latest -O h2o.zip && rm latest && \
233241
unzip h2o.zip && rm h2o.zip && cp h2o-*/h2o.jar . && \
234242
pip install `find . -name "*whl"` && \
235243
# Work around https://github.com/tensorflow/tensorflow/issues/16488
236-
pip install numpy --upgrade && \
244+
pip install numpy --upgrade
245+
237246
# Keras setup
238247
# Keras likes to add a config file in a custom directory when it's
239248
# first imported. This doesn't work with our read-only filesystem, so we
240249
# have it done now.
241-
python -c "from keras.models import Sequential" && \
250+
RUN python -c "from keras.models import Sequential" && \
242251
# Switch to TF backend
243252
sed -i 's/theano/tensorflow/' /root/.keras/keras.json && \
244253
# Re-run it to flush any more disk writes
@@ -276,19 +285,22 @@ RUN apt-get update && \
276285
# Pandoc is a dependency of deap
277286
apt-get install -y pandoc && \
278287
cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \
279-
cd py-earth && python setup.py install && \
288+
cd py-earth && python setup.py install
280289
#cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \
281290
#./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \
282-
#./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \
291+
#./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6
292+
283293
# Install torch and torchvision from source, so we're using the cuda/cudnn libraries installed above.
284-
conda install mkl setuptools cmake cffi typing && \
294+
RUN export CXXFLAGS="-std=c++11" && \
295+
export CFLAGS="-std=c99" && \
296+
conda install mkl mkl-include setuptools cmake cffi typing && \
285297
conda install -c pytorch magma-cuda90 && \
286298
cd /usr/local/src && \
287-
git clone -b v0.3.1 --recursive https://github.com/pytorch/pytorch && \
299+
git clone -b v0.4.0 --recursive https://github.com/pytorch/pytorch && \
288300
cd pytorch && \
289301
python setup.py install && \
290302
cd /usr/local/src && \
291-
git clone -b v0.2.0 --recursive https://github.com/pytorch/vision && \
303+
git clone -b v0.2.1 --recursive https://github.com/pytorch/vision && \
292304
cd vision && \
293305
python setup.py install && \
294306
# PyTorch Audio
@@ -297,7 +309,7 @@ RUN apt-get update && \
297309
cd /usr/local/src && \
298310
git clone https://github.com/pytorch/audio && \
299311
cd audio && \
300-
CXXFLAGS="-std=c++11" CFLAGS="-std=c99" python setup.py install && \
312+
python setup.py install && \
301313
# ggpy / ggplot
302314
pip install git+https://github.com/yhat/ggplot.git && \
303315
# Basic cuda support library for python.
@@ -410,8 +422,9 @@ RUN pip install --upgrade mpld3 && \
410422
pip install geoplot && \
411423
pip install eli5 && \
412424
pip install implicit && \
413-
pip install dask-ml[xgboost] && \
414-
pip install kmeans-smote && \
425+
pip install dask-ml[xgboost]
426+
427+
RUN pip install kmeans-smote --no-dependencies && \
415428
# Add google PAIR-code Facets
416429
cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \
417430
export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \
@@ -420,7 +433,7 @@ RUN pip install --upgrade mpld3 && \
420433
pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \
421434
pip install pycountry && pip install iso3166 && \
422435
pip install pydash && \
423-
pip install kmodes && \
436+
pip install kmodes --no-dependencies && \
424437
pip install librosa && \
425438
pip install polyglot && \
426439
pip install mmh3 && \
@@ -460,10 +473,11 @@ RUN pip install --upgrade cython && \
460473
pip install mlcrate && \
461474
pip install gym && \
462475
pip install ray && \
463-
# clean up pip cache
464-
rm -rf /root/.cache/pip/* && \
465476
# Required to display Altair charts in Jupyter notebook
466-
jupyter nbextension install --user --py vega
477+
pip install vega3 && \
478+
jupyter nbextension install --sys-prefix --py vega3 && \
479+
# clean up pip cache
480+
rm -rf /root/.cache/pip/*
467481

468482
# Fast.ai and dependencies
469483
RUN pip install bcolz && \

‎test_build.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,31 @@
1010
# already handled by patches/sitecustomize.py, we need to import tensorflow
1111
# first as well. This test script (in terms of imports) may not be sufficiently
1212
# realistic for this problem to occur in production uses of the image.
13-
import tensorflow
13+
import tensorflow as tf
14+
print(tf.__version__)
15+
hello = tf.constant('TensorFlow ok')
16+
sess = tf.Session()
17+
print(sess.run(hello))
18+
print("Tensorflow ok")
1419

15-
import os
20+
from keras.models import Sequential
21+
from keras.layers.core import Dense, Dropout, Activation, Flatten
22+
from keras.layers.convolutional import Convolution2D, MaxPooling2D
23+
from keras.optimizers import SGD
24+
print("Keras ok")
25+
26+
# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html
27+
import torch
28+
import torch.nn as tnn
29+
import torch.autograd as autograd
30+
torch.manual_seed(31337)
31+
linear_torch = tnn.Linear(5,3)
32+
data_torch = autograd.Variable(torch.randn(2, 5))
33+
print(linear_torch(data_torch))
34+
print("PyTorch ok")
1635

1736
# General GPU support.
37+
import os
1838
if os.environ.get("EXPECT_GPU") == "1":
1939
import subprocess
2040
import sys
@@ -69,9 +89,6 @@
6989
import plotly.graph_objs as go
7090
print("plotly ok")
7191

72-
from ggplot import *
73-
print("ggplot ok")
74-
7592
import theano
7693
print("Theano ok")
7794

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL