docker-python/test_build.py at crawforc3-patch-1 · HadokenCode/docker-python · GitHub

History

269 lines (209 loc) · 7.44 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

# This script should run without errors whenever we update the

# kaggle/python container. It checks that all our most popular packages can

# be loaded and used without errors.

# When the EXPECT_GPU environment variable is set to 1, the test will

# expect GPU support and a single GPU device to be present. Otherwise no GPU

# device is expected (and GPU support is undetermined).

# For a similar issue to the pytorch static TLS issue, which is referenced and

# already handled by patches/sitecustomize.py, we need to import tensorflow

# first as well. This test script (in terms of imports) may not be sufficiently

# realistic for this problem to occur in production uses of the image.

import tensorflow as tf

print(tf.__version__)

hello = tf.constant('TensorFlow ok')

sess = tf.Session()

print(sess.run(hello))

print("Tensorflow ok")

from keras.models import Sequential

from keras.layers.core import Dense, Dropout, Activation, Flatten

from keras.layers.convolutional import Convolution2D, MaxPooling2D

from keras.optimizers import SGD

print("Keras ok")

# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html

import torch

import torch.nn as tnn

import torch.autograd as autograd

torch.manual_seed(31337)

linear_torch = tnn.Linear(5,3)

data_torch = autograd.Variable(torch.randn(2, 5))

print(linear_torch(data_torch))

print("PyTorch ok")

# General GPU support.

import os

if os.environ.get("EXPECT_GPU") == "1":

import subprocess

import sys

smi = subprocess.Popen(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

print(smi.communicate()[0].decode())

import numpy as np

print("Numpy imported ok")

print("Your lucky number is: " + str(np.random.randint(100)))

# Numpy must be linked to the MKL. (Occasionally, a third-party package will muck up the installation

# and numpy will be reinstalled with an OpenBLAS backing.)

from numpy.distutils.system_info import get_info

# This will throw an exception if the MKL is not linked correctly.

get_info("blas_mkl")

import pandas as pd

print("Pandas imported ok")

from sklearn import datasets

print("sklearn imported ok")

iris = datasets.load_iris()

X, y = iris.data, iris.target

from sklearn.ensemble import RandomForestClassifier

rf1 = RandomForestClassifier()

rf1.fit(X,y)

print("sklearn RandomForestClassifier: ok")

from sklearn.linear_model import LinearRegression

boston = datasets.load_boston()

X, y = boston.data, boston.target

lr1 = LinearRegression()

lr1.fit(X,y)

print("sklearn LinearRegression: ok")

from xgboost import XGBClassifier

xgb1 = XGBClassifier(n_estimators=3)

xgb1.fit(X[0:70],y[0:70])

print("xgboost XGBClassifier: ok")

import matplotlib.pyplot as plt

plt.plot(np.linspace(0,1,50), np.random.rand(50))

plt.savefig("plot1.png")

print("matplotlib.pyplot ok")

from mpl_toolkits.basemap import Basemap

print("Basemap ok")

import plotly.plotly as py

import plotly.graph_objs as go

print("plotly ok")

import theano

print("Theano ok")

from keras.models import Sequential

from keras.layers.core import Dense, Dropout, Activation, Flatten

from keras.layers.convolutional import Convolution2D, MaxPooling2D

from keras.optimizers import SGD

print("keras ok")

import nltk

from nltk.stem import WordNetLemmatizer

print("nltk ok")

import tensorflow as tf

# There should always be at least one CPU available.

with tf.device('/cpu:0'):

hello = tf.constant('TensorFlow ok (CPU)')

sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))

print(sess.run(hello).decode())

sess.close()

print("GPU device name (empty if no GPU present): %s" % tf.test.gpu_device_name())

if os.environ.get("EXPECT_GPU") == "1":

# Basic test:

with tf.device('/gpu:0'):

a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')

b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')

c = tf.matmul(a, b)

sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))

print(sess.run(c))

sess.close()

# Extensive test:

import timeit

# See https://www.tensorflow.org/tutorials/using_gpu#allowing_gpu_memory_growth

config = tf.ConfigProto()

config.gpu_options.allow_growth = True

with tf.device('/gpu:0'):

random_image_gpu = tf.random_normal((128, 128, 128, 3))

net_gpu = tf.layers.conv2d(random_image_gpu, 32, 7)

net_gpu = tf.reduce_sum(net_gpu)

sess = tf.Session(config=config)

sess.run(tf.global_variables_initializer())

def gpu():

sess.run(net_gpu)

print('GPU (s): ', end='')

first_gpu_time = timeit.timeit('gpu()', number=1, setup="from __main__ import gpu")

print('%s (first run), '% first_gpu_time, end='')

gpu_time = timeit.timeit('gpu()', number=10, setup="from __main__ import gpu")

print('%s (10 following runs)' % (gpu_time/10))

sess.close()

import cv2

img = cv2.imread('plot1.png',0)

print("OpenCV ok")

from skimage.io import imread

print("skimage ok")

from wordbatch.extractors import WordBag

print("wordbatch ok")

import pyfasttext

print("pyfasttext ok")

import fastText

print("fastText ok")

import mxnet

import mxnet.gluon

print("mxnet ok")

import pycuda

import pycuda.driver

if os.environ.get("EXPECT_GPU") == "1":

import pycuda.driver

pycuda.driver.Device(0).name()

print("pycuda ok")

import torch

# Note: torch.cuda.is_available() returns whether GPU support is present AND at least one GPU is available.

if os.environ.get("EXPECT_GPU") == "1":

assert torch.cuda.is_available(), "torch reports cuda is not available"

expected_device_count = 1

else:

expected_device_count = 0

assert torch.cuda.device_count() == expected_device_count, (

"%d GPU devices reported, expecting %d" % (torch.cuda.device_count(), expected_device_count))

print("torch ok (gpu available: %s, count: %d)" % (torch.cuda.is_available(), torch.cuda.device_count()))

import bokeh

print("bokeh ok")

import seaborn

print("seaborn ok")

# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html

import torch

import torch.nn as tnn

import torch.autograd as autograd

torch.manual_seed(31337)

linear_torch = tnn.Linear(5,3)

data_torch = autograd.Variable(torch.randn(2, 5))

print(linear_torch(data_torch))

print("PyTorch ok")

import fastai

from fastai.io import get_data

print("fast.ai ok")

import gym

print("gym ok")

import ray

import time

@ray.remote

def f():

time.sleep(0.1)

return 1

ray.init()

results = ray.get([f.remote() for i in range(4)])

assert results == [1]*4

print("ray ok")

# bigquery proxy

import os

import threading

from http.server import BaseHTTPRequestHandler, HTTPServer

from google.cloud import bigquery

HOSTNAME = "127.0.0.1"

PORT = 8000

URL = "http://%s:%s" % (HOSTNAME, PORT)

fake_bq_called = False

fake_bq_header_found = False

class HTTPHandler(BaseHTTPRequestHandler):

def do_HEAD(s):

s.send_response(200)

def do_GET(s):

global fake_bq_called

global fake_bq_header_found

fake_bq_called = True

fake_bq_header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key")

s.send_response(200)

httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler)

threading.Thread(target=httpd.serve_forever).start()

client = bigquery.Client()

try:

for ds in client.list_datasets(): pass

except:

pass

httpd.shutdown()

assert fake_bq_called, "Fake server did not recieve a request from the BQ client."

assert fake_bq_header_found, "X-KAGGLE-PROXY-DATA header was missing from the BQ request."

print("bigquery proxy ok")

docker-python/test_build.py at crawforc3-patch-1 · HadokenCode/docker-python · GitHub

Expand file tree

Breadcrumbs

test_build.py

Latest commit

History

Breadcrumbs

test_build.py

File metadata and controls