FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Merge branch 'master' into gitignore · feibyte/docker-python@61e8e8a · GitHub

/ docker-python Public
forked from Kaggle/docker-python

Commit 61e8e8a

Browse files
rosbo
authored
Merge branch 'master' into gitignore
2 parents 8805a1b + 72ed9c7 commit 61e8e8a

7 files changed

Lines changed: 174 additions & 20 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
*.pyc
22
.idea/
3+
.vscode

‎Dockerfile‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,17 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list &
1616
apt-get update && apt-get install -y build-essential unzip && \
1717
# https://stackoverflow.com/a/46498173
1818
conda update -y conda && conda update -y python && \
19-
pip install --upgrade pip && \
19+
# cartopy package fails to install with the latest version of pip (19.0.1) with:
20+
# "ModuleNotFoundError: No module named 'versioneer'"
21+
# `versioneer` is checked in the cartopy package, the sys path seems to differ with the new version of pip.
22+
pip install pip==18.1 && \
2023
apt-get -y install cmake && \
2124
/tmp/clean-layer.sh
2225

2326
# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517
2427
# Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2
2528
RUN conda install -y python=3.6.6 && \
2629
pip install pandas==0.23.2 && \
27-
# Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518
28-
pip install keras_applications==1.0.4 --no-deps && \
29-
pip install keras_preprocessing==1.0.2 --no-deps && \
3030
/tmp/clean-layer.sh
3131

3232
# The anaconda base image includes outdated versions of these packages. Update them to include the latest version.
@@ -64,16 +64,14 @@ RUN apt-get install -y libfreetype6-dev && \
6464
pip install xgboost && \
6565
pip install lightgbm && \
6666
pip install git+git://github.com/Lasagne/Lasagne.git && \
67-
#keras
68-
cd /usr/local/src && mkdir keras && cd keras && \
69-
git clone --depth 1 https://github.com/fchollet/keras.git && \
70-
cd keras && python setup.py install && \
71-
#keras-rl
72-
cd /usr/local/src && mkdir keras-rl && cd keras-rl && \
73-
git clone --depth 1 https://github.com/matthiasplappert/keras-rl.git && \
74-
cd keras-rl && python setup.py install && \
67+
pip install keras && \
68+
pip install keras-rl && \
7569
#keras-rcnn
7670
pip install git+https://github.com/broadinstitute/keras-rcnn && \
71+
# version 3.7.1 adds a dependency on entrypoints > 3. This causes a reinstall but fails because
72+
# it is a distutils package and can't be uninstalled. Once the anaconda image in updated, this
73+
# pin should be removed.
74+
pip install flake8==3.6.0 && \
7775
#neon
7876
cd /usr/local/src && \
7977
git clone --depth 1 https://github.com/NervanaSystems/neon.git && \
@@ -116,9 +114,7 @@ RUN apt-get install -y libfreetype6-dev && \
116114
vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \
117115
# Stop-words
118116
pip install stop-words && \
119-
# latest scikit-image is not compatible with numpy 1.16.0.
120-
# remove the pin once a new version is released (>0.14.1)
121-
pip install git+git://github.com/scikit-image/scikit-image@31d9ecc2f0d8dd3373af3e80b2dcc7887ff2ca24 && \
117+
pip install --upgrade scikit-image && \
122118
/tmp/clean-layer.sh
123119

124120
# Make sure the dynamic linker finds the right libstdc++
@@ -244,7 +240,10 @@ RUN pip install --upgrade mpld3 && \
244240
pip install git+https://github.com/hyperopt/hyperopt.git && \
245241
# tflean. Deep learning library featuring a higher-level API for TensorFlow. http://tflearn.org
246242
pip install git+https://github.com/tflearn/tflearn.git && \
247-
pip install fitter && \
243+
# fitter 1.1.10 is broken. Fails at setup with:
244+
# install_requires = open("requirements.txt").read(),
245+
# FileNotFoundError: [Errno 2] No such file or directory: 'requirements.txt'
246+
pip install fitter==1.0.9 && \
248247
pip install langid && \
249248
# Delorean. Useful for dealing with datetime
250249
pip install delorean && \
@@ -296,7 +295,9 @@ RUN pip install fancyimpute && \
296295
# See: https://github.com/facebook/prophet/issues/775
297296
pip install fbprophet==0.3.post2 && \
298297
pip install holoviews && \
299-
pip install geoviews && \
298+
# 1.6.2 is not currently supported by the version of matplotlib we are using.
299+
# See other comments about why matplotlib is pinned.
300+
pip install geoviews==1.6.1 && \
300301
pip install hypertools && \
301302
# Nxviz has been causing an installation issue by trying unsuccessfully to remove setuptools.
302303
#pip install nxviz && \
@@ -408,7 +409,8 @@ RUN pip install bcolz && \
408409
pip install notebook==5.5.0 && \
409410
pip install olefile && \
410411
pip install opencv-python && \
411-
pip install --upgrade pandas && \
412+
# tsfresh is not yet compatible with pandas 0.24.0
413+
pip install pandas==0.23.4 && \
412414
pip install pandas_summary && \
413415
pip install pandocfilters && \
414416
pip install pexpect && \
@@ -508,6 +510,7 @@ RUN pip install --upgrade dask && \
508510
# Add BigQuery client proxy settings
509511
ENV PYTHONUSERBASE "/root/.local"
510512
ADD patches/kaggle_gcp.py /root/.local/lib/python3.6/site-packages/kaggle_gcp.py
513+
ADD patches/kaggle_secrets.py /root/.local/lib/python3.6/site-packages/kaggle_secrets.py
511514
ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustomize.py
512515

513516
# Set backend for matplotlib

‎Jenkinsfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1,3" : ""
1+
String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1,3,5" : ""
22

33
pipeline {
44
agent { label 'ephemeral-linux' }
@@ -77,7 +77,7 @@ pipeline {
7777
slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
7878
sh '''#!/bin/bash
7979
set -exo pipefail
80-
docker image prune -a -f # remove previously built image to prevent disk from filling up
80+
docker image prune -f # remove previously built image to prevent disk from filling up
8181
./build --gpu | ts
8282
'''
8383
}

‎patches/kaggle_secrets.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
"""UserSecret client classes.
2+
3+
This library adds support for communicating with the UserSecrets service,
4+
currently used for retrieving an access token for supported integrations
5+
(ie. BigQuery).
6+
7+
"""
8+
9+
import json
10+
import os
11+
import urllib.request
12+
13+
_KAGGLE_DEFAULT_URL_BASE = "https://www.kaggle.com"
14+
_KAGGLE_URL_BASE_ENV_VAR_NAME = "KAGGLE_URL_BASE"
15+
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME = "KAGGLE_USER_SECRETS_TOKEN_KEY"
16+
17+
18+
class CredentialError(Exception):
19+
pass
20+
21+
22+
class BackendError(Exception):
23+
pass
24+
25+
26+
class UserSecretsClient():
27+
GET_USER_SECRET_ENDPOINT = '/requests/GetUserSecretRequest'
28+
BIGQUERY_TARGET_VALUE = 1
29+
30+
def __init__(self):
31+
url_base_override = os.getenv(_KAGGLE_URL_BASE_ENV_VAR_NAME)
32+
self.url_base = url_base_override or _KAGGLE_DEFAULT_URL_BASE
33+
# Follow the OAuth 2.0 Authorization standard (https://tools.ietf.org/html/rfc6750)
34+
jwt_token = os.getenv(_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME)
35+
if jwt_token is None:
36+
raise CredentialError(
37+
'A JWT Token is required to use the UserSecretsClient, '
38+
f'but none found in environment variable {_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME}')
39+
self.headers = {'Content-type': 'application/json',
40+
'Authorization': 'Bearer {}'.format(jwt_token)}
41+
42+
def _make_get_request(self, request_body):
43+
request_params = urllib.parse.urlencode(request_body)
44+
url = f'{self.url_base}{self.GET_USER_SECRET_ENDPOINT}?{request_params}'
45+
req = urllib.request.Request(url, headers=self.headers)
46+
with urllib.request.urlopen(req) as response:
47+
response_json = json.loads(response.read())
48+
return response_json
49+
50+
def get_bigquery_access_token(self):
51+
request_body = {
52+
'Target': self.BIGQUERY_TARGET_VALUE
53+
}
54+
response_json = self._make_get_request(request_body)
55+
if 'Secret' not in response_json:
56+
raise BackendError(
57+
'Unexpected response from the service.')
58+
return response_json['Secret']

‎test‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ sleep 3
8080
docker kill jupyter_test && docker rm jupyter_test
8181
docker run --rm -t --read-only --net=none \
8282
-e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key \
83+
-e KAGGLE_USER_SECRETS_TOKEN_KEY=test-secrets-key \
84+
-e KAGGLE_URL_BASE=http://127.0.0.1:8001 \
8385
-e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 \
8486
-e KAGGLE_DATA_PROXY_PROJECT=test \
8587
-v $PWD:/input:ro -v /tmp/python-build/working:/working \

‎tests/test_imports.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ class TestImport(unittest.TestCase):
55
def test_basic(self):
66
import bq_helper
77
import cleverhans
8+
from rl.agents.dqn import DQNAgent

‎tests/test_user_secrets.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import json
2+
import os
3+
import threading
4+
import unittest
5+
from http.server import BaseHTTPRequestHandler, HTTPServer
6+
from test.support import EnvironmentVarGuard
7+
from urllib.parse import urlparse
8+
9+
from google.auth.exceptions import DefaultCredentialsError
10+
from google.cloud import bigquery
11+
from kaggle_secrets import (_KAGGLE_URL_BASE_ENV_VAR_NAME,
12+
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME,
13+
CredentialError, UserSecretsClient)
14+
15+
_TEST_JWT = 'test-secrets-key'
16+
17+
18+
class UserSecretsHTTPHandler(BaseHTTPRequestHandler):
19+
20+
def set_request(self):
21+
raise NotImplementedError()
22+
23+
def get_response(self):
24+
raise NotImplementedError()
25+
26+
def do_HEAD(s):
27+
s.send_response(200)
28+
29+
def do_GET(s):
30+
s.set_request()
31+
s.send_response(200)
32+
s.send_header("Content-type", "application/json")
33+
s.end_headers()
34+
s.wfile.write(json.dumps(s.get_response()).encode("utf-8"))
35+
36+
37+
class TestUserSecrets(unittest.TestCase):
38+
SERVER_ADDRESS = urlparse(os.getenv(_KAGGLE_URL_BASE_ENV_VAR_NAME))
39+
40+
def _test_client(self, client_func, expected_path, secret):
41+
_request = {}
42+
43+
class AccessTokenHandler(UserSecretsHTTPHandler):
44+
45+
def set_request(self):
46+
_request['path'] = self.path
47+
_request['headers'] = self.headers
48+
49+
def get_response(self):
50+
return {"Secret": secret}
51+
52+
env = EnvironmentVarGuard()
53+
env.set(_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME, _TEST_JWT)
54+
with env:
55+
with HTTPServer((self.SERVER_ADDRESS.hostname, self.SERVER_ADDRESS.port), AccessTokenHandler) as httpd:
56+
threading.Thread(target=httpd.serve_forever).start()
57+
58+
try:
59+
client_func()
60+
finally:
61+
httpd.shutdown()
62+
63+
path, headers = _request['path'], _request['headers']
64+
self.assertEqual(
65+
path,
66+
expected_path,
67+
msg="Fake server did not receive the right request from the UserSecrets client.")
68+
self.assertTrue(
69+
any(
70+
k for k in headers
71+
if k == "Authorization" and headers[k] == f'Bearer {_TEST_JWT}'),
72+
msg="Authorization header was missing from the UserSecrets request.")
73+
74+
def test_no_token_fails(self):
75+
env = EnvironmentVarGuard()
76+
env.unset(_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME)
77+
with env:
78+
with self.assertRaises(CredentialError):
79+
client = UserSecretsClient()
80+
81+
def test_get_access_token_succeeds(self):
82+
secret = '12345'
83+
84+
def call_get_access_token():
85+
client = UserSecretsClient()
86+
secret_response = client.get_bigquery_access_token()
87+
self.assertEqual(secret_response, secret)
88+
self._test_client(call_get_access_token,
89+
'/requests/GetUserSecretRequest?Target=1', secret)

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL