FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

add jenkins setup · atn123-gh/docker-python@38bdd94 · GitHub

/ docker-python Public
forked from Kaggle/docker-python

Commit 38bdd94

Browse files
djherbis
committed
add jenkins setup
1 parent 2ad5f87 commit 38bdd94

7 files changed

Lines changed: 107 additions & 1 deletion

File tree

‎Jenkinsfile‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,34 @@ pipeline {
150150
}
151151
}
152152
}
153-
}
153+
}
154+
stage('TPU VM') {
155+
stages {
156+
stage('Build Tensorflow TPU Image') {
157+
options {
158+
timeout(time: 120, unit: 'MINUTES')
159+
}
160+
steps {
161+
sh '''#!/bin/bash
162+
set -exo pipefail
163+
164+
./tpu/build | ts
165+
./push --tpu ${PRETEST_TAG}
166+
'''
167+
}
168+
}
169+
}
170+
stage('Diff TPU VM Image') {
171+
steps {
172+
sh '''#!/bin/bash
173+
set -exo pipefail
174+
175+
docker pull gcr.io/kaggle-private-byod/python-tpuvm:${PRETEST_TAG}
176+
./diff --tpu --target gcr.io/kaggle-private-byod/python-tpuvm:${PRETEST_TAG}
177+
'''
178+
}
179+
}
180+
}
154181
}
155182
}
156183

@@ -161,6 +188,7 @@ pipeline {
161188
162189
gcloud container images add-tag gcr.io/kaggle-images/python:${PRETEST_TAG} gcr.io/kaggle-images/python:${STAGING_TAG}
163190
gcloud container images add-tag gcr.io/kaggle-private-byod/python:${PRETEST_TAG} gcr.io/kaggle-private-byod/python:${STAGING_TAG}
191+
gcloud container images add-tag gcr.io/kaggle-private-byod/python-tpuvm:${PRETEST_TAG} gcr.io/kaggle-private-byod/python-tpuvm:${STAGING_TAG}
164192
'''
165193
}
166194
}

‎diff‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ while :; do
3232
BASE_IMAGE_TAG='gcr.io/kaggle-private-byod/python:latest'
3333
TARGET_IMAGE_TAG='kaggle/python-gpu-build'
3434
;;
35+
-x|--tpu)
36+
BASE_IMAGE_TAG='gcr.io/kaggle-private-byod/python-tpuvm:latest'
37+
TARGET_IMAGE_TAG='kaggle/python-tpuvm-build'
38+
;;
3539
-b|--base)
3640
if [[ -z "$2" ]]; then
3741
usage

‎push‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ Push a newly-built image with the given LABEL to gcr.io and DockerHub.
88
99
Options:
1010
-g, --gpu Push the image with GPU support.
11+
-t, --tpu Push the image with GPU support.
1112
-s, --source-image IMAGE Tag for the source image.
1213
EOF
1314
}
@@ -26,6 +27,10 @@ while :; do
2627
SOURCE_IMAGE_TAG='kaggle/python-gpu-build:latest'
2728
TARGET_IMAGE='gcr.io/kaggle-private-byod/python'
2829
;;
30+
-t|--tpu)
31+
SOURCE_IMAGE_TAG='kaggle/python-tpuvm-build:latest'
32+
TARGET_IMAGE='gcr.io/kaggle-private-byod/python-tpuvm'
33+
;;
2934
-s|--source-image)
3035
if [[ -z $2 ]]; then
3136
usage

‎tests/common.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@
44
import unittest
55

66
gpu_test = unittest.skipIf(len(os.environ.get('CUDA_VERSION', '')) == 0, 'Not running GPU tests')
7+
tpu_test = unittest.skipIf(len(os.environ.get('ISTPUVM', '')) == 0, 'Not running TPU tests')

‎tpu/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ FROM gcr.io/cloud-tpu-v2-images/libtpu:${LIBTPU_IMAGE_TAG} as libtpu
66
FROM gcr.io/kaggle-images/python-tpu-tensorflow-whl:${TENSORFLOW_WHL_IMAGE_TAG} AS tensorflow_whl
77
FROM gcr.io/kaggle-images/python:${BASE_IMAGE_TAG}
88

9+
ENV ISTPUVM=1
10+
911
COPY --from=libtpu /libtpu.so /lib
1012

1113
COPY --from=tensorflow_whl /tmp/tensorflow_pkg/tensorflow*.whl /tmp/tensorflow_pkg/

‎tpu/build‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
set -e
3+
4+
usage() {
5+
cat << EOF
6+
Usage: $0 [OPTIONS]
7+
Build a new Python TPU 1VM Docker image.
8+
9+
Options:
10+
-c, --use-cache Use layer cache when building a new image.
11+
EOF
12+
}
13+
14+
CACHE_FLAG='--no-cache'
15+
TENSORFLOW_DOCKERFILE='tensorflow.Dockerfile'
16+
TENSORFLOW_IMAGE_TAG='kaggle/python-tpuvm-tensorflow-build'
17+
DOCKERFILE='Dockerfile'
18+
IMAGE_TAG='kaggle/python-tpuvm-build'
19+
BUILD_ARGS=''
20+
21+
while :; do
22+
case "$1" in
23+
-h|--help)
24+
usage
25+
exit
26+
;;
27+
-c|--use-cache)
28+
CACHE_FLAG=''
29+
;;
30+
-?*)
31+
usage
32+
printf 'ERROR: Unknown option: %s\n' "$1" >&2
33+
exit
34+
;;
35+
*)
36+
break
37+
esac
38+
39+
shift
40+
done
41+
42+
BUILD_ARGS+=" --build-arg GIT_COMMIT=$(git rev-parse HEAD)"
43+
BUILD_ARGS+=" --build-arg BUILD_DATE=$(date '+%Y%m%d-%H%M%S')"
44+
45+
# Read build args from config.txt file.
46+
SRCDIR=$(dirname "${BASH_SOURCE[0]}")
47+
for l in `cat ${SRCDIR}/config.txt`; do
48+
BUILD_ARGS+=" --build-arg $l"
49+
done
50+
51+
readonly CACHE_FLAG
52+
readonly TENSORFLOW_DOCKERFILE
53+
readonly TENSORFLOW_IMAGE_TAG
54+
readonly DOCKERFILE
55+
readonly IMAGE_TAG
56+
readonly BUILD_ARGS
57+
58+
TENSORFLOW_DOCKERFILE_PATH="$SRCDIR/$TENSORFLOW_DOCKERFILE"
59+
DOCKERFILE_PATH="$SRCDIR/$DOCKERFILE"
60+
61+
set -x
62+
docker build --rm --pull $CACHE_FLAG -t "$TENSORFLOW_IMAGE_TAG" -f "$TENSORFLOW_DOCKERFILE_PATH" $BUILD_ARGS .
63+
docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE_PATH" $BUILD_ARGS .

‎tpu/config.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BASE_IMAGE_TAG=v108
2+
LIBTPU_IMAGE_TAG=libtpu_1.1.0_RC00
3+
TENSORFLOW_WHL_IMAGE_TAG=local

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL