aws-cli/scripts/make-bundle at develop · github-38438-org/aws-cli · GitHub

History

executable file

208 lines (175 loc) · 6.94 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

#!/usr/bin/env python

"""Script to create self contained install.

The goal of this script is simple:

* Create a self contained install of the CLI that

has requires no external resources during installation.

It does this by using all the normal python tooling

(virtualenv, pip) but provides a simple, easy to use

interface for those not familiar with the python

ecosystem.

"""

import os

import sys

import time

import subprocess

import shutil

import tempfile

import zipfile

from contextlib import contextmanager

# These are package versions that are needed to boostrap

# the installation process. It is *not* the deps required

# by awscli/botocore.

PACKAGE_VERSION = {

'virtualenv': '15.1.0',

# These packages are included because they are required for

# python2.6, but our normal dependency downloading via pip

# only works if you use python2.6. To fix this issue, we're

# explicitly saying that we need to download these packages

# even if pip doesn't think we need them. That way we can

# run make-bundle in python versions besides 2.6.

'ordereddict': '1.1',

'simplejson': '3.3.0',

'argparse': '1.2.1',

'python-dateutil': '2.6.1',

'setuptools-scm': '1.15.7',

'urllib3': '1.22',

}

INSTALL_ARGS = '--allow-all-external --no-use-wheel'

class BadRCError(Exception):

pass

@contextmanager

def cd(dirname):

original = os.getcwd()

os.chdir(dirname)

try:

yield

finally:

os.chdir(original)

def run(cmd):

sys.stdout.write("Running cmd: %s\n" % cmd)

p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,

stderr=subprocess.PIPE)

stdout, stderr = p.communicate()

rc = p.wait()

if p.returncode != 0:

raise BadRCError("Bad rc (%s) for cmd '%s': %s" % (

rc, cmd, stderr + stdout))

return stdout

def create_scratch_dir():

# This creates the dir where all the bundling occurs.

# First we need a top level dir.

dirname = tempfile.mkdtemp(prefix='bundle')

# Then we need to create a dir where all the packages

# will come from.

os.mkdir(os.path.join(dirname, 'packages'))

os.mkdir(os.path.join(dirname, 'packages', 'setup'))

return dirname

def download_package_tarballs(dirname, packages):

with cd(dirname):

for package in packages:

run('%s -m pip install -d . %s==%s %s' % (

sys.executable, package, PACKAGE_VERSION[package], INSTALL_ARGS

))

def download_cli_deps(scratch_dir):

# As far as i can tell this is a bug in pip.

# Running 'pip intall -d . ~/clidir' does not work.

# Running 'pip install -d foo ~/clidir' results

# in python consuming 100% CPU.

# This problem is surprisingly hard, specifically that

# of downloading all the dependent python packages of

# the awscli as tarballs.

# So here's what we do.

# Step one create a virtualenv.

venv_dir = tempfile.mkdtemp()

try:

run('%s -m virtualenv --no-download %s' % (sys.executable, venv_dir))

pip = os.path.join(venv_dir, 'bin', 'pip')

assert os.path.isfile(pip)

awscli_dir = os.path.dirname(

os.path.dirname(os.path.abspath(__file__)))

with cd(scratch_dir):

run('%s install %s -d . -e %s' % (pip, INSTALL_ARGS, awscli_dir))

# Remove the awscli package, we'll create our own sdist.

_remove_cli_zip(scratch_dir)

finally:

shutil.rmtree(venv_dir)

def _remove_cli_zip(scratch_dir):

clidir = [f for f in os.listdir(scratch_dir) if f.startswith('awscli')]

assert len(clidir) == 1

os.remove(os.path.join(scratch_dir, clidir[0]))

def add_cli_sdist(scratch_dir):

awscli_dir = os.path.dirname(

os.path.dirname(os.path.abspath(__file__)))

if os.path.exists(os.path.join(awscli_dir, 'dist')):

shutil.rmtree(os.path.join(awscli_dir, 'dist'))

with cd(awscli_dir):

run('%s setup.py sdist' % sys.executable)

filename = os.listdir('dist')[0]

shutil.move(os.path.join('dist', filename),

os.path.join(scratch_dir, filename))

def create_bootstrap_script(scratch_dir):

install_script = os.path.join(

os.path.dirname(os.path.abspath(__file__)), 'install')

shutil.copy(install_script, os.path.join(scratch_dir, 'install'))

def zip_dir(scratch_dir):

basename = 'awscli-bundle.zip'

dirname, tmpdir = os.path.split(scratch_dir)

final_dir_name = os.path.join(dirname, 'awscli-bundle')

if os.path.isdir(final_dir_name):

shutil.rmtree(final_dir_name)

shutil.move(scratch_dir, final_dir_name)

with cd(dirname):

with zipfile.ZipFile(basename, 'w', zipfile.ZIP_DEFLATED) as zipped:

for root, dirnames, filenames in os.walk('awscli-bundle'):

for filename in filenames:

zipped.write(os.path.join(root, filename))

return os.path.join(dirname, basename)

def verify_preconditions():

# The pip version looks like:

# 'pip 1.4.1 from ....'

pip_version = run(

'%s -m pip --version' % sys.executable).strip().split()[1]

# Virtualenv version just has the version string: '1.14.5\n'

virtualenv_version = run(

'%s -m virtualenv --version' % sys.executable).strip()

_min_version_required('9.0.1', pip_version, 'pip')

_min_version_required('15.1.0', virtualenv_version, 'virtualenv')

def _min_version_required(min_version, actual_version, name):

# precondition: min_version is major.minor.patch

# actual_version is major.minor.patch

min_split = min_version.split('.')

actual_split = actual_version.decode('utf-8').split('.')

for min_version_part, actual_version_part in zip(min_split, actual_split):

if int(actual_version_part) >= int(min_version_part):

return

raise ValueError("%s requires at least version %s, but version %s was "

"found." % (name, min_version, actual_version))

def main():

verify_preconditions()

scratch_dir = create_scratch_dir()

package_dir = os.path.join(scratch_dir, 'packages')

print("Bundle dir at: %s" % scratch_dir)

download_package_tarballs(

package_dir,

packages=[

'virtualenv', 'ordereddict', 'simplejson',

'argparse', 'python-dateutil', 'urllib3',

]

)

# Some packages require setup time dependencies, and so we will need to

# manually install them. We isolate them to a particular directory so we

# can run the install before the things they're dependent on. We have to do

# this because pip won't actually find them since it doesn't handle build

# dependencies.

setup_dir = os.path.join(package_dir, 'setup')

download_package_tarballs(

setup_dir,

packages=['setuptools-scm']

)

download_cli_deps(package_dir)

add_cli_sdist(package_dir)

create_bootstrap_script(scratch_dir)

zip_filename = zip_dir(scratch_dir)

print("Zipped bundle installer is at: %s" % zip_filename)

if __name__ == '__main__':

main()

aws-cli/scripts/make-bundle at develop · github-38438-org/aws-cli · GitHub

Uh oh!

Expand file tree

Breadcrumbs

make-bundle

Latest commit

History

Breadcrumbs

make-bundle

File metadata and controls