google-cloud-python/scripts/script_utils.py at master · charlewn/google-cloud-python · GitHub

History

352 lines (278 loc) · 11.8 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Common helpers for testing scripts."""

from __future__ import print_function

import ast

import os

import subprocess

PROJECT_ROOT = os.path.abspath(

os.path.join(os.path.dirname(__file__), '..'))

LOCAL_REMOTE_ENV = 'GOOGLE_CLOUD_TESTING_REMOTE'

LOCAL_BRANCH_ENV = 'GOOGLE_CLOUD_TESTING_BRANCH'

IN_TRAVIS_ENV = 'TRAVIS'

TRAVIS_PR_ENV = 'TRAVIS_PULL_REQUEST'

TRAVIS_BRANCH_ENV = 'TRAVIS_BRANCH'

INST_REQS_KWARG = 'install_requires'

REQ_VAR = 'REQUIREMENTS'

PACKAGE_PREFIX = 'google-cloud-'

def in_travis():

"""Detect if we are running in Travis.

.. _Travis env docs: https://docs.travis-ci.com/user/\

environment-variables\

#Default-Environment-Variables

See `Travis env docs`_.

:rtype: bool

:returns: Flag indicating if we are running on Travis.

"""

return os.getenv(IN_TRAVIS_ENV) == 'true'

def in_travis_pr():

"""Detect if we are running in a pull request on Travis.

.. _Travis env docs: https://docs.travis-ci.com/user/\

environment-variables\

#Default-Environment-Variables

See `Travis env docs`_.

.. note::

This assumes we already know we are running in Travis.

:rtype: bool

:returns: Flag indicating if we are in a pull request on Travis.

"""

# NOTE: We're a little extra cautious and make sure that the

# PR environment variable is an integer.

try:

int(os.getenv(TRAVIS_PR_ENV, ''))

return True

except ValueError:

return False

def travis_branch():

"""Get the current branch of the PR.

.. _Travis env docs: https://docs.travis-ci.com/user/\

environment-variables\

#Default-Environment-Variables

See `Travis env docs`_.

.. note::

This assumes we already know we are running in Travis

during a PR.

:rtype: str

:returns: The name of the branch the current pull request is

changed against.

:raises: :class:`~exceptions.OSError` if the ``TRAVIS_BRANCH_ENV``

environment variable isn't set during a pull request

build.

"""

try:

return os.environ[TRAVIS_BRANCH_ENV]

except KeyError:

msg = ('Pull request build does not have an '

'associated branch set (via %s)') % (TRAVIS_BRANCH_ENV,)

raise OSError(msg)

def check_output(*args):

"""Run a command on the operation system.

:type args: tuple

:param args: Arguments to pass to ``subprocess.check_output``.

:rtype: str

:returns: The raw STDOUT from the command (converted from bytes

if necessary).

"""

cmd_output = subprocess.check_output(args)

# On Python 3, this returns bytes (from STDOUT), so we

# convert to a string.

cmd_output = cmd_output.decode('utf-8')

# Also strip the output since it usually has a trailing newline.

return cmd_output.strip()

def rootname(filename):

"""Get the root directory that a file is contained in.

:type filename: str

:param filename: The path / name of a file.

:rtype: str

:returns: The root directory containing the file.

"""

if os.path.sep not in filename:

return ''

else:

file_root, _ = filename.split(os.path.sep, 1)

return file_root

def get_changed_packages(blob_name1, blob_name2, package_list):

"""Get a list of packages which have changed between two changesets.

:type blob_name1: str

:param blob_name1: The name of a commit hash or branch name or other

``git`` artifact.

:type blob_name2: str

:param blob_name2: The name of a commit hash or branch name or other

``git`` artifact.

:type package_list: list

:param package_list: The list of **all** valid packages with unit tests.

:rtype: list

:returns: A list of all package directories that have changed

between ``blob_name1`` and ``blob_name2``. Starts

with a list of valid packages (``package_list``)

and filters out the unchanged directories.

"""

changed_files = check_output(

'git', 'diff', '--name-only', blob_name1, blob_name2)

changed_files = changed_files.split('\n')

result = set()

for filename in changed_files:

file_root = rootname(filename)

if file_root in package_list:

result.add(file_root)

return sorted(result)

def local_diff_branch():

"""Get a remote branch to diff against in a local checkout.

Checks if the the local remote and local branch environment

variables specify a remote branch.

:rtype: str

:returns: The diffbase `{remote}/{branch}` if the environment

variables are defined. If not, returns ``None``.

"""

# Only allow specified remote and branch in local dev.

remote = os.getenv(LOCAL_REMOTE_ENV)

branch = os.getenv(LOCAL_BRANCH_ENV)

if remote is not None and branch is not None:

return '%s/%s' % (remote, branch)

def get_affected_files(allow_limited=True):

"""Gets a list of files in the repository.

By default, returns all files via ``git ls-files``. However, in some cases

uses a specific commit or branch (a so-called diff base) to compare

against for changed files. (This requires ``allow_limited=True``.)

To speed up linting on Travis pull requests against master, we manually

set the diff base to the branch the pull request is against. We don't do

this on "push" builds since "master" will be the currently checked out

code. One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base

but this value is not dependable.

To allow faster local ``tox`` runs, the local remote and local branch

environment variables can be set to specify a remote branch to diff

against.

:type allow_limited: bool

:param allow_limited: Boolean indicating if a reduced set of files can

be used.

:rtype: pair

:returns: Tuple of the diff base using the list of filenames to be

linted.

"""

diff_base = None

if in_travis():

# In the case of a pull request into a branch, we want to

# diff against HEAD in that branch.

if in_travis_pr():

diff_base = travis_branch()

else:

diff_base = local_diff_branch()

if diff_base is not None and allow_limited:

result = subprocess.check_output(['git', 'diff', '--name-only',

diff_base])

print('Using files changed relative to %s:' % (diff_base,))

print('-' * 60)

print(result.rstrip('\n')) # Don't print trailing newlines.

print('-' * 60)

else:

print('Diff base not specified, listing all files in repository.')

result = subprocess.check_output(['git', 'ls-files'])

# Only return filenames that exist. For example, 'git diff --name-only'

# could spit out deleted / renamed files. Another alternative could

# be to use 'git diff --name-status' and filter out files with a

# status of 'D'.

filenames = [filename

for filename in result.rstrip('\n').split('\n')

if os.path.exists(filename)]

return filenames, diff_base

def get_required_packages(file_contents):

"""Get required packages from a ``setup.py`` file.

Makes the following assumptions:

* ``install_requires=REQUIREMENTS`` occurs in the call to

``setup()`` in the ``file_contents``.

* The text ``install_requires`` occurs nowhere else in the file.

* The text ``REQUIREMENTS`` only appears when being passed to

``setup()`` (as above) and when being defined.

* The ``REQUIREMENTS`` variable is a list and the text from the

``setup.py`` file containing that list can be parsed using

``ast.literal_eval()``.

:type file_contents: str

:param file_contents: The contents of a ``setup.py`` file.

:rtype: list

:returns: The list of required packages.

:raises: :class:`~exceptions.ValueError` if the file is in an

unexpected format.

"""

# Make sure the only ``install_requires`` happens in the

# call to setup()

if file_contents.count(INST_REQS_KWARG) != 1:

raise ValueError('Expected only one use of keyword',

INST_REQS_KWARG, file_contents)

# Make sure the only usage of ``install_requires`` is to set

# install_requires=REQUIREMENTS.

keyword_stmt = INST_REQS_KWARG + '=' + REQ_VAR

if file_contents.count(keyword_stmt) != 1:

raise ValueError('Expected keyword to be set with variable',

INST_REQS_KWARG, REQ_VAR, file_contents)

# Split file on ``REQUIREMENTS`` variable while asserting that

# it only appear twice.

_, reqs_section, _ = file_contents.split(REQ_VAR)

# Find ``REQUIREMENTS`` list variable defined in ``reqs_section``.

reqs_begin = reqs_section.index('[')

reqs_end = reqs_section.index(']') + 1

# Convert the text to an actual list, but make sure no

# locals or globals can be used.

reqs_list_text = reqs_section[reqs_begin:reqs_end]

# We use literal_eval() because it limits to evaluating

# strings that only consist of a few Python literals: strings,

# numbers, tuples, lists, dicts, booleans, and None.

requirements = ast.literal_eval(reqs_list_text)

# Take the list of requirements and strip off the package name

# from each requirement.

result = []

for required in requirements:

parts = required.split()

result.append(parts[0])

return result

def get_dependency_graph(package_list):

"""Get a directed graph of package dependencies.

:type package_list: list

:param package_list: The list of **all** valid packages.

:rtype: dict

:returns: A dictionary where keys are packages and values are

the set of packages that depend on the key.

"""

result = {package: set() for package in package_list}

for package in package_list:

setup_file = os.path.join(PROJECT_ROOT, package,

'setup.py')

with open(setup_file, 'r') as file_obj:

file_contents = file_obj.read()

requirements = get_required_packages(file_contents)

for requirement in requirements:

if not requirement.startswith(PACKAGE_PREFIX):

continue

_, req_package = requirement.split(PACKAGE_PREFIX)

req_package = req_package.replace('-', '_')

result[req_package].add(package)

return result

def follow_dependencies(subset, package_list):

"""Get a directed graph of package dependencies.

:type subset: list

:param subset: List of a subset of package names.

:type package_list: list

:param package_list: The list of **all** valid packages.

:rtype: list

:returns: An expanded list of packages containing everything

in ``subset`` and any packages that depend on those.

"""

dependency_graph = get_dependency_graph(package_list)

curr_pkgs = None

updated_pkgs = set(subset)

while curr_pkgs != updated_pkgs:

curr_pkgs = updated_pkgs

updated_pkgs = set(curr_pkgs)

for package in curr_pkgs:

updated_pkgs.update(dependency_graph[package])

return sorted(curr_pkgs)

google-cloud-python/scripts/script_utils.py at master · charlewn/google-cloud-python · GitHub

Expand file tree

Breadcrumbs

script_utils.py

Latest commit

History

Breadcrumbs

script_utils.py

File metadata and controls