python-for-android/pythonforandroid/build.py at master · matrix-io/python-for-android · GitHub

History

1036 lines (884 loc) · 39.5 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623

624

625

626

627

628

629

630

631

632

633

634

635

636

637

638

639

640

641

642

643

644

645

646

647

648

649

650

651

652

653

654

655

656

657

658

659

660

661

662

663

664

665

666

667

668

669

670

671

672

673

674

675

676

677

678

679

680

681

682

683

684

685

686

687

688

689

690

691

692

693

694

695

696

697

698

699

700

701

702

703

704

705

706

707

708

709

710

711

712

713

714

715

716

717

718

719

720

721

722

723

724

725

726

727

728

729

730

731

732

733

734

735

736

737

738

739

740

741

742

743

744

745

746

747

748

749

750

751

752

753

754

755

756

757

758

759

760

761

762

763

764

765

766

767

768

769

770

771

772

773

774

775

776

777

778

779

780

781

782

783

784

785

786

787

788

789

790

791

792

793

794

795

796

797

798

799

800

801

802

803

804

805

806

807

808

809

810

811

812

813

814

815

816

817

818

819

820

821

822

823

824

825

826

827

828

829

830

831

832

833

834

835

836

837

838

839

840

841

842

843

844

845

846

847

848

849

850

851

852

853

854

855

856

857

858

859

860

861

862

863

864

865

866

867

868

869

870

871

872

873

874

875

876

877

878

879

880

881

882

883

884

885

886

887

888

889

890

891

892

893

894

895

896

897

898

899

900

901

902

903

904

905

906

907

908

909

910

911

912

913

914

915

916

917

918

919

920

921

922

923

924

925

926

927

928

929

930

931

932

933

934

935

936

937

938

939

940

941

942

943

944

945

946

947

948

949

950

951

952

953

954

955

956

957

958

959

960

961

962

963

964

965

966

967

968

969

970

971

972

973

974

975

976

977

978

979

980

981

982

983

984

985

986

987

988

989

990

991

992

993

994

995

996

997

998

999

1000

from __future__ import print_function

from os.path import (join, realpath, dirname, expanduser, exists,

split, isdir)

from os import environ

import copy

import os

import glob

import sys

import re

import sh

import shutil

import subprocess

from pythonforandroid.util import (

current_directory, ensure_dir, get_virtualenv_executable,

BuildInterruptingException

)

from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)

from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64

from pythonforandroid.pythonpackage import get_package_name

from pythonforandroid.recipe import CythonRecipe, Recipe

from pythonforandroid.recommendations import (

check_ndk_version, check_target_api, check_ndk_api,

RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)

class Context(object):

'''A build context. If anything will be built, an instance this class

will be instantiated and used to hold all the build state.'''

env = environ.copy()

# the filepath of toolchain.py

root_dir = None

# the root dir where builds and dists will be stored

storage_dir = None

# in which bootstraps are copied for building

# and recipes are built

build_dir = None

# the Android project folder where everything ends up

dist_dir = None

# where Android libs are cached after build

# but before being placed in dists

libs_dir = None

aars_dir = None

# Whether setup.py or similar should be used if present:

use_setup_py = False

ccache = None # whether to use ccache

cython = None # the cython interpreter name

ndk_platform = None # the ndk platform directory

dist_name = None # should be deprecated in favour of self.dist.dist_name

bootstrap = None

bootstrap_build_dir = None

recipe_build_order = None # Will hold the list of all built recipes

symlink_java_src = False # If True, will symlink instead of copying during build

java_build_tool = 'auto'

@property

def packages_path(self):

'''Where packages are downloaded before being unpacked'''

return join(self.storage_dir, 'packages')

@property

def templates_dir(self):

return join(self.root_dir, 'templates')

@property

def libs_dir(self):

# Was previously hardcoded as self.build_dir/libs

dir = join(self.build_dir, 'libs_collections',

self.bootstrap.distribution.name)

ensure_dir(dir)

return dir

@property

def javaclass_dir(self):

# Was previously hardcoded as self.build_dir/java

dir = join(self.build_dir, 'javaclasses',

self.bootstrap.distribution.name)

ensure_dir(dir)

return dir

@property

def aars_dir(self):

dir = join(self.build_dir, 'aars', self.bootstrap.distribution.name)

ensure_dir(dir)

return dir

@property

def python_installs_dir(self):

dir = join(self.build_dir, 'python-installs')

ensure_dir(dir)

return dir

def get_python_install_dir(self):

dir = join(self.python_installs_dir, self.bootstrap.distribution.name)

return dir

def setup_dirs(self, storage_dir):

'''Calculates all the storage and build dirs, and makes sure

the directories exist where necessary.'''

self.storage_dir = expanduser(storage_dir)

if ' ' in self.storage_dir:

raise ValueError('storage dir path cannot contain spaces, please '

'specify a path with --storage-dir')

self.build_dir = join(self.storage_dir, 'build')

self.dist_dir = join(self.storage_dir, 'dists')

def ensure_dirs(self):

ensure_dir(self.storage_dir)

ensure_dir(self.build_dir)

ensure_dir(self.dist_dir)

ensure_dir(join(self.build_dir, 'bootstrap_builds'))

ensure_dir(join(self.build_dir, 'other_builds'))

@property

def android_api(self):

'''The Android API being targeted.'''

if self._android_api is None:

raise ValueError('Tried to access android_api but it has not '

'been set - this should not happen, something '

'went wrong!')

return self._android_api

@android_api.setter

def android_api(self, value):

self._android_api = value

@property

def ndk_api(self):

'''The API number compile against'''

if self._ndk_api is None:

raise ValueError('Tried to access ndk_api but it has not '

'been set - this should not happen, something '

'went wrong!')

return self._ndk_api

@ndk_api.setter

def ndk_api(self, value):

self._ndk_api = value

@property

def sdk_dir(self):

'''The path to the Android SDK.'''

if self._sdk_dir is None:

raise ValueError('Tried to access sdk_dir but it has not '

'been set - this should not happen, something '

'went wrong!')

return self._sdk_dir

@sdk_dir.setter

def sdk_dir(self, value):

self._sdk_dir = value

@property

def ndk_dir(self):

'''The path to the Android NDK.'''

if self._ndk_dir is None:

raise ValueError('Tried to access ndk_dir but it has not '

'been set - this should not happen, something '

'went wrong!')

return self._ndk_dir

@ndk_dir.setter

def ndk_dir(self, value):

self._ndk_dir = value

def prepare_build_environment(self,

user_sdk_dir,

user_ndk_dir,

user_android_api,

user_ndk_api):

'''Checks that build dependencies exist and sets internal variables

for the Android SDK etc.

..warning:: This *must* be called before trying any build stuff

'''

self.ensure_dirs()

if self._build_env_prepared:

return

ok = True

# Work out where the Android SDK is

sdk_dir = None

if user_sdk_dir:

sdk_dir = user_sdk_dir

# This is the old P4A-specific var

if sdk_dir is None:

sdk_dir = environ.get('ANDROIDSDK', None)

# This seems used more conventionally

if sdk_dir is None:

sdk_dir = environ.get('ANDROID_HOME', None)

# Checks in the buildozer SDK dir, useful for debug tests of p4a

if sdk_dir is None:

possible_dirs = glob.glob(expanduser(join(

'~', '.buildozer', 'android', 'platform', 'android-sdk-*')))

possible_dirs = [d for d in possible_dirs if not

(d.endswith('.bz2') or d.endswith('.gz'))]

if possible_dirs:

info('Found possible SDK dirs in buildozer dir: {}'.format(

', '.join([d.split(os.sep)[-1] for d in possible_dirs])))

info('Will attempt to use SDK at {}'.format(possible_dirs[0]))

warning('This SDK lookup is intended for debug only, if you '

'use python-for-android much you should probably '

'maintain your own SDK download.')

sdk_dir = possible_dirs[0]

if sdk_dir is None:

raise BuildInterruptingException('Android SDK dir was not specified, exiting.')

self.sdk_dir = realpath(sdk_dir)

# Check what Android API we're using

android_api = None

if user_android_api:

android_api = user_android_api

info('Getting Android API version from user argument: {}'.format(android_api))

elif 'ANDROIDAPI' in environ:

android_api = environ['ANDROIDAPI']

info('Found Android API target in $ANDROIDAPI: {}'.format(android_api))

else:

info('Android API target was not set manually, using '

'the default of {}'.format(RECOMMENDED_TARGET_API))

android_api = RECOMMENDED_TARGET_API

android_api = int(android_api)

self.android_api = android_api

check_target_api(android_api, self.archs[0].arch)

if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):

avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager'))

targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n')

elif exists(join(sdk_dir, 'tools', 'android')):

android = sh.Command(join(sdk_dir, 'tools', 'android'))

targets = android('list').stdout.decode('utf-8').split('\n')

else:

raise BuildInterruptingException(

'Could not find `android` or `sdkmanager` binaries in Android SDK',

instructions='Make sure the path to the Android SDK is correct')

apis = [s for s in targets if re.match(r'^ *API level: ', s)]

apis = [re.findall(r'[0-9]+', s) for s in apis]

apis = [int(s[0]) for s in apis if s]

info('Available Android APIs are ({})'.format(

', '.join(map(str, apis))))

if android_api in apis:

info(('Requested API target {} is available, '

'continuing.').format(android_api))

else:

raise BuildInterruptingException(

('Requested API target {} is not available, install '

'it with the SDK android tool.').format(android_api))

# Find the Android NDK

# Could also use ANDROID_NDK, but doesn't look like many tools use this

ndk_dir = None

if user_ndk_dir:

ndk_dir = user_ndk_dir

info('Getting NDK dir from from user argument')

if ndk_dir is None: # The old P4A-specific dir

ndk_dir = environ.get('ANDROIDNDK', None)

if ndk_dir is not None:

info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir))

if ndk_dir is None: # Apparently the most common convention

ndk_dir = environ.get('NDK_HOME', None)

if ndk_dir is not None:

info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir))

if ndk_dir is None: # Another convention (with maven?)

ndk_dir = environ.get('ANDROID_NDK_HOME', None)

if ndk_dir is not None:

info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir))

if ndk_dir is None: # Checks in the buildozer NDK dir, useful

# # for debug tests of p4a

possible_dirs = glob.glob(expanduser(join(

'~', '.buildozer', 'android', 'platform', 'android-ndk-r*')))

if possible_dirs:

info('Found possible NDK dirs in buildozer dir: {}'.format(

', '.join([d.split(os.sep)[-1] for d in possible_dirs])))

info('Will attempt to use NDK at {}'.format(possible_dirs[0]))

warning('This NDK lookup is intended for debug only, if you '

'use python-for-android much you should probably '

'maintain your own NDK download.')

ndk_dir = possible_dirs[0]

if ndk_dir is None:

raise BuildInterruptingException('Android NDK dir was not specified')

self.ndk_dir = realpath(ndk_dir)

check_ndk_version(ndk_dir)

ndk_api = None

if user_ndk_api:

ndk_api = user_ndk_api

info('Getting NDK API version (i.e. minimum supported API) from user argument')

elif 'NDKAPI' in environ:

ndk_api = environ.get('NDKAPI', None)

info('Found Android API target in $NDKAPI')

else:

ndk_api = min(self.android_api, RECOMMENDED_NDK_API)

warning('NDK API target was not set manually, using '

'the default of {} = min(android-api={}, default ndk-api={})'.format(

ndk_api, self.android_api, RECOMMENDED_NDK_API))

ndk_api = int(ndk_api)

self.ndk_api = ndk_api

check_ndk_api(ndk_api, self.android_api)

virtualenv = get_virtualenv_executable()

if virtualenv is None:

raise IOError('Couldn\'t find a virtualenv executable, '

'you must install this to use p4a.')

self.virtualenv = virtualenv

info('Found virtualenv at {}'.format(virtualenv))

# path to some tools

self.ccache = sh.which("ccache")

if not self.ccache:

info('ccache is missing, the build will not be optimized in the '

'future.')

for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"):

cython = sh.which(cython_fn)

if cython:

self.cython = cython

break

else:

raise BuildInterruptingException('No cython binary found.')

if not self.cython:

ok = False

warning("Missing requirement: cython is not installed")

# This would need to be changed if supporting multiarch APKs

arch = self.archs[0]

platform_dir = arch.platform_dir

toolchain_prefix = arch.toolchain_prefix

toolchain_version = None

self.ndk_platform = join(

self.ndk_dir,

'platforms',

'android-{}'.format(self.ndk_api),

platform_dir)

if not exists(self.ndk_platform):

warning('ndk_platform doesn\'t exist: {}'.format(

self.ndk_platform))

ok = False

py_platform = sys.platform

if py_platform in ['linux2', 'linux3']:

py_platform = 'linux'

toolchain_versions = []

toolchain_path = join(self.ndk_dir, 'toolchains')

if isdir(toolchain_path):

toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path,

toolchain_prefix))

toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:]

for path in toolchain_contents]

else:

warning('Could not find toolchain subdirectory!')

ok = False

toolchain_versions.sort()

toolchain_versions_gcc = []

for toolchain_version in toolchain_versions:

if toolchain_version[0].isdigit():

# GCC toolchains begin with a number

toolchain_versions_gcc.append(toolchain_version)

if toolchain_versions:

info('Found the following toolchain versions: {}'.format(

toolchain_versions))

info('Picking the latest gcc toolchain, here {}'.format(

toolchain_versions_gcc[-1]))

toolchain_version = toolchain_versions_gcc[-1]

else:

warning('Could not find any toolchain for {}!'.format(

toolchain_prefix))

ok = False

self.toolchain_prefix = toolchain_prefix

self.toolchain_version = toolchain_version

# Modify the path so that sh finds modules appropriately

environ['PATH'] = (

'{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/'

'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/'

'{toolchain_prefix}-{toolchain_version}/prebuilt/'

'{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/'

'tools:{path}').format(

sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir,

toolchain_prefix=toolchain_prefix,

toolchain_version=toolchain_version,

py_platform=py_platform, path=environ.get('PATH'))

for executable in ("pkg-config", "autoconf", "automake", "libtoolize",

"tar", "bzip2", "unzip", "make", "gcc", "g++"):

if not sh.which(executable):

warning("Missing executable: {} is not installed".format(

executable))

if not ok:

raise BuildInterruptingException(

'python-for-android cannot continue due to the missing executables above')

def __init__(self):

super(Context, self).__init__()

self.include_dirs = []

self._build_env_prepared = False

self._sdk_dir = None

self._ndk_dir = None

self._android_api = None

self._ndk_api = None

self.ndk = None

self.toolchain_prefix = None

self.toolchain_version = None

self.local_recipes = None

self.copy_libs = False

# this list should contain all Archs, it is pruned later

self.archs = (

ArchARM(self),

ArchARMv7_a(self),

Archx86(self),

Archx86_64(self),

ArchAarch_64(self),

)

self.root_dir = realpath(dirname(__file__))

# remove the most obvious flags that can break the compilation

self.env.pop("LDFLAGS", None)

self.env.pop("ARCHFLAGS", None)

self.env.pop("CFLAGS", None)

self.python_recipe = None # Set by TargetPythonRecipe

def set_archs(self, arch_names):

all_archs = self.archs

new_archs = set()

for name in arch_names:

matching = [arch for arch in all_archs if arch.arch == name]

for match in matching:

new_archs.add(match)

self.archs = list(new_archs)

if not self.archs:

raise BuildInterruptingException('Asked to compile for no Archs, so failing.')

info('Will compile for the following archs: {}'.format(

', '.join([arch.arch for arch in self.archs])))

def prepare_bootstrap(self, bs):

bs.ctx = self

self.bootstrap = bs

self.bootstrap.prepare_build_dir()

self.bootstrap_build_dir = self.bootstrap.build_dir

def prepare_dist(self, name):

self.dist_name = name

self.bootstrap.prepare_dist_dir(self.dist_name)

def get_site_packages_dir(self, arch=None):

'''Returns the location of site-packages in the python-install build

dir.

'''

if self.python_recipe.name == 'python2legacy':

return join(self.get_python_install_dir(),

'lib', 'python2.7', 'site-packages')

return self.get_python_install_dir()

def get_libs_dir(self, arch):

'''The libs dir for a given arch.'''

ensure_dir(join(self.libs_dir, arch))

return join(self.libs_dir, arch)

def has_lib(self, arch, lib):

return exists(join(self.get_libs_dir(arch), lib))

def has_package(self, name, arch=None):

# If this is a file path, it'll need special handling:

if (name.find("/") >= 0 or name.find("\\") >= 0) and \

name.find("://") < 0: # (:// would indicate an url)

if not os.path.exists(name):

# Non-existing dir, cannot look this up.

return False

try:

name = get_package_name(os.path.abspath(name))

except ValueError:

# Failed to look up any meaningful name.

return False

# Try to look up recipe by name:

try:

recipe = Recipe.get_recipe(name, self)

except ValueError:

pass

else:

name = getattr(recipe, 'site_packages_name', None) or name

name = name.replace('.', '/')

site_packages_dir = self.get_site_packages_dir(arch)

return (exists(join(site_packages_dir, name)) or

exists(join(site_packages_dir, name + '.py')) or

exists(join(site_packages_dir, name + '.pyc')) or

exists(join(site_packages_dir, name + '.pyo')) or

exists(join(site_packages_dir, name + '.so')) or

glob.glob(join(site_packages_dir, name + '-*.egg')))

def not_has_package(self, name, arch=None):

return not self.has_package(name, arch)

def build_recipes(build_order, python_modules, ctx, project_dir,

ignore_project_setup_py=False

# Put recipes in correct build order

info_notify("Recipe build order is {}".format(build_order))

if python_modules:

python_modules = sorted(set(python_modules))

info_notify(

('The requirements ({}) were not found as recipes, they will be '

'installed with pip.').format(', '.join(python_modules)))

recipes = [Recipe.get_recipe(name, ctx) for name in build_order]

# download is arch independent

info_main('# Downloading recipes ')

for recipe in recipes:

recipe.download_if_necessary()

for arch in ctx.archs:

info_main('# Building all recipes for arch {}'.format(arch.arch))

info_main('# Unpacking recipes')

for recipe in recipes:

ensure_dir(recipe.get_build_container_dir(arch.arch))

recipe.prepare_build_dir(arch.arch)

info_main('# Prebuilding recipes')

# 2) prebuild packages

for recipe in recipes:

info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))

recipe.prebuild_arch(arch)

recipe.apply_patches(arch)

# 3) build packages

info_main('# Building recipes')

for recipe in recipes:

info_main('Building {} for {}'.format(recipe.name, arch.arch))

if recipe.should_build(arch):

recipe.build_arch(arch)

else:

info('{} said it is already built, skipping'

.format(recipe.name))

# 4) biglink everything

info_main('# Biglinking object files')

if not ctx.python_recipe or not ctx.python_recipe.from_crystax:

biglink(ctx, arch)

else:

info('NDK is crystax, skipping biglink (will this work?)')

# 5) postbuild packages

info_main('# Postbuilding recipes')

for recipe in recipes:

info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))

recipe.postbuild_arch(arch)

info_main('# Installing pure Python modules')

run_pymodules_install(

ctx, python_modules, project_dir,

ignore_setup_py=ignore_project_setup_py

)

return

def project_has_setup_py(project_dir):

if project_dir is not None and \

(os.path.exists(os.path.join(project_dir,

"setup.py")) or

os.path.exists(os.path.join(project_dir,

"pyproject.toml"))

return True

return False

def run_pymodules_install(ctx, modules, project_dir=None,

ignore_setup_py=False):

""" This function will take care of all non-recipe things, by:

1. Processing them from --requirements (the modules argument)

and installing them

2. Installing the user project/app itself via setup.py if

ignore_setup_py=True

"""

info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')

modules = list(filter(ctx.not_has_package, modules))

# Bail out if no python deps and no setup.py to process:

if not modules and (

ignore_setup_py or

project_dir is None or

not project_has_setup_py(project_dir)

info('No Python modules and no setup.py to process, skipping')

return

# Output messages about what we're going to do:

if modules:

info('The requirements ({}) don\'t have recipes, attempting to '

'install them with pip'.format(', '.join(modules)))

info('If this fails, it may mean that the module has compiled '

'components and needs a recipe.')

if project_dir is not None and \

project_has_setup_py(project_dir) and not ignore_setup_py:

info('Will process project install, if it fails then the '

'project may not be compatible for Android install.')

venv = sh.Command(ctx.virtualenv)

with current_directory(join(ctx.build_dir)):

shprint(venv,

'--python=python{}'.format(

ctx.python_recipe.major_minor_version_string.

partition(".")[0]

'venv'

)

# Prepare base environment and upgrade pip:

base_env = copy.copy(os.environ)

base_env["PYTHONPATH"] = ctx.get_site_packages_dir()

info('Upgrade pip to latest version')

shprint(sh.bash, '-c', (

"source venv/bin/activate && pip install -U pip"

), _env=copy.copy(base_env))

# Install Cython in case modules need it to build:

info('Install Cython in case one of the modules needs it to build')

shprint(sh.bash, '-c', (

"venv/bin/pip install Cython"

), _env=copy.copy(base_env))

# Get environment variables for build (with CC/compiler set):

standard_recipe = CythonRecipe()

standard_recipe.ctx = ctx

# (note: following line enables explicit -lpython... linker options)

standard_recipe.call_hostpython_via_targetpython = False

recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])

env = copy.copy(base_env)

env.update(recipe_env)

# Make sure our build package dir is available, and the virtualenv

# site packages come FIRST (so the proper pip version is used):

env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()

env["PYTHONPATH"] = os.path.abspath(join(

ctx.build_dir, "venv", "lib",

"python" + ctx.python_recipe.major_minor_version_string,

"site-packages")) + ":" + env["PYTHONPATH"]

# Install the manually specified requirements first:

if not modules:

info('There are no Python modules to install, skipping')

else:

info('Creating a requirements.txt file for the Python modules')

with open('requirements.txt', 'w') as fileh:

for module in modules:

key = 'VERSION_' + module

if key in environ:

line = '{}=={}\n'.format(module, environ[key])

else:

line = '{}\n'.format(module)

fileh.write(line)

info('Installing Python modules with pip')

info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '

'A reason for this is often modules compiling '

'native code that is unaware of Android cross-compilation '

'and does not work without additional '

'changes / workarounds.')

shprint(sh.bash, '-c', (

"venv/bin/pip " +

"install -v --target '{0}' --no-deps -r requirements.txt"

).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),

_env=copy.copy(env))

# Afterwards, run setup.py if present:

if project_dir is not None and (

project_has_setup_py(project_dir) and not ignore_setup_py

with current_directory(project_dir):

info('got setup.py or similar, running project install. ' +

'(disable this behavior with --ignore-setup-py)')

# Compute & output the constraints we will use:

info('Contents that will be used for constraints.txt:')

constraints = subprocess.check_output([

join(

ctx.build_dir, "venv", "bin", "pip"

"freeze"

], env=copy.copy(env))

try:

constraints = constraints.decode("utf-8", "replace")

except AttributeError:

pass

info(constraints)

# Make sure all packages found are fixed in version

# by writing a constraint file, to avoid recipes being

# upgraded & reinstalled:

with open('constraints.txt', 'wb') as fileh:

fileh.write(constraints.encode("utf-8", "replace"))

info('Populating venv\'s site-packages with '

'ctx.get_site_packages_dir()...')

# Copy dist contents into site-packages for discovery.

# Why this is needed:

# --target is somewhat evil and messes with discovery of

# packages in PYTHONPATH if that also includes the target

# folder. So we need to use the regular virtualenv

# site-packages folder instead.

# Reference:

# https://github.com/pypa/pip/issues/6223

ctx_site_packages_dir = os.path.normpath(

os.path.abspath(ctx.get_site_packages_dir())

)

venv_site_packages_dir = os.path.normpath(os.path.join(

ctx.build_dir, "venv", "lib", [

f for f in os.listdir(os.path.join(

ctx.build_dir, "venv", "lib"

)) if f.startswith("python")

][0], "site-packages"

))

copied_over_contents = []

for f in os.listdir(ctx_site_packages_dir):

full_path = os.path.join(ctx_site_packages_dir, f)

if not os.path.exists(os.path.join(

venv_site_packages_dir, f

)):

if os.path.isdir(full_path):

shutil.copytree(full_path, os.path.join(

venv_site_packages_dir, f

))

else:

shutil.copy2(full_path, os.path.join(

venv_site_packages_dir, f

))

copied_over_contents.append(f)

# Get listing of virtualenv's site-packages, to see the

# newly added things afterwards & copy them back into

# the distribution folder / build context site-packages:

previous_venv_contents = os.listdir(venv_site_packages_dir)

# Actually run setup.py:

info('Launching package install...')

shprint(sh.bash, '-c', (

"'" + join(

ctx.build_dir, "venv", "bin", "pip"

).replace("'", "'\"'\"'") + "' " +

"install -c constraints.txt -v ."

).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),

_env=copy.copy(env))

# Go over all new additions and copy them back:

info('Copying additions resulting from setup.py back ' +

'into ctx.get_site_packages_dir()...')

new_venv_additions = []

for f in (set(os.listdir(venv_site_packages_dir)) -

set(previous_venv_contents)):

new_venv_additions.append(f)

full_path = os.path.join(venv_site_packages_dir, f)

if os.path.isdir(full_path):

shutil.copytree(full_path, os.path.join(

ctx_site_packages_dir, f

))

else:

shutil.copy2(full_path, os.path.join(

ctx_site_packages_dir, f

))

# Undo all the changes we did to the venv-site packages:

info('Reverting additions to virtualenv\'s site-packages...')

for f in set(copied_over_contents + new_venv_additions):

full_path = os.path.join(venv_site_packages_dir, f)

if os.path.isdir(full_path):

shutil.rmtree(full_path)

else:

os.remove(full_path)

elif not ignore_setup_py:

info("No setup.py found in project directory: " +

str(project_dir)

)

# Strip object files after potential Cython or native code builds:

standard_recipe.strip_object_files(ctx.archs[0], env,

build_dir=ctx.build_dir)

def biglink(ctx, arch):

# First, collate object files from each recipe

info('Collating object files from each recipe')

obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')

ensure_dir(obj_dir)

recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]

for recipe in recipes:

recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),

'objects_{}'.format(recipe.name))

if not exists(recipe_obj_dir):

info('{} recipe has no biglinkable files dir, skipping'

.format(recipe.name))

continue

files = glob.glob(join(recipe_obj_dir, '*'))

if not len(files):

info('{} recipe has no biglinkable files, skipping'

.format(recipe.name))

continue

info('{} recipe has object files, copying'.format(recipe.name))

files.append(obj_dir)

shprint(sh.cp, '-r', *files)

env = arch.get_env()

env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(

join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))

if not len(glob.glob(join(obj_dir, '*'))):

info('There seem to be no libraries to biglink, skipping.')

return

info('Biglinking')

info('target {}'.format(join(ctx.get_libs_dir(arch.arch),

'libpymodules.so')))

do_biglink = copylibs_function if ctx.copy_libs else biglink_function

# Move to the directory containing crtstart_so.o and crtend_so.o

# This is necessary with newer NDKs? A gcc bug?

with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):

do_biglink(

join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),

obj_dir.split(' '),

extra_link_dirs=[join(ctx.bootstrap.build_dir,

'obj', 'local', arch.arch),

os.path.abspath('.')],

env=env)

def biglink_function(soname, objs_paths, extra_link_dirs=[], env=None):

print('objs_paths are', objs_paths)

sofiles = []

for directory in objs_paths:

for fn in os.listdir(directory):

fn = os.path.join(directory, fn)

if not fn.endswith(".so.o"):

continue

if not os.path.exists(fn[:-2] + ".libs"):

continue

sofiles.append(fn[:-2])

# The raw argument list.

args = []

for fn in sofiles:

afn = fn + ".o"

libsfn = fn + ".libs"

args.append(afn)

with open(libsfn) as fd:

data = fd.read()

args.extend(data.split(" "))

unique_args = []

while args:

a = args.pop()

if a in ('-L', ):

continue

if a not in unique_args:

unique_args.insert(0, a)

for dir in extra_link_dirs:

link = '-L{}'.format(dir)

if link not in unique_args:

unique_args.append(link)

cc_name = env['CC']

cc = sh.Command(cc_name.split()[0])

cc = cc.bake(*cc_name.split()[1:])

shprint(cc, '-shared', '-O3', '-o', soname, *unique_args, _env=env)

def copylibs_function(soname, objs_paths, extra_link_dirs=[], env=None):

print('objs_paths are', objs_paths)

re_needso = re.compile(r'^.*$NEEDED$\s+Shared library: \[lib(.*)\.so\]\s*$')

blacklist_libs = (

'c',

'stdc++',

'dl',

'python2.7',

'sdl',

'sdl_image',

'sdl_ttf',

'z',

'm',

'GLESv2',

'jpeg',

'png',

'log',

# bootstrap takes care of sdl2 libs (if applicable)

'SDL2',

'SDL2_ttf',

'SDL2_image',

'SDL2_mixer',

)

found_libs = []

sofiles = []

if env and 'READELF' in env:

readelf = env['READELF']

elif 'READELF' in os.environ:

readelf = os.environ['READELF']

else:

readelf = sh.which('readelf').strip()

readelf = sh.Command(readelf).bake('-d')

dest = dirname(soname)

for directory in objs_paths:

for fn in os.listdir(directory):

fn = join(directory, fn)

if not fn.endswith('.libs'):

continue

dirfn = fn[:-1] + 'dirs'

if not exists(dirfn):

continue

with open(fn) as f:

libs = f.read().strip().split(' ')

needed_libs = [lib for lib in libs

if lib and

lib not in blacklist_libs and

lib not in found_libs]

while needed_libs:

print('need libs:\n\t' + '\n\t'.join(needed_libs))

start_needed_libs = needed_libs[:]

found_sofiles = []

with open(dirfn) as f:

libdirs = f.read().split()

for libdir in libdirs:

if not needed_libs:

break

if libdir == dest:

# don't need to copy from dest to dest!

continue

libdir = libdir.strip()

print('scanning', libdir)

for lib in needed_libs[:]:

if lib in found_libs:

continue

if lib.endswith('.a'):

needed_libs.remove(lib)

found_libs.append(lib)

continue

lib_a = 'lib' + lib + '.a'

libpath_a = join(libdir, lib_a)

lib_so = 'lib' + lib + '.so'

libpath_so = join(libdir, lib_so)

plain_so = lib + '.so'

plainpath_so = join(libdir, plain_so)

sopath = None

if exists(libpath_so):

sopath = libpath_so

elif exists(plainpath_so):

sopath = plainpath_so

View remainder of file in raw view

python-for-android/pythonforandroid/build.py at master · matrix-io/python-for-android · GitHub

Uh oh!

Expand file tree

Breadcrumbs

build.py

Latest commit

History

Breadcrumbs

build.py

File metadata and controls