cpython/Lib/pkgutil.py at 3.5 · python/cpython · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623

624

625

626

627

628

629

630

631

632

"""Utilities to support packages."""

from functools import singledispatch as simplegeneric

import importlib

import importlib.util

import importlib.machinery

import os

import os.path

import sys

from types import ModuleType

import warnings

__all__ = [

'get_importer', 'iter_importers', 'get_loader', 'find_loader',

'walk_packages', 'iter_modules', 'get_data',

'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',

]

def _get_spec(finder, name):

"""Return the finder-specific module spec."""

# Works with legacy finders.

try:

find_spec = finder.find_spec

except AttributeError:

loader = finder.find_module(name)

if loader is None:

return None

return importlib.util.spec_from_loader(name, loader)

else:

return find_spec(name)

def read_code(stream):

# This helper is needed in order for the PEP 302 emulation to

# correctly handle compiled files

import marshal

magic = stream.read(4)

if (magic != importlib.util.MAGIC_NUMBER

# Issue #29537: handle issue27286 bytecode incompatibility

# See Lib/importlib/_bootstrap_external.py

and magic != importlib.util._BACKCOMPAT_MAGIC_NUMBER):

return None

stream.read(8) # Skip timestamp and size

return marshal.load(stream)

def walk_packages(path=None, prefix='', onerror=None):

"""Yields (module_finder, name, ispkg) for all modules recursively

on path, or, if path is None, all accessible modules.

'path' should be either None or a list of paths to look for

modules in.

'prefix' is a string to output on the front of every module name

on output.

Note that this function must import all *packages* (NOT all

modules!) on the given path, in order to access the __path__

attribute to find submodules.

'onerror' is a function which gets called with one argument (the

name of the package which was being imported) if any exception

occurs while trying to import a package. If no onerror function is

supplied, ImportErrors are caught and ignored, while all other

exceptions are propagated, terminating the search.

Examples:

# list all modules python can access

walk_packages()

# list all submodules of ctypes

walk_packages(ctypes.__path__, ctypes.__name__+'.')

"""

def seen(p, m={}):

if p in m:

return True

m[p] = True

for importer, name, ispkg in iter_modules(path, prefix):

yield importer, name, ispkg

if ispkg:

try:

__import__(name)

except ImportError:

if onerror is not None:

onerror(name)

except Exception:

if onerror is not None:

onerror(name)

else:

raise

else:

path = getattr(sys.modules[name], '__path__', None) or []

# don't traverse path items we've seen before

path = [p for p in path if not seen(p)]

yield from walk_packages(path, name+'.', onerror)

def iter_modules(path=None, prefix=''):

"""Yields (module_finder, name, ispkg) for all submodules on path,

or, if path is None, all top-level modules on sys.path.

'path' should be either None or a list of paths to look for

modules in.

'prefix' is a string to output on the front of every module name

on output.

"""

if path is None:

importers = iter_importers()

else:

importers = map(get_importer, path)

yielded = {}

for i in importers:

for name, ispkg in iter_importer_modules(i, prefix):

if name not in yielded:

yielded[name] = 1

yield i, name, ispkg

@simplegeneric

def iter_importer_modules(importer, prefix=''):

if not hasattr(importer, 'iter_modules'):

return []

return importer.iter_modules(prefix)

# Implement a file walker for the normal importlib path hook

def _iter_file_finder_modules(importer, prefix=''):

if importer.path is None or not os.path.isdir(importer.path):

return

yielded = {}

import inspect

try:

filenames = os.listdir(importer.path)

except OSError:

# ignore unreadable directories like import does

filenames = []

filenames.sort() # handle packages before same-named modules

for fn in filenames:

modname = inspect.getmodulename(fn)

if modname=='__init__' or modname in yielded:

continue

path = os.path.join(importer.path, fn)

ispkg = False

if not modname and os.path.isdir(path) and '.' not in fn:

modname = fn

try:

dircontents = os.listdir(path)

except OSError:

# ignore unreadable directories like import does

dircontents = []

for fn in dircontents:

subname = inspect.getmodulename(fn)

if subname=='__init__':

ispkg = True

break

else:

continue # not a package

if modname and '.' not in modname:

yielded[modname] = 1

yield prefix + modname, ispkg

iter_importer_modules.register(

importlib.machinery.FileFinder, _iter_file_finder_modules)

def _import_imp():

global imp

with warnings.catch_warnings():

warnings.simplefilter('ignore', PendingDeprecationWarning)

imp = importlib.import_module('imp')

class ImpImporter:

"""PEP 302 Finder that wraps Python's "classic" import algorithm

ImpImporter(dirname) produces a PEP 302 finder that searches that

directory. ImpImporter(None) produces a PEP 302 finder that searches

the current sys.path, plus any modules that are frozen or built-in.

Note that ImpImporter does not currently support being used by placement

on sys.meta_path.

"""

def __init__(self, path=None):

global imp

warnings.warn("This emulation is deprecated, use 'importlib' instead",

DeprecationWarning)

_import_imp()

self.path = path

def find_module(self, fullname, path=None):

# Note: we ignore 'path' argument since it is only used via meta_path

subname = fullname.split(".")[-1]

if subname != fullname and self.path is None:

return None

if self.path is None:

path = None

else:

path = [os.path.realpath(self.path)]

try:

file, filename, etc = imp.find_module(subname, path)

except ImportError:

return None

return ImpLoader(fullname, file, filename, etc)

def iter_modules(self, prefix=''):

if self.path is None or not os.path.isdir(self.path):

return

yielded = {}

import inspect

try:

filenames = os.listdir(self.path)

except OSError:

# ignore unreadable directories like import does

filenames = []

filenames.sort() # handle packages before same-named modules

for fn in filenames:

modname = inspect.getmodulename(fn)

if modname=='__init__' or modname in yielded:

continue

path = os.path.join(self.path, fn)

ispkg = False

if not modname and os.path.isdir(path) and '.' not in fn:

modname = fn

try:

dircontents = os.listdir(path)

except OSError:

# ignore unreadable directories like import does

dircontents = []

for fn in dircontents:

subname = inspect.getmodulename(fn)

if subname=='__init__':

ispkg = True

break

else:

continue # not a package

if modname and '.' not in modname:

yielded[modname] = 1

yield prefix + modname, ispkg

class ImpLoader:

"""PEP 302 Loader that wraps Python's "classic" import algorithm

"""

code = source = None

def __init__(self, fullname, file, filename, etc):

warnings.warn("This emulation is deprecated, use 'importlib' instead",

DeprecationWarning)

_import_imp()

self.file = file

self.filename = filename

self.fullname = fullname

self.etc = etc

def load_module(self, fullname):

self._reopen()

try:

mod = imp.load_module(fullname, self.file, self.filename, self.etc)

finally:

if self.file:

self.file.close()

# Note: we don't set __loader__ because we want the module to look

# normal; i.e. this is just a wrapper for standard import machinery

return mod

def get_data(self, pathname):

with open(pathname, "rb") as file:

return file.read()

def _reopen(self):

if self.file and self.file.closed:

mod_type = self.etc[2]

if mod_type==imp.PY_SOURCE:

self.file = open(self.filename, 'r')

elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):

self.file = open(self.filename, 'rb')

def _fix_name(self, fullname):

if fullname is None:

fullname = self.fullname

elif fullname != self.fullname:

raise ImportError("Loader for module %s cannot handle "

"module %s" % (self.fullname, fullname))

return fullname

def is_package(self, fullname):

fullname = self._fix_name(fullname)

return self.etc[2]==imp.PKG_DIRECTORY

def get_code(self, fullname=None):

fullname = self._fix_name(fullname)

if self.code is None:

mod_type = self.etc[2]

if mod_type==imp.PY_SOURCE:

source = self.get_source(fullname)

self.code = compile(source, self.filename, 'exec')

elif mod_type==imp.PY_COMPILED:

self._reopen()

try:

self.code = read_code(self.file)

finally:

self.file.close()

elif mod_type==imp.PKG_DIRECTORY:

self.code = self._get_delegate().get_code()

return self.code

def get_source(self, fullname=None):

fullname = self._fix_name(fullname)

if self.source is None:

mod_type = self.etc[2]

if mod_type==imp.PY_SOURCE:

self._reopen()

try:

self.source = self.file.read()

finally:

self.file.close()

elif mod_type==imp.PY_COMPILED:

if os.path.exists(self.filename[:-1]):

with open(self.filename[:-1], 'r') as f:

self.source = f.read()

elif mod_type==imp.PKG_DIRECTORY:

self.source = self._get_delegate().get_source()

return self.source

def _get_delegate(self):

finder = ImpImporter(self.filename)

spec = _get_spec(finder, '__init__')

return spec.loader

def get_filename(self, fullname=None):

fullname = self._fix_name(fullname)

mod_type = self.etc[2]

if mod_type==imp.PKG_DIRECTORY:

return self._get_delegate().get_filename()

elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):

return self.filename

return None

try:

import zipimport

from zipimport import zipimporter

def iter_zipimport_modules(importer, prefix=''):

dirlist = sorted(zipimport._zip_directory_cache[importer.archive])

_prefix = importer.prefix

plen = len(_prefix)

yielded = {}

import inspect

for fn in dirlist:

if not fn.startswith(_prefix):

continue

fn = fn[plen:].split(os.sep)

if len(fn)==2 and fn[1].startswith('__init__.py'):

if fn[0] not in yielded:

yielded[fn[0]] = 1

yield prefix + fn[0], True

if len(fn)!=1:

continue

modname = inspect.getmodulename(fn[0])

if modname=='__init__':

continue

if modname and '.' not in modname and modname not in yielded:

yielded[modname] = 1

yield prefix + modname, False

iter_importer_modules.register(zipimporter, iter_zipimport_modules)

except ImportError:

pass

def get_importer(path_item):

"""Retrieve a finder for the given path item

The returned finder is cached in sys.path_importer_cache

if it was newly created by a path hook.

The cache (or part of it) can be cleared manually if a

rescan of sys.path_hooks is necessary.

"""

try:

importer = sys.path_importer_cache[path_item]

except KeyError:

for path_hook in sys.path_hooks:

try:

importer = path_hook(path_item)

sys.path_importer_cache.setdefault(path_item, importer)

break

except ImportError:

pass

else:

importer = None

return importer

def iter_importers(fullname=""):

"""Yield finders for the given module name

If fullname contains a '.', the finders will be for the package

containing fullname, otherwise they will be all registered top level

finders (i.e. those on both sys.meta_path and sys.path_hooks).

If the named module is in a package, that package is imported as a side

effect of invoking this function.

If no module name is specified, all top level finders are produced.

"""

if fullname.startswith('.'):

msg = "Relative module name {!r} not supported".format(fullname)

raise ImportError(msg)

if '.' in fullname:

# Get the containing package's __path__

pkg_name = fullname.rpartition(".")[0]

pkg = importlib.import_module(pkg_name)

path = getattr(pkg, '__path__', None)

if path is None:

return

else:

yield from sys.meta_path

path = sys.path

for item in path:

yield get_importer(item)

def get_loader(module_or_name):

"""Get a "loader" object for module_or_name

Returns None if the module cannot be found or imported.

If the named module is not already imported, its containing package

(if any) is imported, in order to establish the package __path__.

"""

if module_or_name in sys.modules:

module_or_name = sys.modules[module_or_name]

if module_or_name is None:

return None

if isinstance(module_or_name, ModuleType):

module = module_or_name

loader = getattr(module, '__loader__', None)

if loader is not None:

return loader

if getattr(module, '__spec__', None) is None:

return None

fullname = module.__name__

else:

fullname = module_or_name

return find_loader(fullname)

def find_loader(fullname):

"""Find a "loader" object for fullname

This is a backwards compatibility wrapper around

importlib.util.find_spec that converts most failures to ImportError

and only returns the loader rather than the full spec

"""

if fullname.startswith('.'):

msg = "Relative module name {!r} not supported".format(fullname)

raise ImportError(msg)

try:

spec = importlib.util.find_spec(fullname)

except (ImportError, AttributeError, TypeError, ValueError) as ex:

# This hack fixes an impedance mismatch between pkgutil and

# importlib, where the latter raises other errors for cases where

# pkgutil previously raised ImportError

msg = "Error while finding loader for {!r} ({}: {})"

raise ImportError(msg.format(fullname, type(ex), ex)) from ex

return spec.loader if spec is not None else None

def extend_path(path, name):

"""Extend a package's path.

Intended use is to place the following code in a package's __init__.py:

from pkgutil import extend_path

__path__ = extend_path(__path__, __name__)

This will add to the package's __path__ all subdirectories of

directories on sys.path named after the package. This is useful

if one wants to distribute different parts of a single logical

package as multiple directories.

It also looks for *.pkg files beginning where * matches the name

argument. This feature is similar to *.pth files (see site.py),

except that it doesn't special-case lines starting with 'import'.

A *.pkg file is trusted at face value: apart from checking for

duplicates, all entries found in a *.pkg file are added to the

path, regardless of whether they are exist the filesystem. (This

is a feature.)

If the input path is not a list (as is the case for frozen

packages) it is returned unchanged. The input path is not

modified; an extended copy is returned. Items are only appended

to the copy at the end.

It is assumed that sys.path is a sequence. Items of sys.path that

are not (unicode or 8-bit) strings referring to existing

directories are ignored. Unicode items of sys.path that cause

errors when used as filenames may cause this function to raise an

exception (in line with os.path.isdir() behavior).

"""

if not isinstance(path, list):

# This could happen e.g. when this is called from inside a

# frozen package. Return the path unchanged in that case.

return path

sname_pkg = name + ".pkg"

path = path[:] # Start with a copy of the existing path

parent_package, _, final_name = name.rpartition('.')

if parent_package:

try:

search_path = sys.modules[parent_package].__path__

except (KeyError, AttributeError):

# We can't do anything: find_loader() returns None when

# passed a dotted name.

return path

else:

search_path = sys.path

for dir in search_path:

if not isinstance(dir, str):

continue

finder = get_importer(dir)

if finder is not None:

portions = []

if hasattr(finder, 'find_spec'):

spec = finder.find_spec(final_name)

if spec is not None:

portions = spec.submodule_search_locations or []

# Is this finder PEP 420 compliant?

elif hasattr(finder, 'find_loader'):

_, portions = finder.find_loader(final_name)

for portion in portions:

# XXX This may still add duplicate entries to path on

# case-insensitive filesystems

if portion not in path:

path.append(portion)

# XXX Is this the right thing for subpackages like zope.app?

# It looks for a file named "zope.app.pkg"

pkgfile = os.path.join(dir, sname_pkg)

if os.path.isfile(pkgfile):

try:

f = open(pkgfile)

except OSError as msg:

sys.stderr.write("Can't open %s: %s\n" %

(pkgfile, msg))

else:

with f:

for line in f:

line = line.rstrip('\n')

if not line or line.startswith('#'):

continue

path.append(line) # Don't check for existence!

return path

def get_data(package, resource):

"""Get a resource from a package.

This is a wrapper round the PEP 302 loader get_data API. The package

argument should be the name of a package, in standard module format

(foo.bar). The resource argument should be in the form of a relative

filename, using '/' as the path separator. The parent directory name '..'

is not allowed, and nor is a rooted name (starting with a '/').

The function returns a binary string, which is the contents of the

specified resource.

For packages located in the filesystem, which have already been imported,

this is the rough equivalent of

d = os.path.dirname(sys.modules[package].__file__)

data = open(os.path.join(d, resource), 'rb').read()

If the package cannot be located or loaded, or it uses a PEP 302 loader

which does not support get_data(), then None is returned.

"""

spec = importlib.util.find_spec(package)

if spec is None:

return None

loader = spec.loader

if loader is None or not hasattr(loader, 'get_data'):

return None

# XXX needs test

mod = (sys.modules.get(package) or

importlib._bootstrap._load(spec))

if mod is None or not hasattr(mod, '__file__'):

return None

# Modify the resource name to be compatible with the loader.get_data

# signature - an os.path format "filename" starting with the dirname of

# the package's __file__

parts = resource.split('/')

parts.insert(0, os.path.dirname(mod.__file__))

resource_name = os.path.join(*parts)

return loader.get_data(resource_name)

cpython/Lib/pkgutil.py at 3.5 · python/cpython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

pkgutil.py

Latest commit

History

Breadcrumbs

pkgutil.py

File metadata and controls