pythonVSCode/pythonFiles/jedi/cache.py at master · lzfernandes/pythonVSCode · GitHub

History

336 lines (273 loc) · 9.82 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

"""

This caching is very important for speed and memory optimizations. There's

nothing really spectacular, just some decorators. The following cache types are

available:

- module caching (`load_parser` and `save_parser`), which uses pickle and is

really important to assure low load times of modules like ``numpy``.

- ``time_cache`` can be used to cache something for just a limited time span,

which can be useful if there's user interaction and the user cannot react

faster than a certain time.

This module is one of the reasons why |jedi| is not thread-safe. As you can see

there are global variables, which are holding the cache information. Some of

these variables are being cleaned after every API usage.

"""

import time

import os

import sys

import json

import hashlib

import gc

import inspect

import shutil

import re

try:

import cPickle as pickle

except ImportError:

import pickle

from jedi import settings

from jedi import common

from jedi import debug

_time_caches = {}

# for fast_parser, should not be deleted

parser_cache = {}

class ParserCacheItem(object):

def __init__(self, parser, change_time=None):

self.parser = parser

if change_time is None:

change_time = time.time()

self.change_time = change_time

def clear_time_caches(delete_all=False):

""" Jedi caches many things, that should be completed after each completion

finishes.

:param delete_all: Deletes also the cache that is normally not deleted,

like parser cache, which is important for faster parsing.

"""

global _time_caches

if delete_all:

for cache in _time_caches.values():

cache.clear()

parser_cache.clear()

else:

# normally just kill the expired entries, not all

for tc in _time_caches.values():

# check time_cache for expired entries

for key, (t, value) in list(tc.items()):

if t < time.time():

# delete expired entries

del tc[key]

def time_cache(time_add_setting):

"""

This decorator works as follows: Call it with a setting and after that

use the function with a callable that returns the key.

But: This function is only called if the key is not available. After a

certain amount of time (`time_add_setting`) the cache is invalid.

"""

def _temp(key_func):

dct = {}

_time_caches[time_add_setting] = dct

def wrapper(*args, **kwargs):

generator = key_func(*args, **kwargs)

key = next(generator)

try:

expiry, value = dct[key]

if expiry > time.time():

return value

except KeyError:

pass

value = next(generator)

time_add = getattr(settings, time_add_setting)

if key is not None:

dct[key] = time.time() + time_add, value

return value

return wrapper

return _temp

@time_cache("call_signatures_validity")

def cache_call_signatures(evaluator, call, source, user_pos):

"""This function calculates the cache key."""

index = user_pos[0] - 1

lines = common.splitlines(source)

before_cursor = lines[index][:user_pos[1]]

other_lines = lines[call.start_pos[0]:index]

whole = '\n'.join(other_lines + [before_cursor])

before_bracket = re.match(r'.*\(', whole, re.DOTALL)

module_path = call.get_parent_until().path

yield None if module_path is None else (module_path, before_bracket, call.start_pos)

yield evaluator.eval_element(call)

def underscore_memoization(func):

"""

Decorator for methods::

class A(object):

def x(self):

if self._x:

self._x = 10

return self._x

Becomes::

class A(object):

@underscore_memoization

def x(self):

return 10

A now has an attribute ``_x`` written by this decorator.

"""

name = '_' + func.__name__

def wrapper(self):

try:

return getattr(self, name)

except AttributeError:

result = func(self)

if inspect.isgenerator(result):

result = list(result)

setattr(self, name, result)

return result

return wrapper

def memoize_method(method):

"""A normal memoize function."""

def wrapper(self, *args, **kwargs):

dct = self.__dict__.setdefault('_memoize_method_dct', {})

key = (args, frozenset(kwargs.items()))

try:

return dct[key]

except KeyError:

result = method(self, *args, **kwargs)

dct[key] = result

return result

return wrapper

def cache_star_import(func):

@time_cache("star_import_cache_validity")

def wrapper(self):

yield self.base # The cache key

yield func(self)

return wrapper

def _invalidate_star_import_cache_module(module, only_main=False):

""" Important if some new modules are being reparsed """

try:

t, modules = _time_caches['star_import_cache_validity'][module]

except KeyError:

pass

else:

del _time_caches['star_import_cache_validity'][module]

def invalidate_star_import_cache(path):

"""On success returns True."""

try:

parser_cache_item = parser_cache[path]

except KeyError:

pass

else:

_invalidate_star_import_cache_module(parser_cache_item.parser.module)

def load_parser(path):

"""

Returns the module or None, if it fails.

"""

p_time = os.path.getmtime(path) if path else None

try:

parser_cache_item = parser_cache[path]

if not path or p_time <= parser_cache_item.change_time:

return parser_cache_item.parser

else:

# In case there is already a module cached and this module

# has to be reparsed, we also need to invalidate the import

# caches.

_invalidate_star_import_cache_module(parser_cache_item.parser.module)

except KeyError:

if settings.use_filesystem_cache:

return ParserPickling.load_parser(path, p_time)

def save_parser(path, parser, pickling=True):

try:

p_time = None if path is None else os.path.getmtime(path)

except OSError:

p_time = None

pickling = False

item = ParserCacheItem(parser, p_time)

parser_cache[path] = item

if settings.use_filesystem_cache and pickling:

ParserPickling.save_parser(path, item)

class ParserPickling(object):

version = 24

"""

Version number (integer) for file system cache.

Increment this number when there are any incompatible changes in

parser representation classes. For example, the following changes

are regarded as incompatible.

- Class name is changed.

- Class is moved to another module.

- Defined slot of the class is changed.

"""

def __init__(self):

self.__index = None

self.py_tag = 'cpython-%s%s' % sys.version_info[:2]

"""

Short name for distinguish Python implementations and versions.

It's like `sys.implementation.cache_tag` but for Python < 3.3

we generate something similar. See:

http://docs.python.org/3/library/sys.html#sys.implementation

.. todo:: Detect interpreter (e.g., PyPy).

"""

def load_parser(self, path, original_changed_time):

try:

pickle_changed_time = self._index[path]

except KeyError:

return None

if original_changed_time is not None \

and pickle_changed_time < original_changed_time:

# the pickle file is outdated

return None

with open(self._get_hashed_path(path), 'rb') as f:

try:

gc.disable()

parser_cache_item = pickle.load(f)

finally:

gc.enable()

debug.dbg('pickle loaded: %s', path)

parser_cache[path] = parser_cache_item

return parser_cache_item.parser

def save_parser(self, path, parser_cache_item):

self.__index = None

try:

files = self._index

except KeyError:

files = {}

self._index = files

with open(self._get_hashed_path(path), 'wb') as f:

pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)

files[path] = parser_cache_item.change_time

self._flush_index()

@property

def _index(self):

if self.__index is None:

try:

with open(self._get_path('index.json')) as f:

data = json.load(f)

except (IOError, ValueError):

self.__index = {}

else:

# 0 means version is not defined (= always delete cache):

if data.get('version', 0) != self.version:

self.clear_cache()

self.__index = {}

else:

self.__index = data['index']

return self.__index

def _remove_old_modules(self):

# TODO use

change = False

if change:

self._flush_index(self)

self._index # reload index

def _flush_index(self):

data = {'version': self.version, 'index': self._index}

with open(self._get_path('index.json'), 'w') as f:

json.dump(data, f)

self.__index = None

def clear_cache(self):

shutil.rmtree(self._cache_directory())

def _get_hashed_path(self, path):

return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())

def _get_path(self, file):

dir = self._cache_directory()

if not os.path.exists(dir):

os.makedirs(dir)

return os.path.join(dir, file)

def _cache_directory(self):

return os.path.join(settings.cache_directory, self.py_tag)

# is a singleton

ParserPickling = ParserPickling()

pythonVSCode/pythonFiles/jedi/cache.py at master · lzfernandes/pythonVSCode · GitHub

Expand file tree

Breadcrumbs

cache.py

Latest commit

History

Breadcrumbs

cache.py

File metadata and controls