cpython/Lib/shelve.py at main · python/cpython · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

"""Manage shelves of pickled objects.

A "shelf" is a persistent, dictionary-like object. The difference

with dbm databases is that the values (not the keys!) in a shelf can

be essentially arbitrary Python objects -- anything that the "pickle"

module can handle. This includes most class instances, recursive data

types, and objects containing lots of shared sub-objects. The keys

are ordinary strings.

To summarize the interface (key is a string, data is an arbitrary

object):

import shelve

d = shelve.open(filename) # open, with (g)dbm filename -- no suffix

d[key] = data # store data at key (overwrites old data if

# using an existing key)

data = d[key] # retrieve a COPY of the data at key (raise

# KeyError if no such key) -- NOTE that this

# access returns a *copy* of the entry!

del d[key] # delete data stored at key (raises KeyError

# if no such key)

flag = key in d # true if the key exists

list = d.keys() # a list of all existing keys (slow!)

d.close() # close it

Dependent on the implementation, closing a persistent dictionary may

or may not be necessary to flush changes to disk.

Normally, d[key] returns a COPY of the entry. This needs care when

mutable entries are mutated: for example, if d[key] is a list,

d[key].append(anitem)

does NOT modify the entry d[key] itself, as stored in the persistent

mapping -- it only modifies the copy, which is then immediately

discarded, so that the append has NO effect whatsoever. To append an

item to d[key] in a way that will affect the persistent mapping, use:

data = d[key]

data.append(anitem)

d[key] = data

To avoid the problem with mutable entries, you may pass the keyword

argument writeback=True in the call to shelve.open. When you use:

d = shelve.open(filename, writeback=True)

then d keeps a cache of all entries you access, and writes them all back

to the persistent mapping when you call d.close(). This ensures that

such usage as d[key].append(anitem) works as intended.

However, using keyword argument writeback=True may consume vast amount

of memory for the cache, and it may make d.close() very slow, if you

access many of d's entries after opening it in this way: d has no way to

check which of the entries you access are mutable and/or which ones you

actually mutate, so it must cache, and write back at close, all of the

entries that you access. You can call d.sync() to write back all the

entries in the cache, and empty the cache (d.sync() also synchronizes

the persistent dictionary on disk, if feasible).

"""

from pickle import DEFAULT_PROTOCOL, dumps, loads

import collections.abc

__all__ = ["ShelveError", "Shelf", "BsdDbShelf", "DbfilenameShelf", "open"]

class ShelveError(Exception):

pass

class _ClosedDict(collections.abc.MutableMapping):

'Marker for a closed dict. Access attempts raise a ValueError.'

def closed(self, *args):

raise ValueError('invalid operation on closed shelf')

__iter__ = __len__ = __getitem__ = __setitem__ = __delitem__ = keys = closed

def __repr__(self):

return '<Closed Dictionary>'

class Shelf(collections.abc.MutableMapping):

"""Base class for shelf implementations.

This is initialized with a dictionary-like object.

See the module's __doc__ string for an overview of the interface.

"""

def __init__(self, dict, protocol=None, writeback=False,

keyencoding="utf-8", *, serializer=None, deserializer=None):

self.dict = dict

if protocol is None:

protocol = DEFAULT_PROTOCOL

self._protocol = protocol

self.writeback = writeback

self.cache = {}

self.keyencoding = keyencoding

if serializer is None and deserializer is None:

self.serializer = dumps

self.deserializer = loads

elif (serializer is None) ^ (deserializer is None):

raise ShelveError("serializer and deserializer must be "

"defined together")

else:

self.serializer = serializer

self.deserializer = deserializer

def __iter__(self):

for k in self.dict.keys():

yield k.decode(self.keyencoding)

def __len__(self):

return len(self.dict)

def __contains__(self, key):

return key.encode(self.keyencoding) in self.dict

def get(self, key, default=None):

if key.encode(self.keyencoding) in self.dict:

return self[key]

return default

def __getitem__(self, key):

try:

value = self.cache[key]

except KeyError:

f = self.dict[key.encode(self.keyencoding)]

value = self.deserializer(f)

if self.writeback:

self.cache[key] = value

return value

def __setitem__(self, key, value):

if self.writeback:

self.cache[key] = value

serialized_value = self.serializer(value, self._protocol)

self.dict[key.encode(self.keyencoding)] = serialized_value

def __delitem__(self, key):

del self.dict[key.encode(self.keyencoding)]

try:

del self.cache[key]

except KeyError:

pass

def __enter__(self):

return self

def __exit__(self, type, value, traceback):

self.close()

def close(self):

if self.dict is None:

return

try:

self.sync()

try:

self.dict.close()

except AttributeError:

pass

finally:

# Catch errors that may happen when close is called from __del__

# because CPython is in interpreter shutdown.

try:

self.dict = _ClosedDict()

except:

self.dict = None

def __del__(self):

if not hasattr(self, 'writeback'):

# __init__ didn't succeed, so don't bother closing

# see http://bugs.python.org/issue1339007 for details

return

self.close()

def sync(self):

if self.writeback and self.cache:

self.writeback = False

for key, entry in self.cache.items():

self[key] = entry

self.writeback = True

self.cache = {}

if hasattr(self.dict, 'sync'):

self.dict.sync()

def reorganize(self):

self.sync()

if hasattr(self.dict, 'reorganize'):

self.dict.reorganize()

class BsdDbShelf(Shelf):

"""Shelf implementation using the "BSD" db interface.

This adds methods first(), next(), previous(), last() and

set_location() that have no counterpart in [g]dbm databases.

The actual database must be opened using one of the "bsddb"

modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or

bsddb.rnopen) and passed to the constructor.

See the module's __doc__ string for an overview of the interface.

"""

def __init__(self, dict, protocol=None, writeback=False,

keyencoding="utf-8", *, serializer=None, deserializer=None):

Shelf.__init__(self, dict, protocol, writeback, keyencoding,

serializer=serializer, deserializer=deserializer)

def set_location(self, key):

(key, value) = self.dict.set_location(key)