| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent c9dbf20 commit 06590ae
5 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,6 +1,9 @@ | |||
| 1 | 1 | ======= | |
| 2 | 2 | CHANGES | |
| 3 | 3 | ======= | |
| 4 | + 0.3 | ||
| 5 | + === | ||
| 6 | + * ConcurrentWriteOperation was removed, and replaced by LockedFD | ||
| 4 | 7 | ||
| 5 | 8 | 0.2 Beta 2 | |
| 6 | 9 | =========== | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -43,7 +43,7 @@ | |||
| 43 | 43 | from git.utils import ( | |
| 44 | 44 | IndexFileSHA1Writer, | |
| 45 | 45 | LazyMixin, | |
| 46 | - ConcurrentWriteOperation, | ||
| 46 | + LockedFD, | ||
| 47 | 47 | join_path_native | |
| 48 | 48 | ) | |
| 49 | 49 | ||
@@ -89,31 +89,27 @@ def _set_cache_(self, attr): | |||
| 89 | 89 | if attr == "entries": | |
| 90 | 90 | # read the current index | |
| 91 | 91 | # try memory map for speed | |
| 92 | + lfd = LockedFD(self._file_path) | ||
| 92 | 93 | try: | |
| 93 | - fp = open(self._file_path, "rb") | ||
| 94 | - except IOError: | ||
| 94 | + stream = lfd.open(write=False, stream=True) | ||
| 95 | + except OSError: | ||
| 96 | + lfd.rollback() | ||
| 95 | 97 | # in new repositories, there may be no index, which means we are empty | |
| 96 | 98 | self.entries = dict() | |
| 97 | 99 | return | |
| 98 | 100 | # END exception handling | |
| 99 | 101 | ||
| 100 | - stream = fp | ||
| 101 | 102 | try: | |
| 102 | - raise Exception() | ||
| 103 | - stream = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) | ||
| 103 | + stream = mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) | ||
| 104 | 104 | except Exception: | |
| 105 | 105 | pass | |
| 106 | 106 | # END memory mapping | |
| 107 | 107 | ||
| 108 | 108 | try: | |
| 109 | 109 | self._deserialize(stream) | |
| 110 | 110 | finally: | |
| 111 | - pass | ||
| 112 | - # make sure we close the stream ( possibly an mmap ) | ||
| 113 | - # and the file | ||
| 114 | - #stream.close() | ||
| 115 | - #if stream is not fp: | ||
| 116 | - # fp.close() | ||
| 111 | + lfd.rollback() | ||
| 112 | + # The handles will be closed on desctruction | ||
| 117 | 113 | # END read from default index on demand | |
| 118 | 114 | else: | |
| 119 | 115 | super(IndexFile, self)._set_cache_(attr) | |
@@ -267,12 +263,12 @@ def write(self, file_path = None, ignore_tree_extension_data=False): | |||
| 267 | 263 | Note | |
| 268 | 264 | Index writing based on the dulwich implementation | |
| 269 | 265 | """ | |
| 270 | - write_op = ConcurrentWriteOperation(file_path or self._file_path) | ||
| 271 | - stream = write_op._begin_writing() | ||
| 266 | + lfd = LockedFD(file_path or self._file_path) | ||
| 267 | + stream = lfd.open(write=True, stream=True) | ||
| 272 | 268 | ||
| 273 | 269 | self._serialize(stream, ignore_tree_extension_data) | |
| 274 | 270 | ||
| 275 | - write_op._end_writing() | ||
| 271 | + lfd.commit() | ||
| 276 | 272 | ||
| 277 | 273 | # make sure we represent what we have written | |
| 278 | 274 | if file_path is not None: | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -147,8 +147,7 @@ class LockFile(object): | |||
| 147 | 147 | ||
| 148 | 148 | As we are a utility class to be derived from, we only use protected methods. | |
| 149 | 149 | ||
| 150 | - Locks will automatically be released on destruction | ||
| 151 | - """ | ||
| 150 | + Locks will automatically be released on destruction """ | ||
| 152 | 151 | __slots__ = ("_file_path", "_owns_lock") | |
| 153 | 152 | ||
| 154 | 153 | def __init__(self, file_path): | |
@@ -216,8 +215,10 @@ def _release_lock(self): | |||
| 216 | 215 | # if someone removed our file beforhand, lets just flag this issue | |
| 217 | 216 | # instead of failing, to make it more usable. | |
| 218 | 217 | lfp = self._lock_file_path() | |
| 219 | - if os.path.isfile(lfp): | ||
| 218 | + try: | ||
| 220 | 219 | os.remove(lfp) | |
| 220 | + except OSError: | ||
| 221 | + pass | ||
| 221 | 222 | self._owns_lock = False | |
| 222 | 223 | ||
| 223 | 224 | ||
@@ -271,86 +272,144 @@ def _obtain_lock(self): | |||
| 271 | 272 | # END endless loop | |
| 272 | 273 | ||
| 273 | 274 | ||
| 274 | - class ConcurrentWriteOperation(LockFile): | ||
| 275 | - """ | ||
| 276 | - This class facilitates a safe write operation to a file on disk such that we: | ||
| 275 | + class FDStreamWrapper(object): | ||
| 276 | + """A simple wrapper providing the most basic functions on a file descriptor | ||
| 277 | + with the fileobject interface. Cannot use os.fdopen as the resulting stream | ||
| 278 | + takes ownership""" | ||
| 279 | + __slots__ = ("_fd", '_pos') | ||
| 280 | + def __init__(self, fd): | ||
| 281 | + self._fd = fd | ||
| 282 | + self._pos = 0 | ||
| 283 | + | ||
| 284 | + def write(self, data): | ||
| 285 | + self._pos += len(data) | ||
| 286 | + os.write(self._fd, data) | ||
| 277 | 287 | ||
| 278 | - - lock the original file | ||
| 279 | - - write to a temporary file | ||
| 280 | - - rename temporary file back to the original one on close | ||
| 281 | - - unlock the original file | ||
| 288 | + def read(self, count=0): | ||
| 289 | + if count == 0: | ||
| 290 | + count = os.path.getsize(self._filepath) | ||
| 291 | + # END handle read everything | ||
| 292 | + | ||
| 293 | + bytes = os.read(self._fd, count) | ||
| 294 | + self._pos += len(bytes) | ||
| 295 | + return bytes | ||
| 282 | 296 | ||
| 297 | + def fileno(self): | ||
| 298 | + return self._fd | ||
| 299 | + | ||
| 300 | + def tell(self): | ||
| 301 | + return self._pos | ||
| 302 | + | ||
| 303 | + | ||
| 304 | + class LockedFD(LockFile): | ||
| 305 | + """This class facilitates a safe read and write operation to a file on disk. | ||
| 306 | + If we write to 'file', we obtain a lock file at 'file.lock' and write to | ||
| 307 | + that instead. If we succeed, the lock file will be renamed to overwrite | ||
| 308 | + the original file. | ||
| 309 | + | ||
| 310 | + When reading, we obtain a lock file, but to prevent other writers from | ||
| 311 | + succeeding while we are reading the file. | ||
| 312 | + | ||
| 283 | 313 | This type handles error correctly in that it will assure a consistent state | |
| 284 | - on destruction | ||
| 285 | - """ | ||
| 286 | - __slots__ = "_temp_write_fp" | ||
| 314 | + on destruction. | ||
| 287 | 315 | ||
| 288 | - def __init__(self, file_path): | ||
| 289 | - """ | ||
| 290 | - Initialize an instance able to write the given file_path | ||
| 291 | - """ | ||
| 292 | - super(ConcurrentWriteOperation, self).__init__(file_path) | ||
| 293 | - self._temp_write_fp = None | ||
| 316 | + :note: with this setup, parallel reading is not possible""" | ||
| 317 | + __slots__ = ("_filepath", '_fd', '_write') | ||
| 318 | + | ||
| 319 | + def __init__(self, filepath): | ||
| 320 | + """Initialize an instance with the givne filepath""" | ||
| 321 | + self._filepath = filepath | ||
| 322 | + self._fd = None | ||
| 323 | + self._write = None # if True, we write a file | ||
| 294 | 324 | ||
| 295 | 325 | def __del__(self): | |
| 296 | - self._end_writing(successful=False) | ||
| 326 | + # will do nothing if the file descriptor is already closed | ||
| 327 | + if self._fd is not None: | ||
| 328 | + self.rollback() | ||
| 297 | 329 | ||
| 298 | - def _begin_writing(self): | ||
| 299 | - """ | ||
| 300 | - Begin writing our file, hence we get a lock and start writing | ||
| 301 | - a temporary file in the same directory. | ||
| 330 | + def _lockfilepath(self): | ||
| 331 | + return "%s.lock" % self._filepath | ||
| 302 | 332 | ||
| 303 | - Returns | ||
| 304 | - File Object to write to. It is still maintained by this instance | ||
| 305 | - and you do not need to manually close | ||
| 306 | - """ | ||
| 307 | - # already writing ? | ||
| 308 | - if self._temp_write_fp is not None: | ||
| 309 | - return self._temp_write_fp | ||
| 310 | - | ||
| 311 | - self._obtain_lock_or_raise() | ||
| 312 | - dirname, basename = os.path.split(self._file_path) | ||
| 313 | - self._temp_write_fp = open(tempfile.mktemp(basename, '', dirname), "wb") | ||
| 314 | - return self._temp_write_fp | ||
| 333 | + def open(self, write=False, stream=False): | ||
| 334 | + """Open the file descriptor for reading or writing, both in binary mode. | ||
| 335 | + :param write: if True, the file descriptor will be opened for writing. Other | ||
| 336 | + wise it will be opened read-only. | ||
| 337 | + :param stream: if True, the file descriptor will be wrapped into a simple stream | ||
| 338 | + object which supports only reading or writing | ||
| 339 | + :return: fd to read from or write to. It is still maintained by this instance | ||
| 340 | + and must not be closed directly | ||
| 341 | + :raise IOError: if the lock could not be retrieved | ||
| 342 | + :raise OSError: If the actual file could not be opened for reading | ||
| 343 | + :note: must only be called once""" | ||
| 344 | + if self._write is not None: | ||
| 345 | + raise AssertionError("Called %s multiple times" % self.open) | ||
| 346 | + | ||
| 347 | + self._write = write | ||
| 348 | + | ||
| 349 | + # try to open the lock file | ||
| 350 | + binary = getattr(os, 'O_BINARY', 0) | ||
| 351 | + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary | ||
| 352 | + try: | ||
| 353 | + fd = os.open(self._lockfilepath(), lockmode) | ||
| 354 | + if not write: | ||
| 355 | + os.close(fd) | ||
| 356 | + else: | ||
| 357 | + self._fd = fd | ||
| 358 | + # END handle file descriptor | ||
| 359 | + except OSError: | ||
| 360 | + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) | ||
| 361 | + # END handle lock retrieval | ||
| 362 | + | ||
| 363 | + # open actual file if required | ||
| 364 | + if self._fd is None: | ||
| 365 | + # we could specify exlusive here, as we obtained the lock anyway | ||
| 366 | + self._fd = os.open(self._filepath, os.O_RDONLY | binary) | ||
| 367 | + # END open descriptor for reading | ||
| 368 | + | ||
| 369 | + if stream: | ||
| 370 | + return FDStreamWrapper(self._fd) | ||
| 371 | + else: | ||
| 372 | + return self._fd | ||
| 373 | + # END handle stream | ||
| 374 | + | ||
| 375 | + def commit(self): | ||
| 376 | + """When done writing, call this function to commit your changes into the | ||
| 377 | + actual file. | ||
| 378 | + The file descriptor will be closed, and the lockfile handled. | ||
| 379 | + :note: can be called multiple times""" | ||
| 380 | + self._end_writing(successful=True) | ||
| 381 | + | ||
| 382 | + def rollback(self): | ||
| 383 | + """Abort your operation without any changes. The file descriptor will be | ||
| 384 | + closed, and the lock released. | ||
| 385 | + :note: can be called multiple times""" | ||
| 386 | + self._end_writing(successful=False) | ||
| 315 | 387 | ||
| 316 | - def _is_writing(self): | ||
| 317 | - """ | ||
| 318 | - Returns | ||
| 319 | - True if we are currently writing a file | ||
| 320 | - """ | ||
| 321 | - return self._temp_write_fp is not None | ||
| 322 | - | ||
| 323 | 388 | def _end_writing(self, successful=True): | |
| 324 | - """ | ||
| 325 | - Indicate you successfully finished writing the file to: | ||
| 389 | + """Handle the lock according to the write mode """ | ||
| 390 | + if self._write is None: | ||
| 391 | + raise AssertionError("Cannot end operation if it wasn't started yet") | ||
| 326 | 392 | ||
| 327 | - - close the underlying stream | ||
| 328 | - - rename the remporary file to the original one | ||
| 329 | - - release our lock | ||
| 330 | - """ | ||
| 331 | - # did we start a write operation ? | ||
| 332 | - if self._temp_write_fp is None: | ||
| 333 | - return | ||
| 334 | - | ||
| 335 | - if not self._temp_write_fp.closed: | ||
| 336 | - self._temp_write_fp.close() | ||
| 393 | + if self._fd is None: | ||
| 394 | + return | ||
| 337 | 395 | ||
| 338 | - if successful: | ||
| 396 | + os.close(self._fd) | ||
| 397 | + self._fd = None | ||
| 398 | + | ||
| 399 | + lockfile = self._lockfilepath() | ||
| 400 | + if self._write and successful: | ||
| 339 | 401 | # on windows, rename does not silently overwrite the existing one | |
| 340 | 402 | if sys.platform == "win32": | |
| 341 | - if os.path.isfile(self._file_path): | ||
| 342 | - os.remove(self._file_path) | ||
| 403 | + if os.path.isfile(self._filepath): | ||
| 404 | + os.remove(self._filepath) | ||
| 343 | 405 | # END remove if exists | |
| 344 | 406 | # END win32 special handling | |
| 345 | - os.rename(self._temp_write_fp.name, self._file_path) | ||
| 407 | + os.rename(lockfile, self._filepath) | ||
| 346 | 408 | else: | |
| 347 | 409 | # just delete the file so far, we failed | |
| 348 | - os.remove(self._temp_write_fp.name) | ||
| 410 | + os.remove(lockfile) | ||
| 349 | 411 | # END successful handling | |
| 350 | 412 | ||
| 351 | - # finally reset our handle | ||
| 352 | - self._release_lock() | ||
| 353 | - self._temp_write_fp = None | ||
| 354 | 413 | ||
| 355 | 414 | ||
| 356 | 415 | class LazyMixin(object): | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -86,31 +86,55 @@ def test_safe_operation(self): | |||
| 86 | 86 | my_file_fp.close() | |
| 87 | 87 | ||
| 88 | 88 | try: | |
| 89 | - cwrite = ConcurrentWriteOperation(my_file) | ||
| 89 | + lfd = LockedFD(my_file) | ||
| 90 | + lockfilepath = lfd._lockfilepath() | ||
| 90 | 91 | ||
| 91 | - # didn't start writing, doesnt matter | ||
| 92 | - cwrite._end_writing(False) | ||
| 93 | - cwrite._end_writing(True) | ||
| 94 | - assert not cwrite._is_writing() | ||
| 92 | + # cannot end before it was started | ||
| 93 | + self.failUnlessRaises(AssertionError, lfd.rollback) | ||
| 94 | + self.failUnlessRaises(AssertionError, lfd.commit) | ||
| 95 | + | ||
| 96 | + # open for writing | ||
| 97 | + assert not os.path.isfile(lockfilepath) | ||
| 98 | + wfd = lfd.open(write=True) | ||
| 99 | + assert lfd._fd is wfd | ||
| 100 | + assert os.path.isfile(lockfilepath) | ||
| 95 | 101 | ||
| 96 | 102 | # write data and fail | |
| 97 | - stream = cwrite._begin_writing() | ||
| 98 | - assert cwrite._is_writing() | ||
| 99 | - stream.write(new_data) | ||
| 100 | - cwrite._end_writing(successful=False) | ||
| 103 | + os.write(wfd, new_data) | ||
| 104 | + lfd.rollback() | ||
| 105 | + assert lfd._fd is None | ||
| 101 | 106 | self._cmp_contents(my_file, orig_data) | |
| 102 | - assert not os.path.exists(stream.name) | ||
| 107 | + assert not os.path.isfile(lockfilepath) | ||
| 108 | + | ||
| 109 | + # additional call doesnt fail | ||
| 110 | + lfd.commit() | ||
| 111 | + lfd.rollback() | ||
| 112 | + | ||
| 113 | + # test reading | ||
| 114 | + lfd = LockedFD(my_file) | ||
| 115 | + rfd = lfd.open(write=False) | ||
| 116 | + assert os.read(rfd, len(orig_data)) == orig_data | ||
| 117 | + | ||
| 118 | + assert os.path.isfile(lockfilepath) | ||
| 119 | + # deletion rolls back | ||
| 120 | + del(lfd) | ||
| 121 | + assert not os.path.isfile(lockfilepath) | ||
| 122 | + | ||
| 103 | 123 | ||
| 104 | 124 | # write data - concurrently | |
| 105 | - ocwrite = ConcurrentWriteOperation(my_file) | ||
| 106 | - stream = cwrite._begin_writing() | ||
| 107 | - self.failUnlessRaises(IOError, ocwrite._begin_writing) | ||
| 125 | + lfd = LockedFD(my_file) | ||
| 126 | + olfd = LockedFD(my_file) | ||
| 127 | + assert not os.path.isfile(lockfilepath) | ||
| 128 | + wfdstream = lfd.open(write=True, stream=True) # this time as stream | ||
| 129 | + assert os.path.isfile(lockfilepath) | ||
| 130 | + # another one fails | ||
| 131 | + self.failUnlessRaises(IOError, olfd.open) | ||
| 108 | 132 | ||
| 109 | - stream.write("world") | ||
| 110 | - cwrite._end_writing(successful=True) | ||
| 133 | + wfdstream.write(new_data) | ||
| 134 | + lfd.commit() | ||
| 135 | + assert not os.path.isfile(lockfilepath) | ||
| 111 | 136 | self._cmp_contents(my_file, new_data) | |
| 112 | - assert not os.path.exists(stream.name) | ||
| 113 | - | ||
| 137 | + | ||
| 114 | 138 | # could test automatic _end_writing on destruction | |
| 115 | 139 | finally: | |
| 116 | 140 | os.remove(my_file) | |
| Back | FazBrowse Home | New Git URL |
0 commit comments