| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,6 +1,3 @@ | |||
| 1 | - [submodule "async"] | ||
| 2 | - path = gitdb/ext/async | ||
| 3 | - url = https://github.com/gitpython-developers/async.git | ||
| 4 | 1 | [submodule "smmap"] | |
| 5 | 2 | path = gitdb/ext/smmap | |
| 6 | 3 | url = https://github.com/Byron/smmap.git | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2,6 +2,13 @@ | |||
| 2 | 2 | Changelog | |
| 3 | 3 | ######### | |
| 4 | 4 | ||
| 5 | + ***** | ||
| 6 | + 0.6.0 | ||
| 7 | + ***** | ||
| 8 | + | ||
| 9 | + * Added support got python 3.X | ||
| 10 | + * Removed all `async` dependencies and all `*_async` versions of methods with it. | ||
| 11 | + | ||
| 5 | 12 | ***** | |
| 6 | 13 | 0.5.4 | |
| 7 | 14 | ***** | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -10,7 +10,7 @@ | |||
| 10 | 10 | #{ Initialization | |
| 11 | 11 | def _init_externals(): | |
| 12 | 12 | """Initialize external projects by putting them into the path""" | |
| 13 | - for module in ('async', 'smmap'): | ||
| 13 | + for module in ('smmap',): | ||
| 14 | 14 | sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) | |
| 15 | 15 | ||
| 16 | 16 | try: | |
@@ -27,7 +27,7 @@ def _init_externals(): | |||
| 27 | 27 | __author__ = "Sebastian Thiel" | |
| 28 | 28 | __contact__ = "byronimo@gmail.com" | |
| 29 | 29 | __homepage__ = "https://github.com/gitpython-developers/gitdb" | |
| 30 | - version_info = (0, 5, 5) | ||
| 30 | + version_info = (0, 6, 0) | ||
| 31 | 31 | __version__ = '.'.join(str(i) for i in version_info) | |
| 32 | 32 | ||
| 33 | 33 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -4,7 +4,6 @@ | |||
| 4 | 4 | # the New BSD License: http://www.opensource.org/licenses/bsd-license.php | |
| 5 | 5 | """Contains implementations of database retrieveing objects""" | |
| 6 | 6 | from gitdb.util import ( | |
| 7 | - pool, | ||
| 8 | 7 | join, | |
| 9 | 8 | LazyMixin, | |
| 10 | 9 | hex_to_bin | |
@@ -15,10 +14,6 @@ | |||
| 15 | 14 | AmbiguousObjectName | |
| 16 | 15 | ) | |
| 17 | 16 | ||
| 18 | - from async import ( | ||
| 19 | - ChannelThreadTask | ||
| 20 | - ) | ||
| 21 | - | ||
| 22 | 17 | from itertools import chain | |
| 23 | 18 | from functools import reduce | |
| 24 | 19 | ||
@@ -41,47 +36,18 @@ def has_object(self, sha): | |||
| 41 | 36 | binary sha is contained in the database""" | |
| 42 | 37 | raise NotImplementedError("To be implemented in subclass") | |
| 43 | 38 | ||
| 44 | - def has_object_async(self, reader): | ||
| 45 | - """Return a reader yielding information about the membership of objects | ||
| 46 | - as identified by shas | ||
| 47 | - :param reader: Reader yielding 20 byte shas. | ||
| 48 | - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate | ||
| 49 | - whether the given sha exists in the database or not""" | ||
| 50 | - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) | ||
| 51 | - return pool.add_task(task) | ||
| 52 | - | ||
| 53 | 39 | def info(self, sha): | |
| 54 | 40 | """ :return: OInfo instance | |
| 55 | 41 | :param sha: bytes binary sha | |
| 56 | 42 | :raise BadObject:""" | |
| 57 | 43 | raise NotImplementedError("To be implemented in subclass") | |
| 58 | 44 | ||
| 59 | - def info_async(self, reader): | ||
| 60 | - """Retrieve information of a multitude of objects asynchronously | ||
| 61 | - :param reader: Channel yielding the sha's of the objects of interest | ||
| 62 | - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" | ||
| 63 | - task = ChannelThreadTask(reader, str(self.info_async), self.info) | ||
| 64 | - return pool.add_task(task) | ||
| 65 | - | ||
| 66 | 45 | def stream(self, sha): | |
| 67 | 46 | """:return: OStream instance | |
| 68 | 47 | :param sha: 20 bytes binary sha | |
| 69 | 48 | :raise BadObject:""" | |
| 70 | 49 | raise NotImplementedError("To be implemented in subclass") | |
| 71 | 50 | ||
| 72 | - def stream_async(self, reader): | ||
| 73 | - """Retrieve the OStream of multiple objects | ||
| 74 | - :param reader: see ``info`` | ||
| 75 | - :param max_threads: see ``ObjectDBW.store`` | ||
| 76 | - :return: async.Reader yielding OStream|InvalidOStream instances in any order | ||
| 77 | - | ||
| 78 | - **Note:** depending on the system configuration, it might not be possible to | ||
| 79 | - read all OStreams at once. Instead, read them individually using reader.read(x) | ||
| 80 | - where x is small enough.""" | ||
| 81 | - # base implementation just uses the stream method repeatedly | ||
| 82 | - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) | ||
| 83 | - return pool.add_task(task) | ||
| 84 | - | ||
| 85 | 51 | def size(self): | |
| 86 | 52 | """:return: amount of objects in this database""" | |
| 87 | 53 | raise NotImplementedError() | |
@@ -129,27 +95,6 @@ def store(self, istream): | |||
| 129 | 95 | :raise IOError: if data could not be written""" | |
| 130 | 96 | raise NotImplementedError("To be implemented in subclass") | |
| 131 | 97 | ||
| 132 | - def store_async(self, reader): | ||
| 133 | - """ | ||
| 134 | - Create multiple new objects in the database asynchronously. The method will | ||
| 135 | - return right away, returning an output channel which receives the results as | ||
| 136 | - they are computed. | ||
| 137 | - | ||
| 138 | - :return: Channel yielding your IStream which served as input, in any order. | ||
| 139 | - The IStreams sha will be set to the sha it received during the process, | ||
| 140 | - or its error attribute will be set to the exception informing about the error. | ||
| 141 | - | ||
| 142 | - :param reader: async.Reader yielding IStream instances. | ||
| 143 | - The same instances will be used in the output channel as were received | ||
| 144 | - in by the Reader. | ||
| 145 | - | ||
| 146 | - **Note:** As some ODB implementations implement this operation atomic, they might | ||
| 147 | - abort the whole operation if one item could not be processed. Hence check how | ||
| 148 | - many items have actually been produced.""" | ||
| 149 | - # base implementation uses store to perform the work | ||
| 150 | - task = ChannelThreadTask(reader, str(self.store_async), self.store) | ||
| 151 | - return pool.add_task(task) | ||
| 152 | - | ||
| 153 | 98 | #} END edit interface | |
| 154 | 99 | ||
| 155 | 100 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -32,10 +32,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): | |||
| 32 | 32 | """A memory database stores everything to memory, providing fast IO and object | |
| 33 | 33 | retrieval. It should be used to buffer results and obtain SHAs before writing | |
| 34 | 34 | it to the actual physical storage, as it allows to query whether object already | |
| 35 | - exists in the target storage before introducing actual IO | ||
| 36 | - | ||
| 37 | - **Note:** memory is currently not threadsafe, hence the async methods cannot be used | ||
| 38 | - for storing""" | ||
| 35 | + exists in the target storage before introducing actual IO""" | ||
| 39 | 36 | ||
| 40 | 37 | def __init__(self): | |
| 41 | 38 | super(MemoryDB, self).__init__() | |
@@ -62,9 +59,6 @@ def store(self, istream): | |||
| 62 | 59 | ||
| 63 | 60 | return istream | |
| 64 | 61 | ||
| 65 | - def store_async(self, reader): | ||
| 66 | - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") | ||
| 67 | - | ||
| 68 | 62 | def has_object(self, sha): | |
| 69 | 63 | return sha in self._cache | |
| 70 | 64 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -125,10 +125,6 @@ def store(self, istream): | |||
| 125 | 125 | inefficient""" | |
| 126 | 126 | raise UnsupportedOperation() | |
| 127 | 127 | ||
| 128 | - def store_async(self, reader): | ||
| 129 | - # TODO: add ObjectDBRW before implementing this | ||
| 130 | - raise NotImplementedError() | ||
| 131 | - | ||
| 132 | 128 | #} END object db write | |
| 133 | 129 | ||
| 134 | 130 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2,15 +2,3 @@ | |||
| 2 | 2 | # | |
| 3 | 3 | # This module is part of GitDB and is released under | |
| 4 | 4 | # the New BSD License: http://www.opensource.org/licenses/bsd-license.php | |
| 5 | - | ||
| 6 | - import gitdb.util | ||
| 7 | - | ||
| 8 | - #{ Initialization | ||
| 9 | - def _init_pool(): | ||
| 10 | - """Assure the pool is actually threaded""" | ||
| 11 | - size = 2 | ||
| 12 | - print("Setting ThreadPool to %i" % size) | ||
| 13 | - gitdb.util.pool.set_size(size) | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - #} END initialization | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -10,8 +10,6 @@ | |||
| 10 | 10 | TestBase | |
| 11 | 11 | ) | |
| 12 | 12 | ||
| 13 | - | ||
| 14 | - | ||
| 15 | 13 | from gitdb.stream import ( | |
| 16 | 14 | Sha1Writer, | |
| 17 | 15 | ZippedStoreShaWriter | |
@@ -28,8 +26,6 @@ | |||
| 28 | 26 | from gitdb.utils.encoding import force_bytes | |
| 29 | 27 | from gitdb.utils.compat import xrange | |
| 30 | 28 | ||
| 31 | - from async import IteratorReader | ||
| 32 | - | ||
| 33 | 29 | from io import BytesIO | |
| 34 | 30 | ||
| 35 | 31 | from struct import pack | |
@@ -132,82 +128,3 @@ def _assert_object_writing(self, db): | |||
| 132 | 128 | # END for each data set | |
| 133 | 129 | # END for each dry_run mode | |
| 134 | 130 | ||
| 135 | - def _assert_object_writing_async(self, db): | ||
| 136 | - """Test generic object writing using asynchronous access""" | ||
| 137 | - ni = 5000 | ||
| 138 | - def istream_generator(offset=0, ni=ni): | ||
| 139 | - for data_src in xrange(ni): | ||
| 140 | - data = bytes(data_src + offset) | ||
| 141 | - yield IStream(str_blob_type, len(data), BytesIO(data)) | ||
| 142 | - # END for each item | ||
| 143 | - # END generator utility | ||
| 144 | - | ||
| 145 | - # for now, we are very trusty here as we expect it to work if it worked | ||
| 146 | - # in the single-stream case | ||
| 147 | - | ||
| 148 | - # write objects | ||
| 149 | - reader = IteratorReader(istream_generator()) | ||
| 150 | - istream_reader = db.store_async(reader) | ||
| 151 | - istreams = istream_reader.read() # read all | ||
| 152 | - assert istream_reader.task().error() is None | ||
| 153 | - assert len(istreams) == ni | ||
| 154 | - | ||
| 155 | - for stream in istreams: | ||
| 156 | - assert stream.error is None | ||
| 157 | - assert len(stream.binsha) == 20 | ||
| 158 | - assert isinstance(stream, IStream) | ||
| 159 | - # END assert each stream | ||
| 160 | - | ||
| 161 | - # test has-object-async - we must have all previously added ones | ||
| 162 | - reader = IteratorReader( istream.binsha for istream in istreams ) | ||
| 163 | - hasobject_reader = db.has_object_async(reader) | ||
| 164 | - count = 0 | ||
| 165 | - for sha, has_object in hasobject_reader: | ||
| 166 | - assert has_object | ||
| 167 | - count += 1 | ||
| 168 | - # END for each sha | ||
| 169 | - assert count == ni | ||
| 170 | - | ||
| 171 | - # read the objects we have just written | ||
| 172 | - reader = IteratorReader( istream.binsha for istream in istreams ) | ||
| 173 | - ostream_reader = db.stream_async(reader) | ||
| 174 | - | ||
| 175 | - # read items individually to prevent hitting possible sys-limits | ||
| 176 | - count = 0 | ||
| 177 | - for ostream in ostream_reader: | ||
| 178 | - assert isinstance(ostream, OStream) | ||
| 179 | - count += 1 | ||
| 180 | - # END for each ostream | ||
| 181 | - assert ostream_reader.task().error() is None | ||
| 182 | - assert count == ni | ||
| 183 | - | ||
| 184 | - # get info about our items | ||
| 185 | - reader = IteratorReader( istream.binsha for istream in istreams ) | ||
| 186 | - info_reader = db.info_async(reader) | ||
| 187 | - | ||
| 188 | - count = 0 | ||
| 189 | - for oinfo in info_reader: | ||
| 190 | - assert isinstance(oinfo, OInfo) | ||
| 191 | - count += 1 | ||
| 192 | - # END for each oinfo instance | ||
| 193 | - assert count == ni | ||
| 194 | - | ||
| 195 | - | ||
| 196 | - # combined read-write using a converter | ||
| 197 | - # add 2500 items, and obtain their output streams | ||
| 198 | - nni = 2500 | ||
| 199 | - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) | ||
| 200 | - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] | ||
| 201 | - | ||
| 202 | - istream_reader = db.store_async(reader) | ||
| 203 | - istream_reader.set_post_cb(istream_to_sha) | ||
| 204 | - | ||
| 205 | - ostream_reader = db.stream_async(istream_reader) | ||
| 206 | - | ||
| 207 | - count = 0 | ||
| 208 | - # read it individually, otherwise we might run into the ulimit | ||
| 209 | - for ostream in ostream_reader: | ||
| 210 | - assert isinstance(ostream, OStream) | ||
| 211 | - count += 1 | ||
| 212 | - # END for each ostream | ||
| 213 | - assert count == nni | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -48,4 +48,3 @@ def test_writing(self, path): | |||
| 48 | 48 | ||
| 49 | 49 | # its possible to write objects | |
| 50 | 50 | self._assert_object_writing(gdb) | |
| 51 | - self._assert_object_writing_async(gdb) | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments