| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1532,6 +1532,14 @@ def to_dict(self): | |||
| 1532 | 1532 | "transaction_id": snapshot._transaction_id, | |
| 1533 | 1533 | } | |
| 1534 | 1534 | ||
| 1535 | + def __enter__(self): | ||
| 1536 | + """Begin ``with`` block.""" | ||
| 1537 | + return self | ||
| 1538 | + | ||
| 1539 | + def __exit__(self, exc_type, exc_val, exc_tb): | ||
| 1540 | + """End ``with`` block.""" | ||
| 1541 | + self.close() | ||
| 1542 | + | ||
| 1535 | 1543 | @property | |
| 1536 | 1544 | def observability_options(self): | |
| 1537 | 1545 | return getattr(self._database, "observability_options", {}) | |
@@ -1703,6 +1711,7 @@ def process_read_batch( | |||
| 1703 | 1711 | *, | |
| 1704 | 1712 | retry=gapic_v1.method.DEFAULT, | |
| 1705 | 1713 | timeout=gapic_v1.method.DEFAULT, | |
| 1714 | + lazy_decode=False, | ||
| 1706 | 1715 | ): | |
| 1707 | 1716 | """Process a single, partitioned read. | |
| 1708 | 1717 | ||
@@ -1717,6 +1726,14 @@ def process_read_batch( | |||
| 1717 | 1726 | :type timeout: float | |
| 1718 | 1727 | :param timeout: (Optional) The timeout for this request. | |
| 1719 | 1728 | ||
| 1729 | + :type lazy_decode: bool | ||
| 1730 | + :param lazy_decode: | ||
| 1731 | + (Optional) If this argument is set to ``true``, the iterator | ||
| 1732 | + returns the underlying protobuf values instead of decoded Python | ||
| 1733 | + objects. This reduces the time that is needed to iterate through | ||
| 1734 | + large result sets. The application is responsible for decoding | ||
| 1735 | + the data that is needed. | ||
| 1736 | + | ||
| 1720 | 1737 | ||
| 1721 | 1738 | :rtype: :class:`~google.cloud.spanner_v1.streamed.StreamedResultSet` | |
| 1722 | 1739 | :returns: a result set instance which can be used to consume rows. | |
@@ -1844,6 +1861,7 @@ def process_query_batch( | |||
| 1844 | 1861 | self, | |
| 1845 | 1862 | batch, | |
| 1846 | 1863 | *, | |
| 1864 | + lazy_decode: bool = False, | ||
| 1847 | 1865 | retry=gapic_v1.method.DEFAULT, | |
| 1848 | 1866 | timeout=gapic_v1.method.DEFAULT, | |
| 1849 | 1867 | ): | |
@@ -1854,6 +1872,13 @@ def process_query_batch( | |||
| 1854 | 1872 | one of the mappings returned from an earlier call to | |
| 1855 | 1873 | :meth:`generate_query_batches`. | |
| 1856 | 1874 | ||
| 1875 | + :type lazy_decode: bool | ||
| 1876 | + :param lazy_decode: | ||
| 1877 | + (Optional) If this argument is set to ``true``, the iterator | ||
| 1878 | + returns the underlying protobuf values instead of decoded Python | ||
| 1879 | + objects. This reduces the time that is needed to iterate through | ||
| 1880 | + large result sets. | ||
| 1881 | + | ||
| 1857 | 1882 | :type retry: :class:`~google.api_core.retry.Retry` | |
| 1858 | 1883 | :param retry: (Optional) The retry settings for this request. | |
| 1859 | 1884 | ||
@@ -1870,6 +1895,7 @@ def process_query_batch( | |||
| 1870 | 1895 | return self._get_snapshot().execute_sql( | |
| 1871 | 1896 | partition=batch["partition"], | |
| 1872 | 1897 | **batch["query"], | |
| 1898 | + lazy_decode=lazy_decode, | ||
| 1873 | 1899 | retry=retry, | |
| 1874 | 1900 | timeout=timeout, | |
| 1875 | 1901 | ) | |
@@ -1883,6 +1909,7 @@ def run_partitioned_query( | |||
| 1883 | 1909 | max_partitions=None, | |
| 1884 | 1910 | query_options=None, | |
| 1885 | 1911 | data_boost_enabled=False, | |
| 1912 | + lazy_decode=False, | ||
| 1886 | 1913 | ): | |
| 1887 | 1914 | """Start a partitioned query operation to get list of partitions and | |
| 1888 | 1915 | then executes each partition on a separate thread | |
@@ -1943,7 +1970,7 @@ def run_partitioned_query( | |||
| 1943 | 1970 | data_boost_enabled, | |
| 1944 | 1971 | ) | |
| 1945 | 1972 | ) | |
| 1946 | - return MergedResultSet(self, partitions, 0) | ||
| 1973 | + return MergedResultSet(self, partitions, 0, lazy_decode=lazy_decode) | ||
| 1947 | 1974 | ||
| 1948 | 1975 | def process(self, batch): | |
| 1949 | 1976 | """Process a single, partitioned query or read. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -33,10 +33,13 @@ class PartitionExecutor: | |||
| 33 | 33 | rows in the queue | |
| 34 | 34 | """ | |
| 35 | 35 | ||
| 36 | - def __init__(self, batch_snapshot, partition_id, merged_result_set): | ||
| 36 | + def __init__( | ||
| 37 | + self, batch_snapshot, partition_id, merged_result_set, lazy_decode=False | ||
| 38 | + ): | ||
| 37 | 39 | self._batch_snapshot: BatchSnapshot = batch_snapshot | |
| 38 | 40 | self._partition_id = partition_id | |
| 39 | 41 | self._merged_result_set: MergedResultSet = merged_result_set | |
| 42 | + self._lazy_decode = lazy_decode | ||
| 40 | 43 | self._queue: Queue[PartitionExecutorResult] = merged_result_set._queue | |
| 41 | 44 | ||
| 42 | 45 | def run(self): | |
@@ -52,7 +55,9 @@ def run(self): | |||
| 52 | 55 | def __run(self): | |
| 53 | 56 | results = None | |
| 54 | 57 | try: | |
| 55 | - results = self._batch_snapshot.process_query_batch(self._partition_id) | ||
| 58 | + results = self._batch_snapshot.process_query_batch( | ||
| 59 | + self._partition_id, lazy_decode=self._lazy_decode | ||
| 60 | + ) | ||
| 56 | 61 | for row in results: | |
| 57 | 62 | if self._merged_result_set._metadata is None: | |
| 58 | 63 | self._set_metadata(results) | |
@@ -75,6 +80,7 @@ def _set_metadata(self, results, is_exception=False): | |||
| 75 | 80 | try: | |
| 76 | 81 | if not is_exception: | |
| 77 | 82 | self._merged_result_set._metadata = results.metadata | |
| 83 | + self._merged_result_set._result_set = results | ||
| 78 | 84 | finally: | |
| 79 | 85 | self._merged_result_set.metadata_lock.release() | |
| 80 | 86 | self._merged_result_set.metadata_event.set() | |
@@ -94,7 +100,10 @@ class MergedResultSet: | |||
| 94 | 100 | records in the MergedResultSet is not guaranteed. | |
| 95 | 101 | """ | |
| 96 | 102 | ||
| 97 | - def __init__(self, batch_snapshot, partition_ids, max_parallelism): | ||
| 103 | + def __init__( | ||
| 104 | + self, batch_snapshot, partition_ids, max_parallelism, lazy_decode=False | ||
| 105 | + ): | ||
| 106 | + self._result_set = None | ||
| 98 | 107 | self._exception = None | |
| 99 | 108 | self._metadata = None | |
| 100 | 109 | self.metadata_event = Event() | |
@@ -110,7 +119,7 @@ def __init__(self, batch_snapshot, partition_ids, max_parallelism): | |||
| 110 | 119 | partition_executors = [] | |
| 111 | 120 | for partition_id in partition_ids: | |
| 112 | 121 | partition_executors.append( | |
| 113 | - PartitionExecutor(batch_snapshot, partition_id, self) | ||
| 122 | + PartitionExecutor(batch_snapshot, partition_id, self, lazy_decode) | ||
| 114 | 123 | ) | |
| 115 | 124 | executor = ThreadPoolExecutor(max_workers=parallelism) | |
| 116 | 125 | for partition_executor in partition_executors: | |
@@ -144,3 +153,27 @@ def metadata(self): | |||
| 144 | 153 | def stats(self): | |
| 145 | 154 | # TODO: Implement | |
| 146 | 155 | return None | |
| 156 | + | ||
| 157 | + def decode_row(self, row: []) -> []: | ||
| 158 | + """Decodes a row from protobuf values to Python objects. This function | ||
| 159 | + should only be called for result sets that use ``lazy_decoding=True``. | ||
| 160 | + The array that is returned by this function is the same as the array | ||
| 161 | + that would have been returned by the rows iterator if ``lazy_decoding=False``. | ||
| 162 | + | ||
| 163 | + :returns: an array containing the decoded values of all the columns in the given row | ||
| 164 | + """ | ||
| 165 | + if self._result_set is None: | ||
| 166 | + raise ValueError("iterator not started") | ||
| 167 | + return self._result_set.decode_row(row) | ||
| 168 | + | ||
| 169 | + def decode_column(self, row: [], column_index: int): | ||
| 170 | + """Decodes a column from a protobuf value to a Python object. This function | ||
| 171 | + should only be called for result sets that use ``lazy_decoding=True``. | ||
| 172 | + The object that is returned by this function is the same as the object | ||
| 173 | + that would have been returned by the rows iterator if ``lazy_decoding=False``. | ||
| 174 | + | ||
| 175 | + :returns: the decoded column value | ||
| 176 | + """ | ||
| 177 | + if self._result_set is None: | ||
| 178 | + raise ValueError("iterator not started") | ||
| 179 | + return self._result_set.decode_column(row, column_index) | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -3141,6 +3141,7 @@ def test_process_query_batch(self): | |||
| 3141 | 3141 | params=params, | |
| 3142 | 3142 | param_types=param_types, | |
| 3143 | 3143 | partition=token, | |
| 3144 | + lazy_decode=False, | ||
| 3144 | 3145 | retry=gapic_v1.method.DEFAULT, | |
| 3145 | 3146 | timeout=gapic_v1.method.DEFAULT, | |
| 3146 | 3147 | ) | |
@@ -3170,6 +3171,7 @@ def test_process_query_batch_w_retry_timeout(self): | |||
| 3170 | 3171 | params=params, | |
| 3171 | 3172 | param_types=param_types, | |
| 3172 | 3173 | partition=token, | |
| 3174 | + lazy_decode=False, | ||
| 3173 | 3175 | retry=retry, | |
| 3174 | 3176 | timeout=2.0, | |
| 3175 | 3177 | ) | |
@@ -3193,11 +3195,23 @@ def test_process_query_batch_w_directed_read_options(self): | |||
| 3193 | 3195 | snapshot.execute_sql.assert_called_once_with( | |
| 3194 | 3196 | sql=sql, | |
| 3195 | 3197 | partition=token, | |
| 3198 | + lazy_decode=False, | ||
| 3196 | 3199 | retry=gapic_v1.method.DEFAULT, | |
| 3197 | 3200 | timeout=gapic_v1.method.DEFAULT, | |
| 3198 | 3201 | directed_read_options=DIRECTED_READ_OPTIONS, | |
| 3199 | 3202 | ) | |
| 3200 | 3203 | ||
| 3204 | + def test_context_manager(self): | ||
| 3205 | + database = self._make_database() | ||
| 3206 | + batch_txn = self._make_one(database) | ||
| 3207 | + session = batch_txn._session = self._make_session() | ||
| 3208 | + session.is_multiplexed = False | ||
| 3209 | + | ||
| 3210 | + with batch_txn: | ||
| 3211 | + pass | ||
| 3212 | + | ||
| 3213 | + session.delete.assert_called_once_with() | ||
| 3214 | + | ||
| 3201 | 3215 | def test_close_wo_session(self): | |
| 3202 | 3216 | database = self._make_database() | |
| 3203 | 3217 | batch_txn = self._make_one(database) | |
@@ -3292,6 +3306,7 @@ def test_process_w_query_batch(self): | |||
| 3292 | 3306 | params=params, | |
| 3293 | 3307 | param_types=param_types, | |
| 3294 | 3308 | partition=token, | |
| 3309 | + lazy_decode=False, | ||
| 3295 | 3310 | retry=gapic_v1.method.DEFAULT, | |
| 3296 | 3311 | timeout=gapic_v1.method.DEFAULT, | |
| 3297 | 3312 | ) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,119 @@ | |||
| 1 | + # Copyright 2025 Google LLC All rights reserved. | ||
| 2 | + # | ||
| 3 | + # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 4 | + # you may not use this file except in compliance with the License. | ||
| 5 | + # You may obtain a copy of the License at | ||
| 6 | + # | ||
| 7 | + # http://www.apache.org/licenses/LICENSE-2.0 | ||
| 8 | + # | ||
| 9 | + # Unless required by applicable law or agreed to in writing, software | ||
| 10 | + # distributed under the License is distributed on an "AS IS" BASIS, | ||
| 11 | + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 12 | + # See the License for the specific language governing permissions and | ||
| 13 | + # limitations under the License. | ||
| 14 | + | ||
| 15 | + import unittest | ||
| 16 | + | ||
| 17 | + import mock | ||
| 18 | + from google.cloud.spanner_v1.streamed import StreamedResultSet | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + class TestMergedResultSet(unittest.TestCase): | ||
| 22 | + def _get_target_class(self): | ||
| 23 | + from google.cloud.spanner_v1.merged_result_set import MergedResultSet | ||
| 24 | + | ||
| 25 | + return MergedResultSet | ||
| 26 | + | ||
| 27 | + def _make_one(self, *args, **kwargs): | ||
| 28 | + klass = self._get_target_class() | ||
| 29 | + obj = super(klass, klass).__new__(klass) | ||
| 30 | + from threading import Event, Lock | ||
| 31 | + | ||
| 32 | + obj.metadata_event = Event() | ||
| 33 | + obj.metadata_lock = Lock() | ||
| 34 | + obj._metadata = None | ||
| 35 | + obj._result_set = None | ||
| 36 | + return obj | ||
| 37 | + | ||
| 38 | + @staticmethod | ||
| 39 | + def _make_value(value): | ||
| 40 | + from google.cloud.spanner_v1._helpers import _make_value_pb | ||
| 41 | + | ||
| 42 | + return _make_value_pb(value) | ||
| 43 | + | ||
| 44 | + @staticmethod | ||
| 45 | + def _make_scalar_field(name, type_): | ||
| 46 | + from google.cloud.spanner_v1 import StructType | ||
| 47 | + from google.cloud.spanner_v1 import Type | ||
| 48 | + | ||
| 49 | + return StructType.Field(name=name, type_=Type(code=type_)) | ||
| 50 | + | ||
| 51 | + @staticmethod | ||
| 52 | + def _make_result_set_metadata(fields=()): | ||
| 53 | + from google.cloud.spanner_v1 import ResultSetMetadata | ||
| 54 | + from google.cloud.spanner_v1 import StructType | ||
| 55 | + | ||
| 56 | + metadata = ResultSetMetadata(row_type=StructType(fields=[])) | ||
| 57 | + for field in fields: | ||
| 58 | + metadata.row_type.fields.append(field) | ||
| 59 | + return metadata | ||
| 60 | + | ||
| 61 | + def test_stats_property(self): | ||
| 62 | + merged = self._make_one() | ||
| 63 | + # The property is currently not implemented, so it should just return None. | ||
| 64 | + self.assertIsNone(merged.stats) | ||
| 65 | + | ||
| 66 | + def test_decode_row(self): | ||
| 67 | + merged = self._make_one() | ||
| 68 | + | ||
| 69 | + merged._result_set = mock.create_autospec(StreamedResultSet, instance=True) | ||
| 70 | + merged._result_set.decode_row.return_value = ["Phred", 42] | ||
| 71 | + | ||
| 72 | + raw_row = [self._make_value("Phred"), self._make_value(42)] | ||
| 73 | + decoded_row = merged.decode_row(raw_row) | ||
| 74 | + | ||
| 75 | + self.assertEqual(decoded_row, ["Phred", 42]) | ||
| 76 | + merged._result_set.decode_row.assert_called_once_with(raw_row) | ||
| 77 | + | ||
| 78 | + def test_decode_row_no_result_set(self): | ||
| 79 | + merged = self._make_one() | ||
| 80 | + merged._result_set = None | ||
| 81 | + with self.assertRaisesRegex(ValueError, "iterator not started"): | ||
| 82 | + merged.decode_row([]) | ||
| 83 | + | ||
| 84 | + def test_decode_row_type_error(self): | ||
| 85 | + merged = self._make_one() | ||
| 86 | + merged._result_set = mock.create_autospec(StreamedResultSet, instance=True) | ||
| 87 | + merged._result_set.decode_row.side_effect = TypeError | ||
| 88 | + | ||
| 89 | + with self.assertRaises(TypeError): | ||
| 90 | + merged.decode_row("not a list") | ||
| 91 | + | ||
| 92 | + def test_decode_column(self): | ||
| 93 | + merged = self._make_one() | ||
| 94 | + merged._result_set = mock.create_autospec(StreamedResultSet, instance=True) | ||
| 95 | + merged._result_set.decode_column.side_effect = ["Phred", 42] | ||
| 96 | + | ||
| 97 | + raw_row = [self._make_value("Phred"), self._make_value(42)] | ||
| 98 | + decoded_name = merged.decode_column(raw_row, 0) | ||
| 99 | + decoded_age = merged.decode_column(raw_row, 1) | ||
| 100 | + | ||
| 101 | + self.assertEqual(decoded_name, "Phred") | ||
| 102 | + self.assertEqual(decoded_age, 42) | ||
| 103 | + merged._result_set.decode_column.assert_has_calls( | ||
| 104 | + [mock.call(raw_row, 0), mock.call(raw_row, 1)] | ||
| 105 | + ) | ||
| 106 | + | ||
| 107 | + def test_decode_column_no_result_set(self): | ||
| 108 | + merged = self._make_one() | ||
| 109 | + merged._result_set = None | ||
| 110 | + with self.assertRaisesRegex(ValueError, "iterator not started"): | ||
| 111 | + merged.decode_column([], 0) | ||
| 112 | + | ||
| 113 | + def test_decode_column_type_error(self): | ||
| 114 | + merged = self._make_one() | ||
| 115 | + merged._result_set = mock.create_autospec(StreamedResultSet, instance=True) | ||
| 116 | + merged._result_set.decode_column.side_effect = TypeError | ||
| 117 | + | ||
| 118 | + with self.assertRaises(TypeError): | ||
| 119 | + merged.decode_column("not a list", 0) | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments