FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat: update generation for MRD (#1730) · googleapis/python-storage@08bc708 · GitHub

This repository was archived by the owner on Mar 31, 2026. It is now read-only.
/ python-storage Public archive

Commit 08bc708

Browse files
Pulkit0110
authored
feat: update generation for MRD (#1730)
Adding a support to pass `generation` in the constructor of MRD. It is done to make it compatible with the other public interfaces. Earlier, `generation_number` was exposed as the parameter which would still be valid after these changes. But using `generation` is more preferred.
1 parent 74c9ecc commit 08bc708

7 files changed

Lines changed: 95 additions & 78 deletions

File tree

‎google/cloud/storage/asyncio/async_multi_range_downloader.py‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,11 @@ async def create_mrd(
128128
client: AsyncGrpcClient,
129129
bucket_name: str,
130130
object_name: str,
131-
generation_number: Optional[int] = None,
131+
generation: Optional[int] = None,
132132
read_handle: Optional[_storage_v2.BidiReadHandle] = None,
133133
retry_policy: Optional[AsyncRetry] = None,
134134
metadata: Optional[List[Tuple[str, str]]] = None,
135+
**kwargs,
135136
) -> AsyncMultiRangeDownloader:
136137
"""Initializes a MultiRangeDownloader and opens the underlying bidi-gRPC
137138
object for reading.
@@ -145,8 +146,8 @@ async def create_mrd(
145146
:type object_name: str
146147
:param object_name: The name of the object to be read.
147148
148-
:type generation_number: int
149-
:param generation_number: (Optional) If present, selects a specific
149+
:type generation: int
150+
:param generation: (Optional) If present, selects a specific
150151
revision of this object.
151152
152153
:type read_handle: _storage_v2.BidiReadHandle
@@ -162,7 +163,14 @@ async def create_mrd(
162163
:rtype: :class:`~google.cloud.storage.asyncio.async_multi_range_downloader.AsyncMultiRangeDownloader`
163164
:returns: An initialized AsyncMultiRangeDownloader instance for reading.
164165
"""
165-
mrd = cls(client, bucket_name, object_name, generation_number, read_handle)
166+
mrd = cls(
167+
client,
168+
bucket_name,
169+
object_name,
170+
generation=generation,
171+
read_handle=read_handle,
172+
**kwargs,
173+
)
166174
await mrd.open(retry_policy=retry_policy, metadata=metadata)
167175
return mrd
168176

@@ -171,8 +179,9 @@ def __init__(
171179
client: AsyncGrpcClient,
172180
bucket_name: str,
173181
object_name: str,
174-
generation_number: Optional[int] = None,
182+
generation: Optional[int] = None,
175183
read_handle: Optional[_storage_v2.BidiReadHandle] = None,
184+
**kwargs,
176185
) -> None:
177186
"""Constructor for AsyncMultiRangeDownloader, clients are not adviced to
178187
use it directly. Instead it's adviced to use the classmethod `create_mrd`.
@@ -186,20 +195,27 @@ def __init__(
186195
:type object_name: str
187196
:param object_name: The name of the object to be read.
188197
189-
:type generation_number: int
190-
:param generation_number: (Optional) If present, selects a specific revision of
198+
:type generation: int
199+
:param generation: (Optional) If present, selects a specific revision of
191200
this object.
192201
193202
:type read_handle: _storage_v2.BidiReadHandle
194203
:param read_handle: (Optional) An existing read handle.
195204
"""
205+
if "generation_number" in kwargs:
206+
if generation is not None:
207+
raise TypeError(
208+
"Cannot set both 'generation' and 'generation_number'. "
209+
"Use 'generation' for new code."
210+
)
211+
generation = kwargs.pop("generation_number")
196212

197213
raise_if_no_fast_crc32c()
198214

199215
self.client = client
200216
self.bucket_name = bucket_name
201217
self.object_name = object_name
202-
self.generation_number = generation_number
218+
self.generation = generation
203219
self.read_handle: Optional[_storage_v2.BidiReadHandle] = read_handle
204220
self.read_obj_str: Optional[_AsyncReadObjectStream] = None
205221
self._is_stream_open: bool = False
@@ -276,7 +292,7 @@ async def _do_open():
276292
client=self.client.grpc_client,
277293
bucket_name=self.bucket_name,
278294
object_name=self.object_name,
279-
generation_number=self.generation_number,
295+
generation_number=self.generation,
280296
read_handle=self.read_handle,
281297
)
282298

@@ -291,7 +307,7 @@ async def _do_open():
291307
)
292308

293309
if self.read_obj_str.generation_number:
294-
self.generation_number = self.read_obj_str.generation_number
310+
self.generation = self.read_obj_str.generation_number
295311
if self.read_obj_str.read_handle:
296312
self.read_handle = self.read_obj_str.read_handle
297313
if self.read_obj_str.persisted_size is not None:
@@ -435,7 +451,7 @@ async def generator():
435451
client=self.client.grpc_client,
436452
bucket_name=self.bucket_name,
437453
object_name=self.object_name,
438-
generation_number=self.generation_number,
454+
generation_number=self.generation,
439455
read_handle=current_handle,
440456
)
441457

‎tests/system/test_zonal.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,8 @@ async def _run():
264264

265265
event_loop.run_until_complete(_run())
266266

267-
@pytest.mark.skip(reason='Flaky test b/478129078')
267+
268+
@pytest.mark.skip(reason="Flaky test b/478129078")
268269
def test_mrd_open_with_read_handle(event_loop, grpc_client_direct):
269270
object_name = f"test_read_handl-{str(uuid.uuid4())[:4]}"
270271

‎tests/unit/asyncio/retry/test_reads_resumption_strategy.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@
2727
from google.cloud._storage_v2.types.storage import BidiReadObjectRedirectedError
2828

2929
_READ_ID = 1
30-
LOGGER_NAME = (
31-
"google.cloud.storage.asyncio.retry.reads_resumption_strategy"
32-
)
30+
LOGGER_NAME = "google.cloud.storage.asyncio.retry.reads_resumption_strategy"
3331

3432

3533
class TestDownloadState(unittest.TestCase):

‎tests/unit/asyncio/test_async_multi_range_downloader.py‎

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ async def _make_mock_mrd(
4949
mock_cls_async_read_object_stream,
5050
bucket_name=_TEST_BUCKET_NAME,
5151
object_name=_TEST_OBJECT_NAME,
52-
generation_number=_TEST_GENERATION_NUMBER,
52+
generation=_TEST_GENERATION_NUMBER,
5353
read_handle=_TEST_READ_HANDLE,
5454
):
5555
mock_client = mock.MagicMock()
@@ -62,7 +62,7 @@ async def _make_mock_mrd(
6262
mock_stream.read_handle = _TEST_READ_HANDLE
6363

6464
mrd = await AsyncMultiRangeDownloader.create_mrd(
65-
mock_client, bucket_name, object_name, generation_number, read_handle
65+
mock_client, bucket_name, object_name, generation, read_handle
6666
)
6767

6868
return mrd, mock_client
@@ -89,7 +89,7 @@ async def test_create_mrd(self, mock_cls_async_read_object_stream):
8989
assert mrd.client == mock_client
9090
assert mrd.bucket_name == _TEST_BUCKET_NAME
9191
assert mrd.object_name == _TEST_OBJECT_NAME
92-
assert mrd.generation_number == _TEST_GENERATION_NUMBER
92+
assert mrd.generation == _TEST_GENERATION_NUMBER
9393
assert mrd.read_handle == _TEST_READ_HANDLE
9494
assert mrd.persisted_size == _TEST_OBJECT_SIZE
9595
assert mrd.is_stream_open
@@ -303,9 +303,7 @@ async def test_downloading_without_opening_should_throw_error(self):
303303
assert not mrd.is_stream_open
304304

305305
@mock.patch("google.cloud.storage.asyncio._utils.google_crc32c")
306-
def test_init_raises_if_crc32c_c_extension_is_missing(
307-
self, mock_google_crc32c
308-
):
306+
def test_init_raises_if_crc32c_c_extension_is_missing(self, mock_google_crc32c):
309307
mock_google_crc32c.implementation = "python"
310308
mock_client = mock.MagicMock()
311309

@@ -317,9 +315,7 @@ def test_init_raises_if_crc32c_c_extension_is_missing(
317315
)
318316

319317
@pytest.mark.asyncio
320-
@mock.patch(
321-
"google.cloud.storage.asyncio.retry.reads_resumption_strategy.Checksum"
322-
)
318+
@mock.patch("google.cloud.storage.asyncio.retry.reads_resumption_strategy.Checksum")
323319
async def test_download_ranges_raises_on_checksum_mismatch(
324320
self, mock_checksum_class
325321
):
@@ -405,3 +401,47 @@ async def close_side_effect():
405401

406402
mock_close.assert_called_once()
407403
assert not mrd.is_stream_open
404+
405+
@mock.patch(
406+
"google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream"
407+
)
408+
@pytest.mark.asyncio
409+
async def test_create_mrd_with_generation_number(
410+
self, mock_cls_async_read_object_stream
411+
):
412+
# Arrange
413+
mock_client = mock.MagicMock()
414+
mock_client.grpc_client = mock.AsyncMock()
415+
416+
mock_stream = mock_cls_async_read_object_stream.return_value
417+
mock_stream.open = AsyncMock()
418+
mock_stream.generation_number = _TEST_GENERATION_NUMBER
419+
mock_stream.persisted_size = _TEST_OBJECT_SIZE
420+
mock_stream.read_handle = _TEST_READ_HANDLE
421+
422+
# Act
423+
mrd = await AsyncMultiRangeDownloader.create_mrd(
424+
mock_client,
425+
_TEST_BUCKET_NAME,
426+
_TEST_OBJECT_NAME,
427+
generation_number=_TEST_GENERATION_NUMBER,
428+
read_handle=_TEST_READ_HANDLE,
429+
)
430+
431+
# Assert
432+
assert mrd.generation == _TEST_GENERATION_NUMBER
433+
434+
@pytest.mark.asyncio
435+
async def test_create_mrd_with_both_generation_and_generation_number(self):
436+
# Arrange
437+
mock_client = mock.MagicMock()
438+
439+
# Act & Assert
440+
with pytest.raises(TypeError):
441+
await AsyncMultiRangeDownloader.create_mrd(
442+
mock_client,
443+
_TEST_BUCKET_NAME,
444+
_TEST_OBJECT_NAME,
445+
generation=_TEST_GENERATION_NUMBER,
446+
generation_number=_TEST_GENERATION_NUMBER,
447+
)

‎tests/unit/asyncio/test_async_read_object_stream.py‎

Lines changed: 12 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,7 @@ async def instantiate_read_obj_stream_with_read_handle(
7979
return read_obj_stream
8080

8181

82-
@mock.patch(
83-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
84-
)
82+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
8583
@mock.patch(
8684
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
8785
)
@@ -110,9 +108,7 @@ def test_init_with_bucket_object_generation(mock_client, mock_async_bidi_rpc):
110108
assert read_obj_stream.rpc == rpc_sentinel
111109

112110

113-
@mock.patch(
114-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
115-
)
111+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
116112
@mock.patch(
117113
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
118114
)
@@ -136,9 +132,7 @@ async def test_open(mock_client, mock_cls_async_bidi_rpc):
136132
assert read_obj_stream.is_stream_open
137133

138134

139-
@mock.patch(
140-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
141-
)
135+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
142136
@mock.patch(
143137
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
144138
)
@@ -162,9 +156,7 @@ async def test_open_with_read_handle(mock_client, mock_cls_async_bidi_rpc):
162156
assert read_obj_stream.is_stream_open
163157

164158

165-
@mock.patch(
166-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
167-
)
159+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
168160
@mock.patch(
169161
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
170162
)
@@ -185,9 +177,7 @@ async def test_open_when_already_open_should_raise_error(
185177
assert str(exc.value) == "Stream is already open"
186178

187179

188-
@mock.patch(
189-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
190-
)
180+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
191181
@mock.patch(
192182
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
193183
)
@@ -208,9 +198,7 @@ async def test_close(mock_client, mock_cls_async_bidi_rpc):
208198
assert not read_obj_stream.is_stream_open
209199

210200

211-
@mock.patch(
212-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
213-
)
201+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
214202
@mock.patch(
215203
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
216204
)
@@ -232,9 +220,7 @@ async def test_requests_done(mock_client, mock_cls_async_bidi_rpc):
232220
read_obj_stream.socket_like_rpc.recv.assert_called_once()
233221

234222

235-
@mock.patch(
236-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
237-
)
223+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
238224
@mock.patch(
239225
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
240226
)
@@ -255,9 +241,7 @@ async def test_close_without_open_should_raise_error(
255241
assert str(exc.value) == "Stream is not open"
256242

257243

258-
@mock.patch(
259-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
260-
)
244+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
261245
@mock.patch(
262246
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
263247
)
@@ -278,9 +262,7 @@ async def test_send(mock_client, mock_cls_async_bidi_rpc):
278262
)
279263

280264

281-
@mock.patch(
282-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
283-
)
265+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
284266
@mock.patch(
285267
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
286268
)
@@ -301,9 +283,7 @@ async def test_send_without_open_should_raise_error(
301283
assert str(exc.value) == "Stream is not open"
302284

303285

304-
@mock.patch(
305-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
306-
)
286+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
307287
@mock.patch(
308288
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
309289
)
@@ -326,9 +306,7 @@ async def test_recv(mock_client, mock_cls_async_bidi_rpc):
326306
assert response == bidi_read_object_response
327307

328308

329-
@mock.patch(
330-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
331-
)
309+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
332310
@mock.patch(
333311
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
334312
)
@@ -349,9 +327,7 @@ async def test_recv_without_open_should_raise_error(
349327
assert str(exc.value) == "Stream is not open"
350328

351329

352-
@mock.patch(
353-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
354-
)
330+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
355331
@mock.patch(
356332
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
357333
)

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL