FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat(samples): add samples for appendable objects writes and reads (#… · googleapis/python-storage@2e1a1eb · GitHub

This repository was archived by the owner on Mar 31, 2026. It is now read-only.
/ python-storage Public archive

Commit 2e1a1eb

Browse files
chandra-siri
and
gemini-code-assist[bot]
authored
feat(samples): add samples for appendable objects writes and reads (#1705)
feat(samples): add samples for appendable objects writes and reads --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent ea0f5bf commit 2e1a1eb

14 files changed

Lines changed: 1010 additions & 10 deletions

‎cloudbuild/run_zonal_tests.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,4 @@ export RUN_ZONAL_SYSTEM_TESTS=True
2626
CURRENT_ULIMIT=$(ulimit -n)
2727
echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT
2828
pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' tests/system/test_zonal.py
29+
pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py

‎google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,11 +303,19 @@ async def download_ranges(
303303
304304
:type read_ranges: List[Tuple[int, int, "BytesIO"]]
305305
:param read_ranges: A list of tuples, where each tuple represents a
306-
combintaion of byte_range and writeable buffer in format -
306+
combination of byte_range and writeable buffer in format -
307307
(`start_byte`, `bytes_to_read`, `writeable_buffer`). Buffer has
308308
to be provided by the user, and user has to make sure appropriate
309309
memory is available in the application to avoid out-of-memory crash.
310310
311+
Special cases:
312+
if the value of `bytes_to_read` is 0, it'll be interpreted as
313+
download all contents until the end of the file from `start_byte`.
314+
Examples:
315+
* (0, 0, buffer) : downloads 0 to end , i.e. entire object.
316+
* (100, 0, buffer) : downloads from 100 to end.
317+
318+
311319
:type lock: asyncio.Lock
312320
:param lock: (Optional) An asyncio lock to synchronize sends and recvs
313321
on the underlying bidi-GRPC stream. This is required when multiple

‎google/cloud/storage/_experimental/asyncio/async_write_object_stream.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,16 @@ def __init__(
101101
self.object_resource: Optional[_storage_v2.Object] = None
102102

103103
async def open(self) -> None:
104-
"""Opening an object for write , should do it's state lookup
105-
to know what's the persisted size is.
104+
"""
105+
Opens the bidi-gRPC connection to write to the object.
106+
107+
This method sends an initial request to start the stream and receives
108+
the first response containing metadata and a write handle.
109+
110+
:rtype: None
111+
:raises ValueError: If the stream is already open.
112+
:raises google.api_core.exceptions.FailedPrecondition:
113+
if `generation_number` is 0 and object already exists.
106114
"""
107115
if self._is_stream_open:
108116
raise ValueError("Stream is already open")

‎samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2026 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the 'License');
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import argparse
18+
import asyncio
19+
20+
from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import (
21+
AsyncAppendableObjectWriter,
22+
)
23+
from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient
24+
25+
26+
# [START storage_create_and_write_appendable_object]
27+
28+
29+
async def storage_create_and_write_appendable_object(
30+
bucket_name, object_name, grpc_client=None
31+
):
32+
"""Uploads an appendable object to zonal bucket.
33+
34+
grpc_client: an existing grpc_client to use, this is only for testing.
35+
"""
36+
37+
if grpc_client is None:
38+
grpc_client = AsyncGrpcClient().grpc_client
39+
writer = AsyncAppendableObjectWriter(
40+
client=grpc_client,
41+
bucket_name=bucket_name,
42+
object_name=object_name,
43+
generation=0, # throws `FailedPrecondition` if object already exists.
44+
)
45+
# This creates a new appendable object of size 0 and opens it for appending.
46+
await writer.open()
47+
48+
# appends data to the object
49+
# you can perform `.append` multiple times as needed. Data will be appended
50+
# to the end of the object.
51+
await writer.append(b"Some data")
52+
53+
# Once all appends are done, close the gRPC bidirectional stream.
54+
await writer.close()
55+
56+
print(
57+
f"Appended object {object_name} created of size {writer.persisted_size} bytes."
58+
)
59+
60+
61+
# [END storage_create_and_write_appendable_object]
62+
63+
if __name__ == "__main__":
64+
parser = argparse.ArgumentParser(
65+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
66+
)
67+
parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.")
68+
parser.add_argument("--object_name", help="Your Cloud Storage object name.")
69+
70+
args = parser.parse_args()
71+
72+
asyncio.run(
73+
storage_create_and_write_appendable_object(
74+
bucket_name=args.bucket_name,
75+
object_name=args.object_name,
76+
)
77+
)

‎samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2026 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the 'License');
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import argparse
18+
import asyncio
19+
20+
from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import (
21+
AsyncAppendableObjectWriter,
22+
)
23+
from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient
24+
25+
26+
# [START storage_finalize_appendable_object_upload]
27+
async def storage_finalize_appendable_object_upload(
28+
bucket_name, object_name, grpc_client=None
29+
):
30+
"""Creates, writes to, and finalizes an appendable object.
31+
32+
grpc_client: an existing grpc_client to use, this is only for testing.
33+
"""
34+
35+
if grpc_client is None:
36+
grpc_client = AsyncGrpcClient().grpc_client
37+
writer = AsyncAppendableObjectWriter(
38+
client=grpc_client,
39+
bucket_name=bucket_name,
40+
object_name=object_name,
41+
generation=0, # throws `FailedPrecondition` if object already exists.
42+
)
43+
# This creates a new appendable object of size 0 and opens it for appending.
44+
await writer.open()
45+
46+
# Appends data to the object.
47+
await writer.append(b"Some data")
48+
49+
# finalize the appendable object,
50+
# NOTE:
51+
# 1. once finalized no more appends can be done to the object.
52+
# 2. If you don't want to finalize, you can simply call `writer.close`
53+
# 3. calling `.finalize()` also closes the grpc-bidi stream, calling
54+
# `.close` after `.finalize` may lead to undefined behavior.
55+
object_resource = await writer.finalize()
56+
57+
print(f"Appendable object {object_name} created and finalized.")
58+
print("Object Metadata:")
59+
print(object_resource)
60+
61+
62+
# [END storage_finalize_appendable_object_upload]
63+
64+
if __name__ == "__main__":
65+
parser = argparse.ArgumentParser(
66+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
67+
)
68+
parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.")
69+
parser.add_argument("--object_name", help="Your Cloud Storage object name.")
70+
71+
args = parser.parse_args()
72+
73+
asyncio.run(
74+
storage_finalize_appendable_object_upload(
75+
bucket_name=args.bucket_name,
76+
object_name=args.object_name,
77+
)
78+
)

‎samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2026 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the 'License');
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Downloads a range of bytes from multiple objects concurrently."""
18+
import argparse
19+
import asyncio
20+
from io import BytesIO
21+
22+
from google.cloud.storage._experimental.asyncio.async_grpc_client import (
23+
AsyncGrpcClient,
24+
)
25+
from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import (
26+
AsyncMultiRangeDownloader,
27+
)
28+
29+
30+
# [START storage_open_multiple_objects_ranged_read]
31+
async def storage_open_multiple_objects_ranged_read(
32+
bucket_name, object_names, grpc_client=None
33+
):
34+
"""Downloads a range of bytes from multiple objects concurrently.
35+
36+
grpc_client: an existing grpc_client to use, this is only for testing.
37+
"""
38+
if grpc_client is None:
39+
grpc_client = AsyncGrpcClient().grpc_client
40+
41+
async def _download_range(object_name):
42+
"""Helper coroutine to download a range from a single object."""
43+
mrd = AsyncMultiRangeDownloader(grpc_client, bucket_name, object_name)
44+
try:
45+
# Open the object, mrd always opens in read mode.
46+
await mrd.open()
47+
48+
# Each object downloads the first 100 bytes.
49+
start_byte = 0
50+
size = 100
51+
52+
# requested range will be downloaded into this buffer, user may provide
53+
# their own buffer or file-like object.
54+
output_buffer = BytesIO()
55+
await mrd.download_ranges([(start_byte, size, output_buffer)])
56+
finally:
57+
if mrd.is_stream_open:
58+
await mrd.close()
59+
60+
# Downloaded size can differ from requested size if object is smaller.
61+
# mrd will download at most up to the end of the object.
62+
downloaded_size = output_buffer.getbuffer().nbytes
63+
print(f"Downloaded {downloaded_size} bytes from {object_name}")
64+
65+
download_tasks = [_download_range(name) for name in object_names]
66+
await asyncio.gather(*download_tasks)
67+
68+
69+
# [END storage_open_multiple_objects_ranged_read]
70+
71+
72+
if __name__ == "__main__":
73+
parser = argparse.ArgumentParser(
74+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
75+
)
76+
parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.")
77+
parser.add_argument(
78+
"--object_names", nargs="+", help="Your Cloud Storage object name(s)."
79+
)
80+
81+
args = parser.parse_args()
82+
83+
asyncio.run(
84+
storage_open_multiple_objects_ranged_read(args.bucket_name, args.object_names)
85+
)

‎samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2026 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the 'License');
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import argparse
18+
import asyncio
19+
from io import BytesIO
20+
21+
from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient
22+
from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import (
23+
AsyncMultiRangeDownloader,
24+
)
25+
26+
27+
# [START storage_open_object_multiple_ranged_read]
28+
async def storage_open_object_multiple_ranged_read(
29+
bucket_name, object_name, grpc_client=None
30+
):
31+
"""Downloads multiple ranges of bytes from a single object into different buffers.
32+
33+
grpc_client: an existing grpc_client to use, this is only for testing.
34+
"""
35+
if grpc_client is None:
36+
grpc_client = AsyncGrpcClient().grpc_client
37+
38+
mrd = AsyncMultiRangeDownloader(grpc_client, bucket_name, object_name)
39+
40+
try:
41+
# Open the object, mrd always opens in read mode.
42+
await mrd.open()
43+
44+
# Specify four different buffers to download ranges into.
45+
buffers = [BytesIO(), BytesIO(), BytesIO(), BytesIO()]
46+
47+
# Define the ranges to download. Each range is a tuple of (start_byte, size, buffer).
48+
# All ranges will download 10 bytes from different starting positions.
49+
# We choose arbitrary start bytes for this example. An object should be large enough.
50+
# A user can choose any start byte between 0 and `object_size`.
51+
# If `start_bytes` is greater than `object_size`, mrd will throw an error.
52+
ranges = [
53+
(0, 10, buffers[0]),
54+
(20, 10, buffers[1]),
55+
(40, 10, buffers[2]),
56+
(60, 10, buffers[3]),
57+
]
58+
59+
await mrd.download_ranges(ranges)
60+
61+
finally:
62+
await mrd.close()
63+
64+
# Print the downloaded content from each buffer.
65+
for i, output_buffer in enumerate(buffers):
66+
downloaded_size = output_buffer.getbuffer().nbytes
67+
print(
68+
f"Downloaded {downloaded_size} bytes into buffer {i + 1} from start byte {ranges[i][0]}: {output_buffer.getvalue()}"
69+
)
70+
71+
72+
# [END storage_open_object_multiple_ranged_read]
73+
74+
if __name__ == "__main__":
75+
parser = argparse.ArgumentParser(
76+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
77+
)
78+
parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.")
79+
parser.add_argument("--object_name", help="Your Cloud Storage object name.")
80+
81+
args = parser.parse_args()
82+
83+
asyncio.run(
84+
storage_open_object_multiple_ranged_read(args.bucket_name, args.object_name)
85+
)

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL