FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat: add *generation*match args into Blob.compose() (#122) · googleapis/python-storage@dc01c59 · GitHub

This repository was archived by the owner on Mar 31, 2026. It is now read-only.
/ python-storage Public archive

Commit dc01c59

Browse files
Gurov Ilya
and
frankyn
authored
feat: add *generation*match args into Blob.compose() (#122)
* feat: add *generation*match args into Blob.compose() * add test case with error * new compose surface * Revert "new compose surface" This reverts commit 2ddda40. * add an error for different length args, add a usage example * add condition to avoid sending params with None * specify comments Co-authored-by: Frank Natividad <frankyn@users.noreply.github.com>
1 parent 90c020d commit dc01c59

3 files changed

Lines changed: 228 additions & 7 deletions

File tree

‎google/cloud/storage/blob.py‎

Lines changed: 74 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2234,34 +2234,104 @@ def make_private(self, client=None):
22342234
self.acl.all().revoke_read()
22352235
self.acl.save(client=client)
22362236

2237-
def compose(self, sources, client=None, timeout=_DEFAULT_TIMEOUT):
2237+
def compose(
2238+
self,
2239+
sources,
2240+
client=None,
2241+
timeout=_DEFAULT_TIMEOUT,
2242+
if_generation_match=None,
2243+
if_metageneration_match=None,
2244+
):
22382245
"""Concatenate source blobs into this one.
22392246
22402247
If :attr:`user_project` is set on the bucket, bills the API request
22412248
to that project.
22422249
22432250
:type sources: list of :class:`Blob`
2244-
:param sources: blobs whose contents will be composed into this blob.
2251+
:param sources: Blobs whose contents will be composed into this blob.
22452252
22462253
:type client: :class:`~google.cloud.storage.client.Client` or
22472254
``NoneType``
2248-
:param client: (Optional) The client to use. If not passed, falls back
2255+
:param client: (Optional) The client to use. If not passed, falls back
22492256
to the ``client`` stored on the blob's bucket.
2257+
22502258
:type timeout: float or tuple
22512259
:param timeout: (Optional) The amount of time, in seconds, to wait
22522260
for the server response.
22532261
22542262
Can also be passed as a tuple (connect_timeout, read_timeout).
22552263
See :meth:`requests.Session.request` documentation for details.
2264+
2265+
:type if_generation_match: list of long
2266+
:param if_generation_match: (Optional) Make the operation conditional on whether
2267+
the blob's current generation matches the given value.
2268+
Setting to 0 makes the operation succeed only if there
2269+
are no live versions of the blob. The list must match
2270+
``sources`` item-to-item.
2271+
2272+
:type if_metageneration_match: list of long
2273+
:param if_metageneration_match: (Optional) Make the operation conditional on whether
2274+
the blob's current metageneration matches the given
2275+
value. The list must match ``sources`` item-to-item.
2276+
2277+
Example:
2278+
Compose blobs using generation match preconditions.
2279+
2280+
>>> from google.cloud import storage
2281+
>>> client = storage.Client()
2282+
>>> bucket = client.bucket("bucket-name")
2283+
2284+
>>> blobs = [bucket.blob("blob-name-1"), bucket.blob("blob-name-2")]
2285+
>>> if_generation_match = [None] * len(blobs)
2286+
>>> if_generation_match[0] = "123" # precondition for "blob-name-1"
2287+
2288+
>>> composed_blob = bucket.blob("composed-name")
2289+
>>> composed_blob.compose(blobs, if_generation_match)
22562290
"""
2291+
sources_len = len(sources)
2292+
if if_generation_match is not None and len(if_generation_match) != sources_len:
2293+
raise ValueError(
2294+
"'if_generation_match' length must be the same as 'sources' length"
2295+
)
2296+
2297+
if (
2298+
if_metageneration_match is not None
2299+
and len(if_metageneration_match) != sources_len
2300+
):
2301+
raise ValueError(
2302+
"'if_metageneration_match' length must be the same as 'sources' length"
2303+
)
2304+
22572305
client = self._require_client(client)
22582306
query_params = {}
22592307

22602308
if self.user_project is not None:
22612309
query_params["userProject"] = self.user_project
22622310

2311+
source_objects = []
2312+
for index, source in enumerate(sources):
2313+
source_object = {"name": source.name}
2314+
2315+
preconditions = {}
2316+
if (
2317+
if_generation_match is not None
2318+
and if_generation_match[index] is not None
2319+
):
2320+
preconditions["ifGenerationMatch"] = if_generation_match[index]
2321+
2322+
if (
2323+
if_metageneration_match is not None
2324+
and if_metageneration_match[index] is not None
2325+
):
2326+
preconditions["ifMetagenerationMatch"] = if_metageneration_match[index]
2327+
2328+
if preconditions:
2329+
source_object["objectPreconditions"] = preconditions
2330+
2331+
source_objects.append(source_object)
2332+
22632333
request = {
2264-
"sourceObjects": [{"name": source.name} for source in sources],
2334+
"sourceObjects": source_objects,
22652335
"destination": self._properties.copy(),
22662336
}
22672337
api_response = client._connection.api_request(

‎tests/system/test_system.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1441,6 +1441,34 @@ def test_compose_replace_existing_blob(self):
14411441
composed = original.download_as_string()
14421442
self.assertEqual(composed, BEFORE + TO_APPEND)
14431443

1444+
def test_compose_with_generation_match(self):
1445+
BEFORE = b"AAA\n"
1446+
original = self.bucket.blob("original")
1447+
original.content_type = "text/plain"
1448+
original.upload_from_string(BEFORE)
1449+
self.case_blobs_to_delete.append(original)
1450+
1451+
TO_APPEND = b"BBB\n"
1452+
to_append = self.bucket.blob("to_append")
1453+
to_append.upload_from_string(TO_APPEND)
1454+
self.case_blobs_to_delete.append(to_append)
1455+
1456+
with self.assertRaises(google.api_core.exceptions.PreconditionFailed):
1457+
original.compose(
1458+
[original, to_append],
1459+
if_generation_match=[6, 7],
1460+
if_metageneration_match=[8, 9],
1461+
)
1462+
1463+
original.compose(
1464+
[original, to_append],
1465+
if_generation_match=[original.generation, to_append.generation],
1466+
if_metageneration_match=[original.metageneration, to_append.metageneration],
1467+
)
1468+
1469+
composed = original.download_as_string()
1470+
self.assertEqual(composed, BEFORE + TO_APPEND)
1471+
14441472
@unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
14451473
def test_compose_with_user_project(self):
14461474
new_bucket_name = "compose-user-project" + unique_resource_id("-")

‎tests/unit/test_blob.py‎

Lines changed: 126 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2676,7 +2676,7 @@ def test_make_private(self):
26762676
def test_compose_wo_content_type_set(self):
26772677
SOURCE_1 = "source-1"
26782678
SOURCE_2 = "source-2"
2679-
DESTINATION = "destinaton"
2679+
DESTINATION = "destination"
26802680
RESOURCE = {}
26812681
after = ({"status": http_client.OK}, RESOURCE)
26822682
connection = _Connection(after)
@@ -2711,7 +2711,7 @@ def test_compose_wo_content_type_set(self):
27112711
def test_compose_minimal_w_user_project(self):
27122712
SOURCE_1 = "source-1"
27132713
SOURCE_2 = "source-2"
2714-
DESTINATION = "destinaton"
2714+
DESTINATION = "destination"
27152715
RESOURCE = {"etag": "DEADBEEF"}
27162716
USER_PROJECT = "user-project-123"
27172717
after = ({"status": http_client.OK}, RESOURCE)
@@ -2747,7 +2747,7 @@ def test_compose_minimal_w_user_project(self):
27472747
def test_compose_w_additional_property_changes(self):
27482748
SOURCE_1 = "source-1"
27492749
SOURCE_2 = "source-2"
2750-
DESTINATION = "destinaton"
2750+
DESTINATION = "destination"
27512751
RESOURCE = {"etag": "DEADBEEF"}
27522752
after = ({"status": http_client.OK}, RESOURCE)
27532753
connection = _Connection(after)
@@ -2785,6 +2785,129 @@ def test_compose_w_additional_property_changes(self):
27852785
},
27862786
)
27872787

2788+
def test_compose_w_generation_match(self):
2789+
SOURCE_1 = "source-1"
2790+
SOURCE_2 = "source-2"
2791+
DESTINATION = "destination"
2792+
RESOURCE = {}
2793+
GENERATION_NUMBERS = [6, 9]
2794+
METAGENERATION_NUMBERS = [7, 1]
2795+
2796+
after = ({"status": http_client.OK}, RESOURCE)
2797+
connection = _Connection(after)
2798+
client = _Client(connection)
2799+
bucket = _Bucket(client=client)
2800+
source_1 = self._make_one(SOURCE_1, bucket=bucket)
2801+
source_2 = self._make_one(SOURCE_2, bucket=bucket)
2802+
2803+
destination = self._make_one(DESTINATION, bucket=bucket)
2804+
destination.compose(
2805+
sources=[source_1, source_2],
2806+
if_generation_match=GENERATION_NUMBERS,
2807+
if_metageneration_match=METAGENERATION_NUMBERS,
2808+
)
2809+
2810+
kw = connection._requested
2811+
self.assertEqual(len(kw), 1)
2812+
self.assertEqual(
2813+
kw[0],
2814+
{
2815+
"method": "POST",
2816+
"path": "/b/name/o/%s/compose" % DESTINATION,
2817+
"query_params": {},
2818+
"data": {
2819+
"sourceObjects": [
2820+
{
2821+
"name": source_1.name,
2822+
"objectPreconditions": {
2823+
"ifGenerationMatch": GENERATION_NUMBERS[0],
2824+
"ifMetagenerationMatch": METAGENERATION_NUMBERS[0],
2825+
},
2826+
},
2827+
{
2828+
"name": source_2.name,
2829+
"objectPreconditions": {
2830+
"ifGenerationMatch": GENERATION_NUMBERS[1],
2831+
"ifMetagenerationMatch": METAGENERATION_NUMBERS[1],
2832+
},
2833+
},
2834+
],
2835+
"destination": {},
2836+
},
2837+
"_target_object": destination,
2838+
"timeout": self._get_default_timeout(),
2839+
},
2840+
)
2841+
2842+
def test_compose_w_generation_match_bad_length(self):
2843+
SOURCE_1 = "source-1"
2844+
SOURCE_2 = "source-2"
2845+
DESTINATION = "destination"
2846+
GENERATION_NUMBERS = [6]
2847+
METAGENERATION_NUMBERS = [7]
2848+
2849+
after = ({"status": http_client.OK}, {})
2850+
connection = _Connection(after)
2851+
client = _Client(connection)
2852+
bucket = _Bucket(client=client)
2853+
source_1 = self._make_one(SOURCE_1, bucket=bucket)
2854+
source_2 = self._make_one(SOURCE_2, bucket=bucket)
2855+
2856+
destination = self._make_one(DESTINATION, bucket=bucket)
2857+
2858+
with self.assertRaises(ValueError):
2859+
destination.compose(
2860+
sources=[source_1, source_2], if_generation_match=GENERATION_NUMBERS,
2861+
)
2862+
with self.assertRaises(ValueError):
2863+
destination.compose(
2864+
sources=[source_1, source_2],
2865+
if_metageneration_match=METAGENERATION_NUMBERS,
2866+
)
2867+
2868+
def test_compose_w_generation_match_nones(self):
2869+
SOURCE_1 = "source-1"
2870+
SOURCE_2 = "source-2"
2871+
DESTINATION = "destination"
2872+
GENERATION_NUMBERS = [6, None]
2873+
2874+
after = ({"status": http_client.OK}, {})
2875+
connection = _Connection(after)
2876+
client = _Client(connection)
2877+
bucket = _Bucket(client=client)
2878+
source_1 = self._make_one(SOURCE_1, bucket=bucket)
2879+
source_2 = self._make_one(SOURCE_2, bucket=bucket)
2880+
2881+
destination = self._make_one(DESTINATION, bucket=bucket)
2882+
destination.compose(
2883+
sources=[source_1, source_2], if_generation_match=GENERATION_NUMBERS,
2884+
)
2885+
2886+
kw = connection._requested
2887+
self.assertEqual(len(kw), 1)
2888+
self.assertEqual(
2889+
kw[0],
2890+
{
2891+
"method": "POST",
2892+
"path": "/b/name/o/%s/compose" % DESTINATION,
2893+
"query_params": {},
2894+
"data": {
2895+
"sourceObjects": [
2896+
{
2897+
"name": source_1.name,
2898+
"objectPreconditions": {
2899+
"ifGenerationMatch": GENERATION_NUMBERS[0],
2900+
},
2901+
},
2902+
{"name": source_2.name},
2903+
],
2904+
"destination": {},
2905+
},
2906+
"_target_object": destination,
2907+
"timeout": self._get_default_timeout(),
2908+
},
2909+
)
2910+
27882911
def test_rewrite_response_without_resource(self):
27892912
SOURCE_BLOB = "source"
27902913
DEST_BLOB = "dest"

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL