FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Factoring out some Blob helpers. · googleapis/google-cloud-python@74f8152 · GitHub

/ google-cloud-python Public

Commit 74f8152

Browse files
dhermes
committed
Factoring out some Blob helpers.
This is prep work for swapping out the upload implementation to use `google-resumable-media`.
1 parent d1c1a50 commit 74f8152

2 files changed

Lines changed: 159 additions & 35 deletions

File tree

‎storage/google/cloud/storage/blob.py‎

Lines changed: 89 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import time
2828

2929
import httplib2
30-
import six
3130
from six.moves.urllib.parse import quote
3231

3332
import google.auth.transport.requests
@@ -50,8 +49,10 @@
5049

5150

5251
_API_ACCESS_ENDPOINT = 'https://storage.googleapis.com'
52+
_DEFAULT_CONTENT_TYPE = u'application/octet-stream'
5353
_DOWNLOAD_URL_TEMPLATE = (
5454
u'https://www.googleapis.com/download/storage/v1{path}?alt=media')
55+
_CONTENT_TYPE = 'contentType'
5556

5657

5758
class Blob(_PropertyMixin):
@@ -192,7 +193,7 @@ def public_url(self):
192193
:returns: The public URL for this blob.
193194
"""
194195
return '{storage_base_url}/{bucket_name}/{quoted_name}'.format(
195-
storage_base_url='https://storage.googleapis.com',
196+
storage_base_url=_API_ACCESS_ENDPOINT,
196197
bucket_name=self.bucket.name,
197198
quoted_name=_quote(self.name))
198199

@@ -269,7 +270,7 @@ def generate_signed_url(self, expiration, method='GET',
269270

270271
if credentials is None:
271272
client = self._require_client(client)
272-
credentials = client._base_connection.credentials
273+
credentials = client._credentials
273274

274275
return generate_signed_url(
275276
credentials, resource=resource,
@@ -324,6 +325,23 @@ def delete(self, client=None):
324325
"""
325326
return self.bucket.delete_blob(self.name, client=client)
326327

328+
def _make_transport(self, client):
329+
"""Make an authenticated transport with a client's credentials.
330+
331+
:type client: :class:`~google.cloud.storage.client.Client`
332+
:param client: (Optional) The client to use. If not passed, falls back
333+
to the ``client`` stored on the blob's bucket.
334+
:rtype transport:
335+
:class:`~google.auth.transport.requests.AuthorizedSession`
336+
:returns: The transport (with credentials) that will
337+
make authenticated requests.
338+
"""
339+
client = self._require_client(client)
340+
# Create a ``requests`` transport with the client's credentials.
341+
transport = google.auth.transport.requests.AuthorizedSession(
342+
client._credentials)
343+
return transport
344+
327345
def _get_download_url(self):
328346
"""Get the download URL for the current blob.
329347
@@ -403,14 +421,9 @@ def download_to_file(self, file_obj, client=None):
403421
404422
:raises: :class:`google.cloud.exceptions.NotFound`
405423
"""
406-
client = self._require_client(client)
407-
# Get the download URL.
408424
download_url = self._get_download_url()
409-
# Get any extra headers for the request.
410425
headers = _get_encryption_headers(self._encryption_key)
411-
# Create a ``requests`` transport with the client's credentials.
412-
transport = google.auth.transport.requests.AuthorizedSession(
413-
client._credentials)
426+
transport = self._make_transport(client)
414427

415428
try:
416429
self._do_download(transport, file_obj, download_url, headers)
@@ -457,6 +470,36 @@ def download_as_string(self, client=None):
457470
self.download_to_file(string_buffer, client=client)
458471
return string_buffer.getvalue()
459472

473+
def _get_content_type(self, content_type, filename=None):
474+
"""Determine the content type from the current object.
475+
476+
The return value will be determined in order of precedence:
477+
478+
- The value passed in to this method (if not :data:`None)
479+
- The value stored on the current blob
480+
- The default value ('application/octet-stream')
481+
482+
:type content_type: str
483+
:param content_type: (Optional) type of content.
484+
485+
:type filename: str
486+
:param filename: (Optional) The name of the file where the content
487+
is stored.
488+
489+
:rtype: str
490+
:returns: Type of content gathered from the object.
491+
"""
492+
if content_type is None:
493+
content_type = self.content_type
494+
495+
if content_type is None and filename is not None:
496+
content_type, _ = mimetypes.guess_type(filename)
497+
498+
if content_type is None:
499+
content_type = _DEFAULT_CONTENT_TYPE
500+
501+
return content_type
502+
460503
def _create_upload(
461504
self, client, file_obj=None, size=None, content_type=None,
462505
chunk_size=None, strategy=None, extra_headers=None):
@@ -509,8 +552,7 @@ def _create_upload(
509552
# API_BASE_URL and build_api_url).
510553
connection = client._base_connection
511554

512-
content_type = (content_type or self._properties.get('contentType') or
513-
'application/octet-stream')
555+
content_type = self._get_content_type(content_type)
514556

515557
headers = {
516558
'Accept': 'application/json',
@@ -575,10 +617,12 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
575617
content_type=None, num_retries=6, client=None):
576618
"""Upload the contents of this blob from a file-like object.
577619
578-
The content type of the upload will either be
579-
- The value passed in to the function (if any)
620+
The content type of the upload will be determined in order
621+
of precedence:
622+
623+
- The value passed in to this method (if not :data:`None)
580624
- The value stored on the current blob
581-
- The default value of 'application/octet-stream'
625+
- The default value ('application/octet-stream')
582626
583627
.. note::
584628
The effect of uploading to an existing blob depends on the
@@ -640,10 +684,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
640684
# API_BASE_URL and build_api_url).
641685
connection = client._base_connection
642686

643-
# Rewind the file if desired.
644-
if rewind:
645-
file_obj.seek(0, os.SEEK_SET)
646-
687+
_maybe_rewind(file_obj, rewind=rewind)
647688
# Get the basic stats about the file.
648689
total_bytes = size
649690
if total_bytes is None:
@@ -679,18 +720,19 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
679720
self._check_response_error(request, http_response)
680721
response_content = http_response.content
681722

682-
if not isinstance(response_content,
683-
six.string_types): # pragma: NO COVER Python3
684-
response_content = response_content.decode('utf-8')
723+
response_content = _bytes_to_unicode(response_content)
685724
self._set_properties(json.loads(response_content))
686725

687726
def upload_from_filename(self, filename, content_type=None, client=None):
688727
"""Upload this blob's contents from the content of a named file.
689728
690-
The content type of the upload will either be
691-
- The value passed in to the function (if any)
729+
The content type of the upload will be determined in order
730+
of precedence:
731+
732+
- The value passed in to this method (if not :data:`None)
692733
- The value stored on the current blob
693-
- The value given by mimetypes.guess_type
734+
- The value given by ``mimetypes.guess_type``
735+
- The default value ('application/octet-stream')
694736
695737
.. note::
696738
The effect of uploading to an existing blob depends on the
@@ -714,9 +756,7 @@ def upload_from_filename(self, filename, content_type=None, client=None):
714756
:param client: Optional. The client to use. If not passed, falls back
715757
to the ``client`` stored on the blob's bucket.
716758
"""
717-
content_type = content_type or self._properties.get('contentType')
718-
if content_type is None:
719-
content_type, _ = mimetypes.guess_type(filename)
759+
content_type = self._get_content_type(content_type, filename=filename)
720760

721761
with open(filename, 'rb') as file_obj:
722762
self.upload_from_file(
@@ -749,8 +789,7 @@ def upload_from_string(self, data, content_type='text/plain', client=None):
749789
:param client: Optional. The client to use. If not passed, falls back
750790
to the ``client`` stored on the blob's bucket.
751791
"""
752-
if isinstance(data, six.text_type):
753-
data = data.encode('utf-8')
792+
data = _to_bytes(data, encoding='utf-8')
754793
string_buffer = BytesIO()
755794
string_buffer.write(data)
756795
self.upload_from_file(
@@ -777,10 +816,12 @@ def create_resumable_upload_session(
777816
.. _documentation on signed URLs: https://cloud.google.com/storage\
778817
/docs/access-control/signed-urls#signing-resumable
779818
780-
The content type of the upload will either be
781-
- The value passed in to the function (if any)
819+
The content type of the upload will be determined in order
820+
of precedence:
821+
822+
- The value passed in to this method (if not :data:`None)
782823
- The value stored on the current blob
783-
- The default value of 'application/octet-stream'
824+
- The default value ('application/octet-stream')
784825
785826
.. note::
786827
The effect of uploading to an existing blob depends on the
@@ -1080,7 +1121,7 @@ def update_storage_class(self, new_class, client=None):
10801121
:rtype: str or ``NoneType``
10811122
"""
10821123

1083-
content_type = _scalar_property('contentType')
1124+
content_type = _scalar_property(_CONTENT_TYPE)
10841125
"""HTTP 'Content-Type' header for this object.
10851126
10861127
See: https://tools.ietf.org/html/rfc2616#section-14.17 and
@@ -1353,8 +1394,8 @@ def _get_encryption_headers(key, source=False):
13531394

13541395
key = _to_bytes(key)
13551396
key_hash = hashlib.sha256(key).digest()
1356-
key_hash = base64.b64encode(key_hash).rstrip()
1357-
key = base64.b64encode(key).rstrip()
1397+
key_hash = base64.b64encode(key_hash)
1398+
key = base64.b64encode(key)
13581399

13591400
if source:
13601401
prefix = 'X-Goog-Copy-Source-Encryption-'
@@ -1384,3 +1425,16 @@ def _quote(value):
13841425
"""
13851426
value = _to_bytes(value, encoding='utf-8')
13861427
return quote(value, safe='')
1428+
1429+
1430+
def _maybe_rewind(stream, rewind=False):
1431+
"""Rewind the stream if desired.
1432+
1433+
:type stream: IO[Bytes]
1434+
:param stream: A bytes IO object open for reading.
1435+
1436+
:type rewind: bool
1437+
:param rewind: Indicates if we should seek to the beginning of the stream.
1438+
"""
1439+
if rewind:
1440+
stream.seek(0, os.SEEK_SET)

‎storage/tests/unit/test_blob.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import os
1516
import unittest
1617

1718
import mock
@@ -325,6 +326,15 @@ def test_delete(self):
325326
self.assertFalse(blob.exists())
326327
self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
327328

329+
@mock.patch('google.auth.transport.requests.AuthorizedSession')
330+
def test__make_transport(self, fake_session_factory):
331+
client = mock.Mock(spec=[u'_credentials'])
332+
blob = self._make_one(u'blob-name', bucket=None)
333+
transport = blob._make_transport(client)
334+
335+
self.assertIs(transport, fake_session_factory.return_value)
336+
fake_session_factory.assert_called_once_with(client._credentials)
337+
328338
def test__get_download_url_with_media_link(self):
329339
blob_name = 'something.txt'
330340
bucket = mock.Mock(spec=[])
@@ -674,6 +684,32 @@ def test_download_as_string(self, fake_session_factory):
674684

675685
self._check_session_mocks(client, fake_session_factory, media_link)
676686

687+
def test__get_content_type_explicit(self):
688+
blob = self._make_one(u'blob-name', bucket=None)
689+
690+
content_type = u'text/plain'
691+
return_value = blob._get_content_type(content_type)
692+
self.assertEqual(return_value, content_type)
693+
694+
def test__get_content_type_from_blob(self):
695+
blob = self._make_one(u'blob-name', bucket=None)
696+
blob.content_type = u'video/mp4'
697+
698+
return_value = blob._get_content_type(None)
699+
self.assertEqual(return_value, blob.content_type)
700+
701+
def test__get_content_type_from_filename(self):
702+
blob = self._make_one(u'blob-name', bucket=None)
703+
704+
return_value = blob._get_content_type(None, filename='archive.tar')
705+
self.assertEqual(return_value, 'application/x-tar')
706+
707+
def test__get_content_type_default(self):
708+
blob = self._make_one(u'blob-name', bucket=None)
709+
710+
return_value = blob._get_content_type(None)
711+
self.assertEqual(return_value, u'application/octet-stream')
712+
677713
def test_upload_from_file_size_failure(self):
678714
BLOB_NAME = 'blob-name'
679715
connection = _Connection()
@@ -2270,6 +2306,36 @@ def test_bad_type(self):
22702306
self._call_fut(None)
22712307

22722308

2309+
class Test__maybe_rewind(unittest.TestCase):
2310+
2311+
@staticmethod
2312+
def _call_fut(*args, **kwargs):
2313+
from google.cloud.storage.blob import _maybe_rewind
2314+
2315+
return _maybe_rewind(*args, **kwargs)
2316+
2317+
def test_default(self):
2318+
stream = mock.Mock(spec=[u'seek'])
2319+
ret_val = self._call_fut(stream)
2320+
self.assertIsNone(ret_val)
2321+
2322+
stream.seek.assert_not_called()
2323+
2324+
def test_do_not_rewind(self):
2325+
stream = mock.Mock(spec=[u'seek'])
2326+
ret_val = self._call_fut(stream, rewind=False)
2327+
self.assertIsNone(ret_val)
2328+
2329+
stream.seek.assert_not_called()
2330+
2331+
def test_do_rewind(self):
2332+
stream = mock.Mock(spec=[u'seek'])
2333+
ret_val = self._call_fut(stream, rewind=True)
2334+
self.assertIsNone(ret_val)
2335+
2336+
stream.seek.assert_called_once_with(0, os.SEEK_SET)
2337+
2338+
22732339
class _Responder(object):
22742340

22752341
def __init__(self, *responses):
@@ -2363,6 +2429,10 @@ def __init__(self, connection):
23632429
def _connection(self):
23642430
return self._base_connection
23652431

2432+
@property
2433+
def _credentials(self):
2434+
return self._base_connection.credentials
2435+
23662436

23672437
class _Stream(object):
23682438
_closed = False

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL