| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -21,6 +21,7 @@ | |||
| 21 | 21 | if you want to use these Rapid Storage APIs. | |
| 22 | 22 | ||
| 23 | 23 | """ | |
| 24 | + from io import BufferedReader | ||
| 24 | 25 | from typing import Optional, Union | |
| 25 | 26 | ||
| 26 | 27 | from google_crc32c import Checksum | |
@@ -339,6 +340,16 @@ async def append_from_stream(self, stream_obj): | |||
| 339 | 340 | """ | |
| 340 | 341 | raise NotImplementedError("append_from_stream is not implemented yet.") | |
| 341 | 342 | ||
| 342 | - async def append_from_file(self, file_path: str): | ||
| 343 | - """Create a file object from `file_path` and call append_from_stream(file_obj)""" | ||
| 344 | - raise NotImplementedError("append_from_file is not implemented yet.") | ||
| 343 | + async def append_from_file( | ||
| 344 | + self, file_obj: BufferedReader, block_size: int = _DEFAULT_FLUSH_INTERVAL_BYTES | ||
| 345 | + ): | ||
| 346 | + """ | ||
| 347 | + Appends data to an Appendable Object using file_handle which is opened | ||
| 348 | + for reading in binary mode. | ||
| 349 | + | ||
| 350 | + :type file_obj: file | ||
| 351 | + :param file_obj: A file handle opened in binary mode for reading. | ||
| 352 | + | ||
| 353 | + """ | ||
| 354 | + while block := file_obj.read(block_size): | ||
| 355 | + await self.append(block) | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -12,6 +12,7 @@ | |||
| 12 | 12 | # See the License for the specific language governing permissions and | |
| 13 | 13 | # limitations under the License. | |
| 14 | 14 | ||
| 15 | + from io import BytesIO | ||
| 15 | 16 | import pytest | |
| 16 | 17 | from unittest import mock | |
| 17 | 18 | ||
@@ -23,6 +24,7 @@ | |||
| 23 | 24 | ) | |
| 24 | 25 | from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( | |
| 25 | 26 | _MAX_CHUNK_SIZE_BYTES, | |
| 27 | + _DEFAULT_FLUSH_INTERVAL_BYTES, | ||
| 26 | 28 | ) | |
| 27 | 29 | from google.cloud import _storage_v2 | |
| 28 | 30 | ||
@@ -287,9 +289,6 @@ async def test_unimplemented_methods_raise_error(mock_client): | |||
| 287 | 289 | with pytest.raises(NotImplementedError): | |
| 288 | 290 | await writer.append_from_stream(mock.Mock()) | |
| 289 | 291 | ||
| 290 | - with pytest.raises(NotImplementedError): | ||
| 291 | - await writer.append_from_file("file.txt") | ||
| 292 | - | ||
| 293 | 292 | ||
| 294 | 293 | @pytest.mark.asyncio | |
| 295 | 294 | @mock.patch( | |
@@ -536,9 +535,6 @@ async def test_append_flushes_when_buffer_is_full( | |||
| 536 | 535 | mock_write_object_stream, mock_client | |
| 537 | 536 | ): | |
| 538 | 537 | """Test that append flushes the stream when the buffer size is reached.""" | |
| 539 | - from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( | ||
| 540 | - _DEFAULT_FLUSH_INTERVAL_BYTES, | ||
| 541 | - ) | ||
| 542 | 538 | ||
| 543 | 539 | writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) | |
| 544 | 540 | writer._is_stream_open = True | |
@@ -559,9 +555,6 @@ async def test_append_flushes_when_buffer_is_full( | |||
| 559 | 555 | ) | |
| 560 | 556 | async def test_append_handles_large_data(mock_write_object_stream, mock_client): | |
| 561 | 557 | """Test that append handles data larger than the buffer size.""" | |
| 562 | - from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( | ||
| 563 | - _DEFAULT_FLUSH_INTERVAL_BYTES, | ||
| 564 | - ) | ||
| 565 | 558 | ||
| 566 | 559 | writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) | |
| 567 | 560 | writer._is_stream_open = True | |
@@ -602,3 +595,32 @@ async def test_append_data_two_times(mock_write_object_stream, mock_client): | |||
| 602 | 595 | total_data_length = len(data1) + len(data2) | |
| 603 | 596 | assert writer.offset == total_data_length | |
| 604 | 597 | assert writer.simple_flush.await_count == 0 | |
| 598 | + | ||
| 599 | + | ||
| 600 | + @pytest.mark.asyncio | ||
| 601 | + @pytest.mark.parametrize( | ||
| 602 | + "file_size, block_size", | ||
| 603 | + [ | ||
| 604 | + (10, 4 * 1024), | ||
| 605 | + (0, _DEFAULT_FLUSH_INTERVAL_BYTES), | ||
| 606 | + (20 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES), | ||
| 607 | + (16 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES), | ||
| 608 | + ], | ||
| 609 | + ) | ||
| 610 | + async def test_append_from_file(file_size, block_size, mock_client): | ||
| 611 | + # arrange | ||
| 612 | + fp = BytesIO(b"a" * file_size) | ||
| 613 | + writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) | ||
| 614 | + writer._is_stream_open = True | ||
| 615 | + writer.append = mock.AsyncMock() | ||
| 616 | + | ||
| 617 | + # act | ||
| 618 | + await writer.append_from_file(fp, block_size=block_size) | ||
| 619 | + | ||
| 620 | + # assert | ||
| 621 | + exepected_calls = ( | ||
| 622 | + file_size // block_size | ||
| 623 | + if file_size % block_size == 0 | ||
| 624 | + else file_size // block_size + 1 | ||
| 625 | + ) | ||
| 626 | + assert writer.append.await_count == exepected_calls | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments