FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

perf: improve row merging (#619) · googleapis/google-cloud-python@a4911fd · GitHub

/ google-cloud-python Public

Commit a4911fd

Browse files
igorbernstein2
authored
perf: improve row merging (#619)
The underlying GAPIC client uses protoplus for all requests and responses. However the underlying protos for ReadRowsResponse are never exposed to end users directly: the underlying chunks get merged into logic rows. The readability benefits provided by protoplus for ReadRows do not justify the costs. This change unwraps the protoplus messages and uses the raw protobuff message as input for row merging. This improves row merging performance by 10x. For 10k rows, each with 100 cells where each cell is 100 bytes and in groups of 100 rows per ReadRowsResponse, cProfile showed a 10x improvement: old: 124266037 function calls in 68.208 seconds new: 13042837 function calls in 7.787 seconds There are still a few more low hanging fruits to optimize performance and those will come in follow up PRs
1 parent 94dcc2d commit a4911fd

3 files changed

Lines changed: 31 additions & 18 deletions

File tree

‎packages/google-cloud-bigtable/google/cloud/bigtable/row_data.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,11 @@ def _read_next(self):
474474

475475
def _read_next_response(self):
476476
"""Helper for :meth:`__iter__`."""
477-
return self.retry(self._read_next, on_error=self._on_error)()
477+
resp_protoplus = self.retry(self._read_next, on_error=self._on_error)()
478+
# unwrap the underlying protobuf, there is a significant amount of
479+
# overhead that protoplus imposes for very little gain. The protos
480+
# are not user visible, so we just use the raw protos for merging.
481+
return data_messages_v2_pb2.ReadRowsResponse.pb(resp_protoplus)
478482

479483
def __iter__(self):
480484
"""Consume the ``ReadRowsResponse`` s from the stream.
@@ -543,11 +547,12 @@ def _process_chunk(self, chunk):
543547
def _update_cell(self, chunk):
544548
if self._cell is None:
545549
qualifier = None
546-
if "qualifier" in chunk:
547-
qualifier = chunk.qualifier
550+
if chunk.HasField("qualifier"):
551+
qualifier = chunk.qualifier.value
552+
548553
family = None
549-
if "family_name" in chunk:
550-
family = chunk.family_name
554+
if chunk.HasField("family_name"):
555+
family = chunk.family_name.value
551556

552557
self._cell = PartialCellData(
553558
chunk.row_key,
@@ -577,8 +582,8 @@ def _validate_chunk_reset_row(self, chunk):
577582

578583
# No reset with other keys
579584
_raise_if(chunk.row_key)
580-
_raise_if("family_name" in chunk)
581-
_raise_if("qualifier" in chunk)
585+
_raise_if(chunk.HasField("family_name"))
586+
_raise_if(chunk.HasField("qualifier"))
582587
_raise_if(chunk.timestamp_micros)
583588
_raise_if(chunk.labels)
584589
_raise_if(chunk.value_size)

‎packages/google-cloud-bigtable/tests/unit/test_row_data.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -637,15 +637,15 @@ def test_partial_rows_data__copy_from_previous_filled():
637637

638638
def test_partial_rows_data_valid_last_scanned_row_key_on_start():
639639
client = _Client()
640-
response = _ReadRowsResponseV2(chunks=(), last_scanned_row_key="2.AFTER")
640+
response = _ReadRowsResponseV2([], last_scanned_row_key=b"2.AFTER")
641641
iterator = _MockCancellableIterator(response)
642642
client._data_stub = mock.MagicMock()
643643
client._data_stub.read_rows.side_effect = [iterator]
644644
request = object()
645645
yrd = _make_partial_rows_data(client._data_stub.read_rows, request)
646-
yrd.last_scanned_row_key = "1.BEFORE"
646+
yrd.last_scanned_row_key = b"1.BEFORE"
647647
_partial_rows_data_consume_all(yrd)
648-
assert yrd.last_scanned_row_key == "2.AFTER"
648+
assert yrd.last_scanned_row_key == b"2.AFTER"
649649

650650

651651
def test_partial_rows_data_invalid_empty_chunk():
@@ -666,6 +666,7 @@ def test_partial_rows_data_invalid_empty_chunk():
666666

667667
def test_partial_rows_data_state_cell_in_progress():
668668
from google.cloud.bigtable_v2.services.bigtable import BigtableClient
669+
from google.cloud.bigtable_v2.types import bigtable as messages_v2_pb2
669670

670671
LABELS = ["L1", "L2"]
671672

@@ -682,6 +683,9 @@ def test_partial_rows_data_state_cell_in_progress():
682683
value=VALUE,
683684
labels=LABELS,
684685
)
686+
# _update_cell expects to be called after the protoplus wrapper has been
687+
# shucked
688+
chunk = messages_v2_pb2.ReadRowsResponse.CellChunk.pb(chunk)
685689
yrd._update_cell(chunk)
686690

687691
more_cell_data = _ReadRowsResponseCellChunkPB(value=VALUE)
@@ -1455,10 +1459,12 @@ def __init__(self, **kw):
14551459
self.__dict__.update(kw)
14561460

14571461

1458-
class _ReadRowsResponseV2(object):
1459-
def __init__(self, chunks, last_scanned_row_key=""):
1460-
self.chunks = chunks
1461-
self.last_scanned_row_key = last_scanned_row_key
1462+
def _ReadRowsResponseV2(chunks, last_scanned_row_key=b""):
1463+
from google.cloud.bigtable_v2.types import bigtable as messages_v2_pb2
1464+
1465+
return messages_v2_pb2.ReadRowsResponse(
1466+
chunks=chunks, last_scanned_row_key=last_scanned_row_key
1467+
)
14621468

14631469

14641470
def _generate_cell_chunks(chunk_text_pbs):

‎packages/google-cloud-bigtable/tests/unit/test_table.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2206,10 +2206,12 @@ def next(self):
22062206
__next__ = next
22072207

22082208

2209-
class _ReadRowsResponseV2(object):
2210-
def __init__(self, chunks, last_scanned_row_key=""):
2211-
self.chunks = chunks
2212-
self.last_scanned_row_key = last_scanned_row_key
2209+
def _ReadRowsResponseV2(chunks, last_scanned_row_key=b""):
2210+
from google.cloud.bigtable_v2.types import bigtable as messages_v2_pb2
2211+
2212+
return messages_v2_pb2.ReadRowsResponse(
2213+
chunks=chunks, last_scanned_row_key=last_scanned_row_key
2214+
)
22132215

22142216

22152217
def _TablePB(*args, **kw):

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL