FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test: move the pydantic check inside test by JoanFM · Pull Request #1812 · docarray/docarray · GitHub

/ docarray Public

test: move the pydantic check inside test #1812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JoanFM merged 3 commits into from
Sep 28, 2023
Conversation 2 Commits 3 Checks 0 Files changed
Merged

test: move the pydantic check inside test #1812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
78df9bb
test: move the pydantic check inside test
Sep 28, 2023
cf38ec0
Merge branch 'main' into pull-stream-file
Sep 28, 2023
f4f56c3
Merge branch 'main' into pull-stream-file
Sep 28, 2023
File filter

Filter by extension

Filter by extension .py  (2) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
54 changes: 29 additions & 25 deletions docarray/store/helpers.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -174,32 +174,36 @@ def _from_binary_stream(
compress: Optional[str] = None,
show_progress: bool = False,
) -> Iterator['T']:
if show_progress:
pbar, t = _get_progressbar(
'Deserializing', disable=not show_progress, total=total
)
else:
pbar = nullcontext()

with pbar:
try:
if show_progress:
_total_size = 0
pbar.start_task(t)
while True:
len_bytes = stream.read(4)
if len(len_bytes) < 4:
raise ValueError('Unexpected end of stream')
len_item = int.from_bytes(len_bytes, 'big', signed=False)
if len_item == 0:
break
item_bytes = stream.read(len_item)
if len(item_bytes) < len_item:
raise ValueError('Unexpected end of stream')
item = cls.from_bytes(item_bytes, protocol=protocol, compress=compress)

yield item
pbar, t = _get_progressbar(
'Deserializing', disable=not show_progress, total=total
)
else:
pbar = nullcontext()

with pbar:
if show_progress:
_total_size += len_item + 4
pbar.update(t, advance=1, total_size=str(filesize.decimal(_total_size)))
_total_size = 0
pbar.start_task(t)
while True:
len_bytes = stream.read(4)
if len(len_bytes) < 4:
raise ValueError('Unexpected end of stream')
len_item = int.from_bytes(len_bytes, 'big', signed=False)
if len_item == 0:
break
item_bytes = stream.read(len_item)
if len(item_bytes) < len_item:
raise ValueError('Unexpected end of stream')
item = cls.from_bytes(item_bytes, protocol=protocol, compress=compress)

yield item

if show_progress:
_total_size += len_item + 4
pbar.update(
t, advance=1, total_size=str(filesize.decimal(_total_size))
)
finally:
stream.close()
28 changes: 15 additions & 13 deletions tests/integrations/store/test_file.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
from docarray import DocList
from docarray.documents import TextDoc
from docarray.store.file import ConcurrentPushException, FileDocStore
from docarray.utils._internal.cache import _get_cache_path
from docarray.utils._internal.pydantic import is_pydantic_v2
from docarray.utils._internal.cache import _get_cache_path
from tests.integrations.store import gen_text_docs, get_test_da, profile_memory

DA_LEN: int = 2**10
TOLERANCE_RATIO = 0.1 # Percentage of difference allowed in stream vs non-stream test
TOLERANCE_RATIO = 0.1 # Percentage of difference allowed when streaming between a long and a shorter DA


def test_path_resolution():
Expand All @@ -23,7 +23,6 @@ def test_path_resolution():


def test_pushpull_correct(capsys, tmp_path: Path):
tmp_path.mkdir(parents=True, exist_ok=True)
namespace_dir = tmp_path
da1 = get_test_da(DA_LEN)

Expand Down Expand Up @@ -51,7 +50,6 @@ def test_pushpull_correct(capsys, tmp_path: Path):


def test_pushpull_stream_correct(capsys, tmp_path: Path):
tmp_path.mkdir(parents=True, exist_ok=True)
namespace_dir = tmp_path
da1 = get_test_da(DA_LEN)

Expand Down Expand Up @@ -85,10 +83,8 @@ def test_pushpull_stream_correct(capsys, tmp_path: Path):


# for some reason this test is failing with pydantic v2
@pytest.mark.skipif(is_pydantic_v2, reason="Not working with pydantic v2 for now")
@pytest.mark.slow
def test_pull_stream_vs_pull_full(tmp_path: Path):
tmp_path.mkdir(parents=True, exist_ok=True)
namespace_dir = tmp_path
DocList[TextDoc].push_stream(
gen_text_docs(DA_LEN * 1),
Expand Down Expand Up @@ -136,15 +132,23 @@ def get_total_full(url: str):
), 'Streamed and non-streamed pull should have similar statistics'

assert (
abs(long_stream_peak - short_stream_peak) / short_stream_peak < TOLERANCE_RATIO
), 'Streamed memory usage should not be dependent on the size of the data'
long_full_peak > long_stream_peak
), 'Peak of memory using full should be larger than when streaming'
assert (
abs(long_full_peak - short_full_peak) / short_full_peak > TOLERANCE_RATIO
), 'Full pull memory usage should be dependent on the size of the data'
short_full_peak > short_stream_peak
), 'Peak of memory using full should be larger than when streaming'
if not is_pydantic_v2:
# I bet there is some memory that Pydantic is leaking
assert (
abs(long_stream_peak - short_stream_peak) / short_stream_peak
< TOLERANCE_RATIO
), 'Streamed memory usage should not be dependent on the size of the data'
assert (
abs(long_full_peak - short_full_peak) / short_full_peak > TOLERANCE_RATIO
), 'Full pull memory usage should be dependent on the size of the data'


def test_list_and_delete(tmp_path: Path):
tmp_path.mkdir(parents=True, exist_ok=True)
namespace_dir = str(tmp_path)

da_names = FileDocStore.list(namespace_dir, show_table=False)
Expand Down Expand Up @@ -179,7 +183,6 @@ def test_list_and_delete(tmp_path: Path):

def test_concurrent_push_pull(tmp_path: Path):
# Push to DA that is being pulled should not mess up the pull
tmp_path.mkdir(parents=True, exist_ok=True)
namespace_dir = tmp_path

DocList[TextDoc].push_stream(
Expand Down Expand Up @@ -214,7 +217,6 @@ def test_concurrent_push(tmp_path: Path):
# Double push should fail the second push
import time

tmp_path.mkdir(parents=True, exist_ok=True)
namespace_dir = tmp_path

DocList[TextDoc].push_stream(
Expand Down
Back | FazBrowse Home | New Git URL