| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -24,6 +24,7 @@ | |||
| 24 | 24 | import requests.utils | |
| 25 | 25 | from requests_toolbelt.multipart.encoder import MultipartEncoder # type: ignore | |
| 26 | 26 | ||
| 27 | + import gitlab | ||
| 27 | 28 | import gitlab.config | |
| 28 | 29 | import gitlab.const | |
| 29 | 30 | import gitlab.exceptions | |
@@ -37,6 +38,12 @@ | |||
| 37 | 38 | ||
| 38 | 39 | RETRYABLE_TRANSIENT_ERROR_CODES = [500, 502, 503, 504] + list(range(520, 531)) | |
| 39 | 40 | ||
| 41 | + # https://docs.gitlab.com/ee/api/#offset-based-pagination | ||
| 42 | + _PAGINATION_URL = ( | ||
| 43 | + f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/" | ||
| 44 | + f"api-usage.html#pagination" | ||
| 45 | + ) | ||
| 46 | + | ||
| 40 | 47 | ||
| 41 | 48 | class Gitlab: | |
| 42 | 49 | """Represents a GitLab server connection. | |
@@ -826,20 +833,59 @@ def http_list( | |||
| 826 | 833 | # In case we want to change the default behavior at some point | |
| 827 | 834 | as_list = True if as_list is None else as_list | |
| 828 | 835 | ||
| 829 | - get_all = kwargs.pop("all", False) | ||
| 836 | + get_all = kwargs.pop("all", None) | ||
| 830 | 837 | url = self._build_url(path) | |
| 831 | 838 | ||
| 832 | 839 | page = kwargs.get("page") | |
| 833 | 840 | ||
| 834 | - if get_all is True and as_list is True: | ||
| 835 | - return list(GitlabList(self, url, query_data, **kwargs)) | ||
| 841 | + if as_list is False: | ||
| 842 | + # Generator requested | ||
| 843 | + return GitlabList(self, url, query_data, **kwargs) | ||
| 836 | 844 | ||
| 837 | - if page or as_list is True: | ||
| 838 | - # pagination requested, we return a list | ||
| 839 | - return list(GitlabList(self, url, query_data, get_next=False, **kwargs)) | ||
| 845 | + if get_all is True: | ||
| 846 | + return list(GitlabList(self, url, query_data, **kwargs)) | ||
| 840 | 847 | ||
| 841 | - # No pagination, generator requested | ||
| 842 | - return GitlabList(self, url, query_data, **kwargs) | ||
| 848 | + # pagination requested, we return a list | ||
| 849 | + gl_list = GitlabList(self, url, query_data, get_next=False, **kwargs) | ||
| 850 | + items = list(gl_list) | ||
| 851 | + | ||
| 852 | + def should_emit_warning() -> bool: | ||
| 853 | + # No warning is emitted if any of the following conditions apply: | ||
| 854 | + # * `all=False` was set in the `list()` call. | ||
| 855 | + # * `page` was set in the `list()` call. | ||
| 856 | + # * GitLab did not return the `x-per-page` header. | ||
| 857 | + # * Number of items received is less than per-page value. | ||
| 858 | + # * Number of items received is >= total available. | ||
| 859 | + if get_all is False: | ||
| 860 | + return False | ||
| 861 | + if page is not None: | ||
| 862 | + return False | ||
| 863 | + if gl_list.per_page is None: | ||
| 864 | + return False | ||
| 865 | + if len(items) < gl_list.per_page: | ||
| 866 | + return False | ||
| 867 | + if gl_list.total is not None and len(items) >= gl_list.total: | ||
| 868 | + return False | ||
| 869 | + return True | ||
| 870 | + | ||
| 871 | + if not should_emit_warning(): | ||
| 872 | + return items | ||
| 873 | + | ||
| 874 | + # Warn the user that they are only going to retrieve `per_page` | ||
| 875 | + # maximum items. This is a common cause of issues filed. | ||
| 876 | + total_items = "many" if gl_list.total is None else gl_list.total | ||
| 877 | + utils.warn( | ||
| 878 | + message=( | ||
| 879 | + f"Calling a `list()` method without specifying `all=True` or " | ||
| 880 | + f"`as_list=False` will return a maximum of {gl_list.per_page} items. " | ||
| 881 | + f"Your query returned {len(items)} of {total_items} items. See " | ||
| 882 | + f"{_PAGINATION_URL} for more details. If this was done intentionally, " | ||
| 883 | + f"then this warning can be supressed by adding the argument " | ||
| 884 | + f"`all=False` to the `list()` call." | ||
| 885 | + ), | ||
| 886 | + category=UserWarning, | ||
| 887 | + ) | ||
| 888 | + return items | ||
| 843 | 889 | ||
| 844 | 890 | def http_post( | |
| 845 | 891 | self, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,3 +1,5 @@ | |||
| 1 | + import warnings | ||
| 2 | + | ||
| 1 | 3 | import pytest | |
| 2 | 4 | ||
| 3 | 5 | import gitlab | |
@@ -181,3 +183,46 @@ def test_rate_limits(gl): | |||
| 181 | 183 | settings.throttle_authenticated_api_enabled = False | |
| 182 | 184 | settings.save() | |
| 183 | 185 | [project.delete() for project in projects] | |
| 186 | + | ||
| 187 | + | ||
| 188 | + def test_list_default_warning(gl): | ||
| 189 | + """When there are more than 20 items and use default `list()` then warning is | ||
| 190 | + generated""" | ||
| 191 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 192 | + gl.gitlabciymls.list() | ||
| 193 | + assert len(caught_warnings) == 1 | ||
| 194 | + warning = caught_warnings[0] | ||
| 195 | + assert isinstance(warning.message, UserWarning) | ||
| 196 | + message = str(warning.message) | ||
| 197 | + assert "python-gitlab.readthedocs.io" in message | ||
| 198 | + assert __file__ == warning.filename | ||
| 199 | + | ||
| 200 | + | ||
| 201 | + def test_list_page_nowarning(gl): | ||
| 202 | + """Using `page=X` will disable the warning""" | ||
| 203 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 204 | + gl.gitlabciymls.list(page=1) | ||
| 205 | + assert len(caught_warnings) == 0 | ||
| 206 | + | ||
| 207 | + | ||
| 208 | + def test_list_all_false_nowarning(gl): | ||
| 209 | + """Using `all=False` will disable the warning""" | ||
| 210 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 211 | + gl.gitlabciymls.list(all=False) | ||
| 212 | + assert len(caught_warnings) == 0 | ||
| 213 | + | ||
| 214 | + | ||
| 215 | + def test_list_all_true_nowarning(gl): | ||
| 216 | + """Using `all=True` will disable the warning""" | ||
| 217 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 218 | + items = gl.gitlabciymls.list(all=True) | ||
| 219 | + assert len(caught_warnings) == 0 | ||
| 220 | + assert len(items) > 20 | ||
| 221 | + | ||
| 222 | + | ||
| 223 | + def test_list_as_list_false_nowarning(gl): | ||
| 224 | + """Using `as_list=False` will disable the warning""" | ||
| 225 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 226 | + items = gl.gitlabciymls.list(as_list=False) | ||
| 227 | + assert len(caught_warnings) == 0 | ||
| 228 | + assert len(list(items)) > 20 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,3 +1,6 @@ | |||
| 1 | + import copy | ||
| 2 | + import warnings | ||
| 3 | + | ||
| 1 | 4 | import pytest | |
| 2 | 5 | import requests | |
| 3 | 6 | import responses | |
@@ -425,20 +428,115 @@ def test_list_request(gl): | |||
| 425 | 428 | match=MATCH_EMPTY_QUERY_PARAMS, | |
| 426 | 429 | ) | |
| 427 | 430 | ||
| 428 | - result = gl.http_list("/projects", as_list=True) | ||
| 431 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 432 | + result = gl.http_list("/projects", as_list=True) | ||
| 433 | + assert len(caught_warnings) == 0 | ||
| 429 | 434 | assert isinstance(result, list) | |
| 430 | 435 | assert len(result) == 1 | |
| 431 | 436 | ||
| 432 | 437 | result = gl.http_list("/projects", as_list=False) | |
| 433 | 438 | assert isinstance(result, GitlabList) | |
| 434 | - assert len(result) == 1 | ||
| 439 | + assert len(list(result)) == 1 | ||
| 435 | 440 | ||
| 436 | 441 | result = gl.http_list("/projects", all=True) | |
| 437 | 442 | assert isinstance(result, list) | |
| 438 | 443 | assert len(result) == 1 | |
| 439 | 444 | assert responses.assert_call_count(url, 3) is True | |
| 440 | 445 | ||
| 441 | 446 | ||
| 447 | + large_list_response = { | ||
| 448 | + "method": responses.GET, | ||
| 449 | + "url": "http://localhost/api/v4/projects", | ||
| 450 | + "json": [ | ||
| 451 | + {"name": "project01"}, | ||
| 452 | + {"name": "project02"}, | ||
| 453 | + {"name": "project03"}, | ||
| 454 | + {"name": "project04"}, | ||
| 455 | + {"name": "project05"}, | ||
| 456 | + {"name": "project06"}, | ||
| 457 | + {"name": "project07"}, | ||
| 458 | + {"name": "project08"}, | ||
| 459 | + {"name": "project09"}, | ||
| 460 | + {"name": "project10"}, | ||
| 461 | + {"name": "project11"}, | ||
| 462 | + {"name": "project12"}, | ||
| 463 | + {"name": "project13"}, | ||
| 464 | + {"name": "project14"}, | ||
| 465 | + {"name": "project15"}, | ||
| 466 | + {"name": "project16"}, | ||
| 467 | + {"name": "project17"}, | ||
| 468 | + {"name": "project18"}, | ||
| 469 | + {"name": "project19"}, | ||
| 470 | + {"name": "project20"}, | ||
| 471 | + ], | ||
| 472 | + "headers": {"X-Total": "30", "x-per-page": "20"}, | ||
| 473 | + "status": 200, | ||
| 474 | + "match": MATCH_EMPTY_QUERY_PARAMS, | ||
| 475 | + } | ||
| 476 | + | ||
| 477 | + | ||
| 478 | + @responses.activate | ||
| 479 | + def test_list_request_pagination_warning(gl): | ||
| 480 | + responses.add(**large_list_response) | ||
| 481 | + | ||
| 482 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 483 | + result = gl.http_list("/projects", as_list=True) | ||
| 484 | + assert len(caught_warnings) == 1 | ||
| 485 | + warning = caught_warnings[0] | ||
| 486 | + assert isinstance(warning.message, UserWarning) | ||
| 487 | + message = str(warning.message) | ||
| 488 | + assert "Calling a `list()` method" in message | ||
| 489 | + assert "python-gitlab.readthedocs.io" in message | ||
| 490 | + assert __file__ == warning.filename | ||
| 491 | + assert isinstance(result, list) | ||
| 492 | + assert len(result) == 20 | ||
| 493 | + assert len(responses.calls) == 1 | ||
| 494 | + | ||
| 495 | + | ||
| 496 | + @responses.activate | ||
| 497 | + def test_list_request_as_list_false_nowarning(gl): | ||
| 498 | + responses.add(**large_list_response) | ||
| 499 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 500 | + result = gl.http_list("/projects", as_list=False) | ||
| 501 | + assert len(caught_warnings) == 0 | ||
| 502 | + assert isinstance(result, GitlabList) | ||
| 503 | + assert len(list(result)) == 20 | ||
| 504 | + assert len(responses.calls) == 1 | ||
| 505 | + | ||
| 506 | + | ||
| 507 | + @responses.activate | ||
| 508 | + def test_list_request_all_true_nowarning(gl): | ||
| 509 | + responses.add(**large_list_response) | ||
| 510 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 511 | + result = gl.http_list("/projects", all=True) | ||
| 512 | + assert len(caught_warnings) == 0 | ||
| 513 | + assert isinstance(result, list) | ||
| 514 | + assert len(result) == 20 | ||
| 515 | + assert len(responses.calls) == 1 | ||
| 516 | + | ||
| 517 | + | ||
| 518 | + @responses.activate | ||
| 519 | + def test_list_request_all_false_nowarning(gl): | ||
| 520 | + responses.add(**large_list_response) | ||
| 521 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 522 | + result = gl.http_list("/projects", all=False) | ||
| 523 | + assert len(caught_warnings) == 0 | ||
| 524 | + assert isinstance(result, list) | ||
| 525 | + assert len(result) == 20 | ||
| 526 | + assert len(responses.calls) == 1 | ||
| 527 | + | ||
| 528 | + | ||
| 529 | + @responses.activate | ||
| 530 | + def test_list_request_page_nowarning(gl): | ||
| 531 | + response_dict = copy.deepcopy(large_list_response) | ||
| 532 | + response_dict["match"] = [responses.matchers.query_param_matcher({"page": "1"})] | ||
| 533 | + responses.add(**response_dict) | ||
| 534 | + with warnings.catch_warnings(record=True) as caught_warnings: | ||
| 535 | + gl.http_list("/projects", page=1) | ||
| 536 | + assert len(caught_warnings) == 0 | ||
| 537 | + assert len(responses.calls) == 1 | ||
| 538 | + | ||
| 539 | + | ||
| 442 | 540 | @responses.activate | |
| 443 | 541 | def test_list_request_404(gl): | |
| 444 | 542 | url = "http://localhost/api/v4/not_there" | |
| Back | FazBrowse Home | New Git URL |
0 commit comments