FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat(client): add retry handling to GraphQL client · python-gitlab/python-gitlab@8898c38 · GitHub

  1. python-gitlab
  2. python-gitlab

Repository navigation

Commit 8898c38

Browse files
nejch
authored and
max-wittig
committed
feat(client): add retry handling to GraphQL client
1 parent 3235c48 commit 8898c38

6 files changed

Lines changed: 113 additions & 38 deletions

File tree

‎gitlab/client.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
try:
2828
import gql
29+
import gql.transport.exceptions
2930
import graphql
3031
import httpx
3132

@@ -746,7 +747,9 @@ def http_request(
746747
if 200 <= result.status_code < 300:
747748
return result.response
748749

749-
if retry.handle_retry_on_status(result):
750+
if retry.handle_retry_on_status(
751+
result.status_code, result.headers, result.reason
752+
):
750753
continue
751754

752755
error_message = result.content
@@ -1329,4 +1332,28 @@ def execute(
13291332
self, request: Union[str, graphql.Source], *args: Any, **kwargs: Any
13301333
) -> Any:
13311334
parsed_document = self._gql(request)
1332-
return self._client.execute(parsed_document, *args, **kwargs)
1335+
retry = utils.Retry(
1336+
max_retries=3, obey_rate_limit=True, retry_transient_errors=False
1337+
)
1338+
1339+
while True:
1340+
try:
1341+
result = self._client.execute(parsed_document, *args, **kwargs)
1342+
except gql.transport.exceptions.TransportServerError as e:
1343+
if retry.handle_retry_on_status(
1344+
status_code=e.code, headers=self._transport.response_headers
1345+
):
1346+
continue
1347+
1348+
if e.code == 401:
1349+
raise gitlab.exceptions.GitlabAuthenticationError(
1350+
response_code=e.code,
1351+
error_message=str(e),
1352+
)
1353+
1354+
raise gitlab.exceptions.GitlabHttpError(
1355+
response_code=e.code,
1356+
error_message=str(e),
1357+
)
1358+
1359+
return result

‎gitlab/utils.py‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,22 @@
66
import traceback
77
import urllib.parse
88
import warnings
9-
from typing import Any, Callable, Dict, Iterator, Literal, Optional, Tuple, Type, Union
9+
from typing import (
10+
Any,
11+
Callable,
12+
Dict,
13+
Iterator,
14+
Literal,
15+
MutableMapping,
16+
Optional,
17+
Tuple,
18+
Type,
19+
Union,
20+
)
1021

1122
import requests
1223

1324
from gitlab import const, types
14-
from gitlab._backends import requests_backend
1525

1626

1727
class _StdoutStream:
@@ -100,33 +110,40 @@ def __init__(
100110
self.retry_transient_errors = retry_transient_errors
101111

102112
def _retryable_status_code(
103-
self,
104-
result: requests_backend.RequestsResponse,
113+
self, status_code: Optional[int], reason: str = ""
105114
) -> bool:
106-
if result.status_code == 429 and self.obey_rate_limit:
115+
if status_code == 429 and self.obey_rate_limit:
107116
return True
108117

109118
if not self.retry_transient_errors:
110119
return False
111-
if result.status_code in const.RETRYABLE_TRANSIENT_ERROR_CODES:
120+
if status_code in const.RETRYABLE_TRANSIENT_ERROR_CODES:
112121
return True
113-
if result.status_code == 409 and "Resource lock" in result.reason:
122+
if status_code == 409 and "Resource lock" in reason:
114123
return True
115124

116125
return False
117126

118-
def handle_retry_on_status(self, result: requests_backend.RequestsResponse) -> bool:
119-
if not self._retryable_status_code(result):
127+
def handle_retry_on_status(
128+
self,
129+
status_code: Optional[int],
130+
headers: Optional[MutableMapping[str, str]] = None,
131+
reason: str = "",
132+
) -> bool:
133+
if not self._retryable_status_code(status_code, reason):
120134
return False
121135

136+
if headers is None:
137+
headers = {}
138+
122139
# Response headers documentation:
123140
# https://docs.gitlab.com/ee/user/admin_area/settings/user_and_ip_rate_limits.html#response-headers
124141
if self.max_retries == -1 or self.cur_retries < self.max_retries:
125142
wait_time = 2**self.cur_retries * 0.1
126-
if "Retry-After" in result.headers:
127-
wait_time = int(result.headers["Retry-After"])
128-
elif "RateLimit-Reset" in result.headers:
129-
wait_time = int(result.headers["RateLimit-Reset"]) - time.time()
143+
if "Retry-After" in headers:
144+
wait_time = int(headers["Retry-After"])
145+
elif "RateLimit-Reset" in headers:
146+
wait_time = int(headers["RateLimit-Reset"]) - time.time()
130147
self.cur_retries += 1
131148
time.sleep(wait_time)
132149
return True

‎requirements-lint.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ mypy==1.11.2
88
pylint==3.2.7
99
pytest==8.3.2
1010
responses==0.25.3
11+
respx==0.21.1
1112
types-PyYAML==6.0.12.20240808
1213
types-requests==2.32.0.20240907
1314
types-setuptools==74.1.0.20240907

‎requirements-test.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,5 @@ pytest-github-actions-annotate-failures==0.2.0
77
pytest==8.3.2
88
PyYaml==6.0.2
99
responses==0.25.3
10+
respx==0.21.1
1011
wheel==0.44.0

‎tests/unit/test_graphql.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,15 @@
1+
import httpx
12
import pytest
3+
import respx
24

35
import gitlab
46

57

8+
@pytest.fixture
9+
def gl_gql() -> gitlab.GraphQL:
10+
return gitlab.GraphQL("https://gitlab.example.com")
11+
12+
613
def test_import_error_includes_message(monkeypatch: pytest.MonkeyPatch):
714
monkeypatch.setattr(gitlab.client, "_GQL_INSTALLED", False)
815
with pytest.raises(ImportError, match="GraphQL client could not be initialized"):
@@ -12,3 +19,42 @@ def test_import_error_includes_message(monkeypatch: pytest.MonkeyPatch):
1219
def test_graphql_as_context_manager_exits():
1320
with gitlab.GraphQL() as gl:
1421
assert isinstance(gl, gitlab.GraphQL)
22+
23+
24+
def test_graphql_retries_on_429_response(
25+
gl_gql: gitlab.GraphQL, respx_mock: respx.MockRouter
26+
):
27+
url = "https://gitlab.example.com/api/graphql"
28+
responses = [
29+
httpx.Response(429, headers={"retry-after": "1"}),
30+
httpx.Response(
31+
200, json={"data": {"currentUser": {"id": "gid://gitlab/User/1"}}}
32+
),
33+
]
34+
respx_mock.post(url).mock(side_effect=responses)
35+
gl_gql.execute("query {currentUser {id}}")
36+
37+
38+
def test_graphql_raises_when_max_retries_exceeded(
39+
gl_gql: gitlab.GraphQL, respx_mock: respx.MockRouter
40+
):
41+
url = "https://gitlab.example.com/api/graphql"
42+
responses = [
43+
httpx.Response(502),
44+
httpx.Response(502),
45+
httpx.Response(502),
46+
httpx.Response(502),
47+
httpx.Response(502),
48+
]
49+
respx_mock.post(url).mock(side_effect=responses)
50+
with pytest.raises(gitlab.GitlabHttpError):
51+
gl_gql.execute("query {currentUser {id}}")
52+
53+
54+
def test_graphql_raises_on_401_response(
55+
gl_gql: gitlab.GraphQL, respx_mock: respx.MockRouter
56+
):
57+
url = "https://gitlab.example.com/api/graphql"
58+
respx_mock.post(url).mock(return_value=httpx.Response(401))
59+
with pytest.raises(gitlab.GitlabAuthenticationError):
60+
gl_gql.execute("query {currentUser {id}}")

‎tests/unit/test_retry.py‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,24 @@
22
from unittest import mock
33

44
import pytest
5-
import requests
65

76
from gitlab import utils
8-
from gitlab._backends import requests_backend
97

108

119
def test_handle_retry_on_status_ignores_unknown_status_code():
1210
retry = utils.Retry(max_retries=1, retry_transient_errors=True)
13-
response = requests.Response()
14-
response.status_code = 418
15-
backend_response = requests_backend.RequestsResponse(response)
16-
17-
assert retry.handle_retry_on_status(backend_response) is False
11+
assert retry.handle_retry_on_status(418) is False
1812

1913

2014
def test_handle_retry_on_status_accepts_retry_after_header(
2115
monkeypatch: pytest.MonkeyPatch,
2216
):
2317
mock_sleep = mock.Mock()
2418
monkeypatch.setattr(time, "sleep", mock_sleep)
25-
2619
retry = utils.Retry(max_retries=1)
27-
response = requests.Response()
28-
response.status_code = 429
29-
response.headers["Retry-After"] = "1"
30-
backend_response = requests_backend.RequestsResponse(response)
20+
headers = {"Retry-After": "1"}
3121

32-
assert retry.handle_retry_on_status(backend_response) is True
22+
assert retry.handle_retry_on_status(429, headers=headers) is True
3323
assert isinstance(mock_sleep.call_args[0][0], int)
3424

3525

@@ -40,19 +30,12 @@ def test_handle_retry_on_status_accepts_ratelimit_reset_header(
4030
monkeypatch.setattr(time, "sleep", mock_sleep)
4131

4232
retry = utils.Retry(max_retries=1)
43-
response = requests.Response()
44-
response.status_code = 429
45-
response.headers["RateLimit-Reset"] = str(int(time.time() + 1))
46-
backend_response = requests_backend.RequestsResponse(response)
33+
headers = {"RateLimit-Reset": str(int(time.time() + 1))}
4734

48-
assert retry.handle_retry_on_status(backend_response) is True
35+
assert retry.handle_retry_on_status(429, headers=headers) is True
4936
assert isinstance(mock_sleep.call_args[0][0], float)
5037

5138

5239
def test_handle_retry_on_status_returns_false_when_max_retries_reached():
5340
retry = utils.Retry(max_retries=0)
54-
response = requests.Response()
55-
response.status_code = 429
56-
backend_response = requests_backend.RequestsResponse(response)
57-
58-
assert retry.handle_retry_on_status(backend_response) is False
41+
assert retry.handle_retry_on_status(429) is False

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL