Repository navigation
Commit 7a3724f
committed
fix: handle empty 'Retry-After' header from GitLab
When requests are throttled (HTTP response code 429), python-gitlab
assumed that 'Retry-After' existed in the response headers. This is
not always the case and so the request fails due to a KeyError. The
change in this commit adds a rudimentary exponential backoff to the
'http_request' method, which defaults to 10 retries but can be set
to -1 to retry without bound.1 parent ce2c835 commit 7a3724f
2 files changed
Lines changed: 26 additions & 4 deletions
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -299,7 +299,9 @@ Rate limits | |||
| 299 | 299 | ||
| 300 | 300 | python-gitlab obeys the rate limit of the GitLab server by default. On | |
| 301 | 301 | receiving a 429 response (Too Many Requests), python-gitlab sleeps for the | |
| 302 | - amount of time in the Retry-After header that GitLab sends back. | ||
| 302 | + amount of time in the Retry-After header that GitLab sends back. If GitLab | ||
| 303 | + does not return a response with the Retry-After header, python-gitlab will | ||
| 304 | + perform an exponential backoff. | ||
| 303 | 305 | ||
| 304 | 306 | If you don't want to wait, you can disable the rate-limiting feature, by | |
| 305 | 307 | supplying the ``obey_rate_limit`` argument. | |
@@ -312,6 +314,18 @@ supplying the ``obey_rate_limit`` argument. | |||
| 312 | 314 | gl = gitlab.gitlab(url, token, api_version=4) | |
| 313 | 315 | gl.projects.list(all=True, obey_rate_limit=False) | |
| 314 | 316 | ||
| 317 | + If you do not disable the rate-limiting feature, you can supply a custom value | ||
| 318 | + for ``max_retries``; by default, this is set to 10. To retry without bound when | ||
| 319 | + throttled, you can set this parameter to -1. This parameter is ignored if | ||
| 320 | + ``obey_rate_limit`` is set to ``False``. | ||
| 321 | + | ||
| 322 | + .. code-block:: python | ||
| 323 | + | ||
| 324 | + import gitlab | ||
| 325 | + import requests | ||
| 326 | + | ||
| 327 | + gl = gitlab.gitlab(url, token, api_version=4) | ||
| 328 | + gl.projects.list(all=True, max_retries=12) | ||
| 315 | 329 | ||
| 316 | 330 | .. warning:: | |
| 317 | 331 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -477,6 +477,10 @@ def http_request(self, verb, path, query_data={}, post_data=None, | |||
| 477 | 477 | # obey the rate limit by default | |
| 478 | 478 | obey_rate_limit = kwargs.get("obey_rate_limit", True) | |
| 479 | 479 | ||
| 480 | + # set max_retries to 10 by default, disable by setting it to -1 | ||
| 481 | + max_retries = kwargs.get("max_retries", 10) | ||
| 482 | + cur_retries = 0 | ||
| 483 | + | ||
| 480 | 484 | while True: | |
| 481 | 485 | result = self.session.send(prepped, timeout=timeout, **settings) | |
| 482 | 486 | ||
@@ -486,9 +490,13 @@ def http_request(self, verb, path, query_data={}, post_data=None, | |||
| 486 | 490 | return result | |
| 487 | 491 | ||
| 488 | 492 | if 429 == result.status_code and obey_rate_limit: | |
| 489 | - wait_time = int(result.headers["Retry-After"]) | ||
| 490 | - time.sleep(wait_time) | ||
| 491 | - continue | ||
| 493 | + if max_retries == -1 or cur_retries < max_retries: | ||
| 494 | + wait_time = 2 ** cur_retries * 0.1 | ||
| 495 | + if "Retry-After" in result.headers: | ||
| 496 | + wait_time = int(result.headers["Retry-After"]) | ||
| 497 | + cur_retries += 1 | ||
| 498 | + time.sleep(wait_time) | ||
| 499 | + continue | ||
| 492 | 500 | ||
| 493 | 501 | error_message = result.content | |
| 494 | 502 | try: | |
0 commit comments