| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent de108bc commit d28bafa
4 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -147,6 +147,10 @@ or on combining URL components into a URL string. | |||
| 147 | 147 | ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', | |
| 148 | 148 | params='', query='', fragment='') | |
| 149 | 149 | ||
| 150 | + .. warning:: | ||
| 151 | + | ||
| 152 | + :func:`urlparse` does not perform validation. See :ref:`URL parsing | ||
| 153 | + security <url-parsing-security>` for details. | ||
| 150 | 154 | ||
| 151 | 155 | .. versionchanged:: 3.2 | |
| 152 | 156 | Added IPv6 URL parsing capabilities. | |
@@ -311,8 +315,14 @@ or on combining URL components into a URL string. | |||
| 311 | 315 | ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is | |
| 312 | 316 | decomposed before parsing, no error will be raised. | |
| 313 | 317 | ||
| 314 | - Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline | ||
| 315 | - ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. | ||
| 318 | + Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 | ||
| 319 | + control and space characters are stripped from the URL. ``\n``, | ||
| 320 | + ``\r`` and tab ``\t`` characters are removed from the URL at any position. | ||
| 321 | + | ||
| 322 | + .. warning:: | ||
| 323 | + | ||
| 324 | + :func:`urlsplit` does not perform validation. See :ref:`URL parsing | ||
| 325 | + security <url-parsing-security>` for details. | ||
| 316 | 326 | ||
| 317 | 327 | .. versionchanged:: 3.6 | |
| 318 | 328 | Out-of-range port numbers now raise :exc:`ValueError`, instead of | |
@@ -325,6 +335,9 @@ or on combining URL components into a URL string. | |||
| 325 | 335 | .. versionchanged:: 3.7.11 | |
| 326 | 336 | ASCII newline and tab characters are stripped from the URL. | |
| 327 | 337 | ||
| 338 | + .. versionchanged:: 3.7.17 | ||
| 339 | + Leading WHATWG C0 control and space characters are stripped from the URL. | ||
| 340 | + | ||
| 328 | 341 | .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser | |
| 329 | 342 | ||
| 330 | 343 | .. function:: urlunsplit(parts) | |
@@ -394,6 +407,27 @@ or on combining URL components into a URL string. | |||
| 394 | 407 | .. versionchanged:: 3.2 | |
| 395 | 408 | Result is a structured object rather than a simple 2-tuple. | |
| 396 | 409 | ||
| 410 | + .. _url-parsing-security: | ||
| 411 | + | ||
| 412 | + URL parsing security | ||
| 413 | + -------------------- | ||
| 414 | + | ||
| 415 | + The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of | ||
| 416 | + inputs. They may not raise errors on inputs that other applications consider | ||
| 417 | + invalid. They may also succeed on some inputs that might not be considered | ||
| 418 | + URLs elsewhere. Their purpose is for practical functionality rather than | ||
| 419 | + purity. | ||
| 420 | + | ||
| 421 | + Instead of raising an exception on unusual input, they may instead return some | ||
| 422 | + component parts as empty strings. Or components may contain more than perhaps | ||
| 423 | + they should. | ||
| 424 | + | ||
| 425 | + We recommend that users of these APIs where the values may be used anywhere | ||
| 426 | + with security implications code defensively. Do some verification within your | ||
| 427 | + code before trusting a returned component part. Does that ``scheme`` make | ||
| 428 | + sense? Is that a sensible ``path``? Is there anything strange about that | ||
| 429 | + ``hostname``? etc. | ||
| 430 | + | ||
| 397 | 431 | .. _parsing-ascii-encoded-bytes: | |
| 398 | 432 | ||
| 399 | 433 | Parsing ASCII Encoded Bytes | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -660,14 +660,73 @@ def test_urlsplit_remove_unsafe_bytes(self): | |||
| 660 | 660 | self.assertEqual(p.scheme, "https") | |
| 661 | 661 | self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment") | |
| 662 | 662 | ||
| 663 | + def test_urlsplit_strip_url(self): | ||
| 664 | + noise = bytes(range(0, 0x20 + 1)) | ||
| 665 | + base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" | ||
| 666 | + | ||
| 667 | + url = noise.decode("utf-8") + base_url | ||
| 668 | + p = urllib.parse.urlsplit(url) | ||
| 669 | + self.assertEqual(p.scheme, "http") | ||
| 670 | + self.assertEqual(p.netloc, "User:Pass@www.python.org:080") | ||
| 671 | + self.assertEqual(p.path, "/doc/") | ||
| 672 | + self.assertEqual(p.query, "query=yes") | ||
| 673 | + self.assertEqual(p.fragment, "frag") | ||
| 674 | + self.assertEqual(p.username, "User") | ||
| 675 | + self.assertEqual(p.password, "Pass") | ||
| 676 | + self.assertEqual(p.hostname, "www.python.org") | ||
| 677 | + self.assertEqual(p.port, 80) | ||
| 678 | + self.assertEqual(p.geturl(), base_url) | ||
| 679 | + | ||
| 680 | + url = noise + base_url.encode("utf-8") | ||
| 681 | + p = urllib.parse.urlsplit(url) | ||
| 682 | + self.assertEqual(p.scheme, b"http") | ||
| 683 | + self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") | ||
| 684 | + self.assertEqual(p.path, b"/doc/") | ||
| 685 | + self.assertEqual(p.query, b"query=yes") | ||
| 686 | + self.assertEqual(p.fragment, b"frag") | ||
| 687 | + self.assertEqual(p.username, b"User") | ||
| 688 | + self.assertEqual(p.password, b"Pass") | ||
| 689 | + self.assertEqual(p.hostname, b"www.python.org") | ||
| 690 | + self.assertEqual(p.port, 80) | ||
| 691 | + self.assertEqual(p.geturl(), base_url.encode("utf-8")) | ||
| 692 | + | ||
| 693 | + # Test that trailing space is preserved as some applications rely on | ||
| 694 | + # this within query strings. | ||
| 695 | + query_spaces_url = "https://www.python.org:88/doc/?query= " | ||
| 696 | + p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url) | ||
| 697 | + self.assertEqual(p.scheme, "https") | ||
| 698 | + self.assertEqual(p.netloc, "www.python.org:88") | ||
| 699 | + self.assertEqual(p.path, "/doc/") | ||
| 700 | + self.assertEqual(p.query, "query= ") | ||
| 701 | + self.assertEqual(p.port, 88) | ||
| 702 | + self.assertEqual(p.geturl(), query_spaces_url) | ||
| 703 | + | ||
| 704 | + p = urllib.parse.urlsplit("www.pypi.org ") | ||
| 705 | + # That "hostname" gets considered a "path" due to the | ||
| 706 | + # trailing space and our existing logic... YUCK... | ||
| 707 | + # and re-assembles via geturl aka unurlsplit into the original. | ||
| 708 | + # django.core.validators.URLValidator (at least through v3.2) relies on | ||
| 709 | + # this, for better or worse, to catch it in a ValidationError via its | ||
| 710 | + # regular expressions. | ||
| 711 | + # Here we test the basic round trip concept of such a trailing space. | ||
| 712 | + self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ") | ||
| 713 | + | ||
| 714 | + # with scheme as cache-key | ||
| 715 | + url = "//www.python.org/" | ||
| 716 | + scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8") | ||
| 717 | + for _ in range(2): | ||
| 718 | + p = urllib.parse.urlsplit(url, scheme=scheme) | ||
| 719 | + self.assertEqual(p.scheme, "https") | ||
| 720 | + self.assertEqual(p.geturl(), "https://www.python.org/") | ||
| 721 | + | ||
| 663 | 722 | def test_attributes_bad_port(self): | |
| 664 | 723 | """Check handling of invalid ports.""" | |
| 665 | 724 | for bytes in (False, True): | |
| 666 | 725 | for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): | |
| 667 | 726 | for port in ("foo", "1.5", "-1", "0x10"): | |
| 668 | 727 | with self.subTest(bytes=bytes, parse=parse, port=port): | |
| 669 | 728 | netloc = "www.example.net:" + port | |
| 670 | - url = "http://" + netloc | ||
| 729 | + url = "http://" + netloc + "/" | ||
| 671 | 730 | if bytes: | |
| 672 | 731 | netloc = netloc.encode("ascii") | |
| 673 | 732 | url = url.encode("ascii") | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -25,6 +25,10 @@ | |||
| 25 | 25 | scenarios for parsing, and for backward compatibility purposes, some | |
| 26 | 26 | parsing quirks from older RFCs are retained. The testcases in | |
| 27 | 27 | test_urlparse.py provides a good indicator of parsing behavior. | |
| 28 | + | ||
| 29 | + The WHATWG URL Parser spec should also be considered. We are not compliant with | ||
| 30 | + it either due to existing user code API behavior expectations (Hyrum's Law). | ||
| 31 | + It serves as a useful guide when making changes. | ||
| 28 | 32 | """ | |
| 29 | 33 | ||
| 30 | 34 | import re | |
@@ -76,6 +80,10 @@ | |||
| 76 | 80 | '0123456789' | |
| 77 | 81 | '+-.') | |
| 78 | 82 | ||
| 83 | + # Leading and trailing C0 control and space to be stripped per WHATWG spec. | ||
| 84 | + # == "".join([chr(i) for i in range(0, 0x20 + 1)]) | ||
| 85 | + _WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' | ||
| 86 | + | ||
| 79 | 87 | # Unsafe bytes to be removed per WHATWG spec | |
| 80 | 88 | _UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] | |
| 81 | 89 | ||
@@ -426,6 +434,10 @@ def urlsplit(url, scheme='', allow_fragments=True): | |||
| 426 | 434 | url, scheme, _coerce_result = _coerce_args(url, scheme) | |
| 427 | 435 | url = _remove_unsafe_bytes_from_url(url) | |
| 428 | 436 | scheme = _remove_unsafe_bytes_from_url(scheme) | |
| 437 | + # Only lstrip url as some applications rely on preserving trailing space. | ||
| 438 | + # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) | ||
| 439 | + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) | ||
| 440 | + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) | ||
| 429 | 441 | allow_fragments = bool(allow_fragments) | |
| 430 | 442 | key = url, scheme, allow_fragments, type(url), type(scheme) | |
| 431 | 443 | cached = _parse_cache.get(key, None) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,3 @@ | |||
| 1 | + :func:`urllib.parse.urlsplit` now strips leading C0 control and space | ||
| 2 | + characters following the specification for URLs defined by WHATWG in | ||
| 3 | + response to CVE-2023-24329. Patch by Illia Volochii. | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments