Commit b145459
committed
fix(commit): parse the encoding header when deserializing
`Commit._deserialize` compared a 10-byte slice of each stripped header
line against the 9-byte literal `b"encoding "`, so the comparison never
matched and the branch reading a commit's declared encoding was
unreachable. Every commit parsed from the object database was decoded
with the default UTF-8: `Commit.encoding` reported `UTF-8` regardless of
the header, and the author, committer, and message of a commit written
under `i18n.commitencoding` came back with U+FFFD replacement characters
while `git log` shows them correctly. Re-serializing such a commit also
dropped its `encoding` header and stored the garbled text.
Compare the right slice length so the header is read. Adopt the declared
encoding only when Python has a text codec for it: the identity lines
are decoded with `errors="replace"`, which does not guard against an
unknown codec name, so a commit declaring one would otherwise raise
`LookupError` on attribute access. An unknown name keeps the previous
behavior (the default encoding) and logs the error.
Validated against a repository committed under `i18n.commitencoding =
ISO-8859-1`: `commit.encoding`, `commit.author.name`, and
`commit.message` now match what `git log` shows. Adds regression tests
in `test/test_commit.py` that fail before the change.1 parent 6eedd2c commit b145459
2 files changed
Lines changed: 46 additions & 2 deletions
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -896,8 +896,17 @@ def _deserialize(self, stream: BytesIO) -> "Commit": | |||
| 896 | 896 | enc = next_line | |
| 897 | 897 | buf = enc.strip() | |
| 898 | 898 | while buf: | |
| 899 | - if buf[0:10] == b"encoding ": | ||
| 900 | - self.encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") | ||
| 899 | + if buf[0:9] == b"encoding ": | ||
| 900 | + encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") | ||
| 901 | + try: | ||
| 902 | + # Probe one byte: decoding b"" skips the codec lookup entirely. | ||
| 903 | + b" ".decode(encoding, "replace") | ||
| 904 | + except (LookupError, ValueError): | ||
| 905 | + # The encoding is not a text encoding Python knows; keep the | ||
| 906 | + # default so the identity and message lines still decode below. | ||
| 907 | + _logger.error("Ignoring unknown commit encoding %r", encoding) | ||
| 908 | + else: | ||
| 909 | + self.encoding = encoding | ||
| 901 | 910 | elif buf[0:7] == b"gpgsig ": | |
| 902 | 911 | sig_lines = [buf[buf.find(b" ") + 1 :] + b"\n"] | |
| 903 | 912 | is_next_header = False | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -458,6 +458,41 @@ def test_invalid_commit(self): | |||
| 458 | 458 | self.assertEqual(cmt.author.name, "E.Azer Ko�o�o�oculu", cmt.author.name) | |
| 459 | 459 | self.assertEqual(cmt.author.email, "azer@kodfabrik.com", cmt.author.email) | |
| 460 | 460 | ||
| 461 | + def test_deserialization_declared_encoding(self): | ||
| 462 | + # The "encoding" header names the encoding of the identity and message bytes. | ||
| 463 | + data = ( | ||
| 464 | + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" | ||
| 465 | + b"author Andr\xe9 <andre@example.com> 1112591163 +0200\n" | ||
| 466 | + b"committer Andr\xe9 <andre@example.com> 1112591163 +0200\n" | ||
| 467 | + b"encoding ISO-8859-1\n" | ||
| 468 | + b"\n" | ||
| 469 | + b"R\xe9sum\xe9\n" | ||
| 470 | + ) | ||
| 471 | + cmt = self.rorepo.commit() | ||
| 472 | + cmt._deserialize(BytesIO(data)) | ||
| 473 | + | ||
| 474 | + self.assertEqual(cmt.encoding, "ISO-8859-1") | ||
| 475 | + self.assertEqual(cmt.author.name, "André") | ||
| 476 | + self.assertEqual(cmt.committer.name, "André") | ||
| 477 | + self.assertEqual(cmt.message, "Résumé\n") | ||
| 478 | + | ||
| 479 | + def test_deserialization_unknown_declared_encoding(self): | ||
| 480 | + # An encoding Python has no text codec for falls back to the default encoding. | ||
| 481 | + data = ( | ||
| 482 | + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" | ||
| 483 | + b"author Andr\xe9 <andre@example.com> 1112591163 +0200\n" | ||
| 484 | + b"committer Andr\xe9 <andre@example.com> 1112591163 +0200\n" | ||
| 485 | + b"encoding pony-speak\n" | ||
| 486 | + b"\n" | ||
| 487 | + b"R\xe9sum\xe9\n" | ||
| 488 | + ) | ||
| 489 | + cmt = self.rorepo.commit() | ||
| 490 | + cmt._deserialize(BytesIO(data)) | ||
| 491 | + | ||
| 492 | + self.assertEqual(cmt.encoding, cmt.default_encoding) | ||
| 493 | + self.assertEqual(cmt.author.name, "Andr�") | ||
| 494 | + self.assertEqual(cmt.message, "R�sum�\n") | ||
| 495 | + | ||
| 461 | 496 | @with_rw_directory | |
| 462 | 497 | def test_identity_cannot_alter_headers(self, rw_dir): | |
| 463 | 498 | """A name or email must not add header lines or present another identity.""" | |
0 commit comments