FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-103477: Write gzip trailer with zlib · python/cpython@3be4530 · GitHub

/ cpython Public

Commit 3be4530

Browse files
iii-i
committed
gh-103477: Write gzip trailer with zlib
RHEL, SLES and Ubuntu for IBM zSystems (aka s390x) ship with a zlib optimization [1] that significantly improves deflate performance by using a specialized CPU instruction. This instruction not only compresses the data, but also computes a checksum. At the moment Pyhton's gzip support performs compression and checksum calculation separately, which creates unnecessary overhead. The reason is that Python needs to write specific values into gzip header, so it uses a raw stream instead of a gzip stream, and zlib does not compute a checksum for raw streams. The challenge with using gzip streams instead of zlib streams is dealing with zlib-generated gzip header, which we need to rather generate manually. Implement the method proposed by @rhpvorderman: use Z_BLOCK on the first deflate() call in order to stop before the first deflate block is emitted. The data that is emitted up until this point is zlib-generated gzip header, which should be discarded. Expose this new functionality by adding a boolean gzip_trailer argument to zlib.compress() and zlib.compressobj(). Make use of it in gzip.compress(), GzipFile and TarFile. The performance improvement varies depending on data being compressed, but it's in the ballpark of 40%. An alternative approach is to use the deflateSetHeader() function, introduced in zlib v1.2.2.1 (2011). This also works, but the change was deemed too intrusive [2]. 📜🤖 Added by blurb_it. [1] madler/zlib#410 [2] #103478
1 parent a71e32c commit 3be4530

9 files changed

Lines changed: 141 additions & 49 deletions

‎Include/internal/pycore_global_objects_fini_generated.h‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,7 @@ struct _Py_global_strings {
463463
STRUCT_FOR_ID(globals)
464464
STRUCT_FOR_ID(groupindex)
465465
STRUCT_FOR_ID(groups)
466+
STRUCT_FOR_ID(gzip_trailer)
466467
STRUCT_FOR_ID(h)
467468
STRUCT_FOR_ID(handle)
468469
STRUCT_FOR_ID(handle_seq)

‎Include/internal/pycore_runtime_init_generated.h‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_unicodeobject_generated.h‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/gzip.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,8 @@ def __init__(self, filename=None, mode=None,
221221
zlib.DEFLATED,
222222
-zlib.MAX_WBITS,
223223
zlib.DEF_MEM_LEVEL,
224-
0)
224+
0,
225+
gzip_trailer=True)
225226
self._write_mtime = mtime
226227
self._buffer_size = _WRITE_BUFFER_SIZE
227228
self._buffer = io.BufferedWriter(_WriteBufferStream(self),
@@ -245,8 +246,6 @@ def __repr__(self):
245246

246247
def _init_write(self, filename):
247248
self.name = filename
248-
self.crc = zlib.crc32(b"")
249-
self.size = 0
250249
self.writebuf = []
251250
self.bufsize = 0
252251
self.offset = 0 # Current file offset for seek(), tell(), etc
@@ -310,8 +309,6 @@ def _write_raw(self, data):
310309

311310
if length > 0:
312311
self.fileobj.write(self.compress.compress(data))
313-
self.size += length
314-
self.crc = zlib.crc32(data, self.crc)
315312
self.offset += length
316313

317314
return length
@@ -355,9 +352,6 @@ def close(self):
355352
if self.mode == WRITE:
356353
self._buffer.flush()
357354
fileobj.write(self.compress.flush())
358-
write32u(fileobj, self.crc)
359-
# self.size may exceed 2 GiB, or even 4 GiB
360-
write32u(fileobj, self.size & 0xffffffff)
361355
elif self.mode == READ:
362356
self._buffer.close()
363357
finally:
@@ -611,10 +605,11 @@ def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
611605
# This is faster and with less overhead.
612606
return zlib.compress(data, level=compresslevel, wbits=31)
613607
header = _create_simple_gzip_header(compresslevel, mtime)
614-
trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
615-
# Wbits=-15 creates a raw deflate block.
616-
return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
617-
trailer)
608+
# Wbits=-15 creates a raw deflate block. Gzip_trailer=True computes CRC32
609+
# and writes gzip trailer with zlib, which on some platforms is faster
610+
# than doing it manually.
611+
return (header + zlib.compress(data, level=compresslevel, wbits=-15,
612+
gzip_trailer=True))
618613

619614

620615
def decompress(data):

‎Lib/tarfile.py‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize,
370370
except ImportError:
371371
raise CompressionError("zlib module is not available") from None
372372
self.zlib = zlib
373-
self.crc = zlib.crc32(b"")
374373
if mode == "r":
375374
self.exception = zlib.error
376375
self._init_read_gz()
@@ -421,7 +420,8 @@ def _init_write_gz(self, compresslevel):
421420
self.zlib.DEFLATED,
422421
-self.zlib.MAX_WBITS,
423422
self.zlib.DEF_MEM_LEVEL,
424-
0)
423+
0,
424+
gzip_trailer=True)
425425
timestamp = struct.pack("<L", int(time.time()))
426426
self.__write(b"\037\213\010\010" + timestamp + b"\002\377")
427427
if self.name.endswith(".gz"):
@@ -434,8 +434,6 @@ def _init_write_gz(self, compresslevel):
434434
def write(self, s):
435435
"""Write string s to the stream.
436436
"""
437-
if self.comptype == "gz":
438-
self.crc = self.zlib.crc32(s, self.crc)
439437
self.pos += len(s)
440438
if self.comptype != "tar":
441439
s = self.cmp.compress(s)
@@ -465,9 +463,6 @@ def close(self):
465463
if self.mode == "w" and self.buf:
466464
self.fileobj.write(self.buf)
467465
self.buf = b""
468-
if self.comptype == "gz":
469-
self.fileobj.write(struct.pack("<L", self.crc))
470-
self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF))
471466
finally:
472467
if not self._extfileobj:
473468
self.fileobj.close()

‎Misc/NEWS.d/next/Library/2023-11-17-12-26-47.gh-issue-103477._7cTsK.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Write gzip trailer with zlib, improving gzip compression performance on s390x by roughly 40%.

‎Modules/clinic/zlibmodule.c.h‎

Lines changed: 49 additions & 22 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL