FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

zlib: add zstd support · nodejs/node@4cc7907 · GitHub

/ node Public

Commit 4cc7907

Browse files
hybrist
authored and
targos
committed
zlib: add zstd support
Fixes: #48412 PR-URL: #52100 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
1 parent 380a8d8 commit 4cc7907

22 files changed

Lines changed: 997 additions & 21 deletions

‎benchmark/zlib/creation.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ const zlib = require('zlib');
55
const bench = common.createBenchmark(main, {
66
type: [
77
'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
8-
'BrotliCompress', 'BrotliDecompress',
8+
'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress',
99
],
1010
options: ['true', 'false'],
1111
n: [5e5],

‎benchmark/zlib/pipe.js‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,27 @@ const bench = common.createBenchmark(main, {
77
inputLen: [1024],
88
duration: [5],
99
type: ['string', 'buffer'],
10-
algorithm: ['gzip', 'brotli'],
10+
algorithm: ['gzip', 'brotli', 'zstd'],
1111
}, {
1212
test: {
1313
inputLen: 1024,
1414
duration: 0.2,
1515
},
1616
});
1717

18+
const algorithms = {
19+
'gzip': [zlib.createGzip, zlib.createGunzip],
20+
'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress],
21+
'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress],
22+
};
23+
1824
function main({ inputLen, duration, type, algorithm }) {
1925
const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
2026
const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
2127

22-
const input = algorithm === 'gzip' ?
23-
zlib.createGzip() : zlib.createBrotliCompress();
24-
const output = algorithm === 'gzip' ?
25-
zlib.createGunzip() : zlib.createBrotliDecompress();
28+
const [createCompress, createUncompress] = algorithms[algorithm];
29+
const input = createCompress();
30+
const output = createUncompress();
2631

2732
let readFromOutput = 0;
2833
input.pipe(output);

‎doc/api/errors.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3314,6 +3314,12 @@ The requested functionality is not supported in worker threads.
33143314

33153315
Creation of a [`zlib`][] object failed due to incorrect configuration.
33163316

3317+
<a id="ERR_ZSTD_INVALID_PARAM"></a>
3318+
3319+
### `ERR_ZSTD_INVALID_PARAM`
3320+
3321+
An invalid parameter key was passed during construction of a Zstd stream.
3322+
33173323
<a id="HPE_CHUNK_EXTENSIONS_OVERFLOW"></a>
33183324

33193325
### `HPE_CHUNK_EXTENSIONS_OVERFLOW`

‎doc/api/zlib.md‎

Lines changed: 175 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<!-- source_link=lib/zlib.js -->
88

99
The `node:zlib` module provides compression functionality implemented using
10-
Gzip, Deflate/Inflate, and Brotli.
10+
Gzip, Deflate/Inflate, Brotli, and Zstd.
1111

1212
To access it:
1313

@@ -220,8 +220,8 @@ operations be cached to avoid duplication of effort.
220220

221221
## Compressing HTTP requests and responses
222222

223-
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`
224-
and `br` content-encoding mechanisms defined by
223+
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`,
224+
`br`, and `zstd` content-encoding mechanisms defined by
225225
[HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
226226

227227
The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
@@ -284,7 +284,7 @@ const { pipeline } = require('node:stream');
284284
const request = http.get({ host: 'example.com',
285285
path: '/',
286286
port: 80,
287-
headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
287+
headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } });
288288
request.on('response', (response) => {
289289
const output = fs.createWriteStream('example.com_index.html');
290290

@@ -306,6 +306,9 @@ request.on('response', (response) => {
306306
case 'deflate':
307307
pipeline(response, zlib.createInflate(), output, onError);
308308
break;
309+
case 'zstd':
310+
pipeline(response, zlib.createZstdDecompress(), output, onError);
311+
break;
309312
default:
310313
pipeline(response, output, onError);
311314
break;
@@ -396,6 +399,9 @@ http.createServer((request, response) => {
396399
} else if (/\bbr\b/.test(acceptEncoding)) {
397400
response.writeHead(200, { 'Content-Encoding': 'br' });
398401
pipeline(raw, zlib.createBrotliCompress(), response, onError);
402+
} else if (/\bzstd\b/.test(acceptEncoding)) {
403+
response.writeHead(200, { 'Content-Encoding': 'zstd' });
404+
pipeline(raw, zlib.createZstdCompress(), response, onError);
399405
} else {
400406
response.writeHead(200, {});
401407
pipeline(raw, response, onError);
@@ -416,6 +422,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64');
416422
zlib.unzip(
417423
buffer,
418424
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
425+
// For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.
419426
{ finishFlush: zlib.constants.Z_SYNC_FLUSH },
420427
(err, buffer) => {
421428
if (err) {
@@ -487,6 +494,16 @@ these options have different ranges than the zlib ones:
487494

488495
See [below][Brotli parameters] for more details on Brotli-specific options.
489496

497+
### For Zstd-based streams
498+
499+
There are equivalents to the zlib options for Zstd-based streams, although
500+
these options have different ranges than the zlib ones:
501+
502+
* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option.
503+
* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option.
504+
505+
See [below][Zstd parameters] for more details on Zstd-specific options.
506+
490507
## Flushing
491508

492509
Calling [`.flush()`][] on a compression stream will make `zlib` return as much
@@ -701,6 +718,50 @@ These advanced options are available for controlling decompression:
701718
* Boolean flag enabling “Large Window Brotli” mode (not compatible with the
702719
Brotli format as standardized in [RFC 7932][]).
703720

721+
### Zstd constants
722+
723+
<!-- YAML
724+
added: REPLACEME
725+
-->
726+
727+
There are several options and other constants available for Zstd-based
728+
streams:
729+
730+
#### Flush operations
731+
732+
The following values are valid flush operations for Zstd-based streams:
733+
734+
* `zlib.constants.ZSTD_e_continue` (default for all operations)
735+
* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`)
736+
* `zlib.constants.ZSTD_e_end` (default for the last chunk)
737+
738+
#### Compressor options
739+
740+
There are several options that can be set on Zstd encoders, affecting
741+
compression efficiency and speed. Both the keys and the values can be accessed
742+
as properties of the `zlib.constants` object.
743+
744+
The most important options are:
745+
746+
* `ZSTD_c_compressionLevel`
747+
* Set compression parameters according to pre-defined cLevel table. Default
748+
level is ZSTD\_CLEVEL\_DEFAULT==3.
749+
750+
#### Pledged Source Size
751+
752+
It's possible to specify the expected total size of the uncompressed input via
753+
`opts.pledgedSrcSize`. If the size doesn't match at the end of the input,
754+
compression will fail with the code `ZSTD_error_srcSize_wrong`.
755+
756+
#### Decompressor options
757+
758+
These advanced options are available for controlling decompression:
759+
760+
* `ZSTD_d_windowLogMax`
761+
* Select a size limit (in power of 2) beyond which the streaming API will
762+
refuse to allocate memory buffer in order to protect the host from
763+
unreasonable memory requirements.
764+
704765
## Class: `Options`
705766

706767
<!-- YAML
@@ -962,6 +1023,51 @@ added: v0.7.0
9621023
Reset the compressor/decompressor to factory defaults. Only applicable to
9631024
the inflate and deflate algorithms.
9641025

1026+
## Class: `ZstdOptions`
1027+
1028+
<!-- YAML
1029+
added: REPLACEME
1030+
-->
1031+
1032+
<!--type=misc-->
1033+
1034+
Each Zstd-based class takes an `options` object. All options are optional.
1035+
1036+
* `flush` {integer} **Default:** `zlib.constants.ZSTD_e_continue`
1037+
* `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end`
1038+
* `chunkSize` {integer} **Default:** `16 * 1024`
1039+
* `params` {Object} Key-value object containing indexed [Zstd parameters][].
1040+
* `maxOutputLength` {integer} Limits output size when using
1041+
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
1042+
1043+
For example:
1044+
1045+
```js
1046+
const stream = zlib.createZstdCompress({
1047+
chunkSize: 32 * 1024,
1048+
params: {
1049+
[zlib.constants.ZSTD_c_compressionLevel]: 10,
1050+
[zlib.constants.ZSTD_c_checksumFlag]: 1,
1051+
},
1052+
});
1053+
```
1054+
1055+
## Class: `zlib.ZstdCompress`
1056+
1057+
<!-- YAML
1058+
added: REPLACEME
1059+
-->
1060+
1061+
Compress data using the Zstd algorithm.
1062+
1063+
## Class: `zlib.ZstdDecompress`
1064+
1065+
<!-- YAML
1066+
added: REPLACEME
1067+
-->
1068+
1069+
Decompress data using the Zstd algorithm.
1070+
9651071
## `zlib.constants`
9661072

9671073
<!-- YAML
@@ -1135,6 +1241,26 @@ added: v0.5.8
11351241

11361242
Creates and returns a new [`Unzip`][] object.
11371243

1244+
## `zlib.createZstdCompress([options])`
1245+
1246+
<!-- YAML
1247+
added: REPLACEME
1248+
-->
1249+
1250+
* `options` {zstd options}
1251+
1252+
Creates and returns a new [`ZstdCompress`][] object.
1253+
1254+
## `zlib.createZstdDecompress([options])`
1255+
1256+
<!-- YAML
1257+
added: REPLACEME
1258+
-->
1259+
1260+
* `options` {zstd options}
1261+
1262+
Creates and returns a new [`ZstdDecompress`][] object.
1263+
11381264
## Convenience methods
11391265

11401266
<!--type=misc-->
@@ -1481,11 +1607,54 @@ changes:
14811607

14821608
Decompress a chunk of data with [`Unzip`][].
14831609

1610+
### `zlib.zstdCompress(buffer[, options], callback)`
1611+
1612+
<!-- YAML
1613+
added: REPLACEME
1614+
-->
1615+
1616+
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
1617+
* `options` {zstd options}
1618+
* `callback` {Function}
1619+
1620+
### `zlib.zstdCompressSync(buffer[, options])`
1621+
1622+
<!-- YAML
1623+
added: REPLACEME
1624+
-->
1625+
1626+
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
1627+
* `options` {zstd options}
1628+
1629+
Compress a chunk of data with [`ZstdCompress`][].
1630+
1631+
### `zlib.zstdDecompress(buffer[, options], callback)`
1632+
1633+
<!-- YAML
1634+
added: REPLACEME
1635+
-->
1636+
1637+
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
1638+
* `options` {zstd options}
1639+
* `callback` {Function}
1640+
1641+
### `zlib.zstdDecompressSync(buffer[, options])`
1642+
1643+
<!-- YAML
1644+
added: REPLACEME
1645+
-->
1646+
1647+
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
1648+
* `options` {zstd options}
1649+
1650+
Decompress a chunk of data with [`ZstdDecompress`][].
1651+
14841652
[Brotli parameters]: #brotli-constants
14851653
[Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
14861654
[Memory usage tuning]: #memory-usage-tuning
14871655
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
14881656
[Streams API]: stream.md
1657+
[Zstd parameters]: #zstd-constants
14891658
[`.flush()`]: #zlibflushkind-callback
14901659
[`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
14911660
[`BrotliCompress`]: #class-zlibbrotlicompress
@@ -1498,6 +1667,8 @@ Decompress a chunk of data with [`Unzip`][].
14981667
[`InflateRaw`]: #class-zlibinflateraw
14991668
[`Inflate`]: #class-zlibinflate
15001669
[`Unzip`]: #class-zlibunzip
1670+
[`ZstdCompress`]: #class-zlibzstdcompress
1671+
[`ZstdDecompress`]: #class-zlibzstddecompress
15011672
[`buffer.kMaxLength`]: buffer.md#bufferkmaxlength
15021673
[`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced
15031674
[`stream.Transform`]: stream.md#class-streamtransform

‎lib/internal/errors.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1889,3 +1889,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR',
18891889
'Serializing an uncaught exception failed', Error);
18901890
E('ERR_WORKER_UNSUPPORTED_OPERATION',
18911891
'%s is not supported in workers', TypeError);
1892+
E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError);

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL