FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

zlib: add dictionary support to zstdCompress and zstdDecompress · nodejs/node@8e697d1 · GitHub

/ node Public

Commit 8e697d1

Browse files
lluisemper
authored and
aduh95
committed
zlib: add dictionary support to zstdCompress and zstdDecompress
Adds optional dictionary support to zlib’s zstdCompress and zstdDecompress APIs. This enables better compression ratios when the dictionary matches expected input structure or content patterns. The implementation allows passing a `dictionary` buffer through the options object. Support was added to both streaming and convenience methods. Tests and documentation were also updated to reflect this new capability. Fixes: #59105 PR-URL: #59240 Reviewed-By: Anna Henningsen <anna@addaleax.net>
1 parent 390a9dc commit 8e697d1

4 files changed

Lines changed: 79 additions & 7 deletions

File tree

‎doc/api/zlib.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1081,6 +1081,9 @@ Each Zstd-based class takes an `options` object. All options are optional.
10811081
* `maxOutputLength` {integer} Limits output size when using
10821082
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
10831083
* `info` {boolean} If `true`, returns an object with `buffer` and `engine`. **Default:** `false`
1084+
* `dictionary` {Buffer} Optional dictionary used to
1085+
improve compression efficiency when compressing or decompressing data that
1086+
shares common patterns with the dictionary.
10841087

10851088
For example:
10861089

‎lib/zlib.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -891,12 +891,15 @@ class Zstd extends ZlibBase {
891891
const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;
892892

893893
const writeState = new Uint32Array(2);
894+
894895
handle.init(
895896
initParamsArray,
896897
pledgedSrcSize,
897898
writeState,
898899
processCallback,
900+
opts?.dictionary && isArrayBufferView(opts.dictionary) ? opts.dictionary : undefined,
899901
);
902+
900903
super(opts, mode, handle, zstdDefaultOpts);
901904
this._writeState = writeState;
902905
}

‎src/node_zlib.cc‎

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,8 @@ class ZstdCompressContext final : public ZstdContext {
324324
CompressionError ResetStream();
325325

326326
// Zstd specific:
327-
CompressionError Init(uint64_t pledged_src_size);
327+
CompressionError Init(uint64_t pledged_src_size,
328+
std::string_view dictionary = {});
328329
CompressionError SetParameter(int key, int value);
329330

330331
// Wrap ZSTD_freeCCtx to remove the return type.
@@ -349,7 +350,9 @@ class ZstdDecompressContext final : public ZstdContext {
349350
CompressionError ResetStream();
350351

351352
// Zstd specific:
352-
CompressionError Init(uint64_t pledged_src_size);
353+
CompressionError Init(uint64_t pledged_src_size,
354+
std::string_view dictionary = {});
355+
353356
CompressionError SetParameter(int key, int value);
354357

355358
// Wrap ZSTD_freeDCtx to remove the return type.
@@ -874,8 +877,10 @@ class ZstdStream final : public CompressionStream<CompressionContext> {
874877
Environment* env = Environment::GetCurrent(args);
875878
Local<Context> context = env->context();
876879

877-
CHECK(args.Length() == 4 &&
878-
"init(params, pledgedSrcSize, writeResult, writeCallback)");
880+
CHECK((args.Length() == 4 || args.Length() == 5) &&
881+
"init(params, pledgedSrcSize, writeResult, writeCallback[, "
882+
"dictionary])");
883+
879884
ZstdStream* wrap;
880885
ASSIGN_OR_RETURN_UNWRAP(&wrap, args.This());
881886

@@ -903,7 +908,19 @@ class ZstdStream final : public CompressionStream<CompressionContext> {
903908
}
904909

905910
AllocScope alloc_scope(wrap);
906-
CompressionError err = wrap->context()->Init(pledged_src_size);
911+
std::string_view dictionary;
912+
ArrayBufferViewContents<char> contents;
913+
if (args.Length() == 5 && !args[4]->IsUndefined()) {
914+
if (!args[4]->IsArrayBufferView()) {
915+
THROW_ERR_INVALID_ARG_TYPE(
916+
wrap->env(), "dictionary must be an ArrayBufferView if provided");
917+
return;
918+
}
919+
contents.ReadValue(args[4]);
920+
dictionary = std::string_view(contents.data(), contents.length());
921+
}
922+
923+
CompressionError err = wrap->context()->Init(pledged_src_size, dictionary);
907924
if (err.IsError()) {
908925
wrap->EmitError(err);
909926
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
@@ -1508,14 +1525,26 @@ CompressionError ZstdCompressContext::SetParameter(int key, int value) {
15081525
return {};
15091526
}
15101527

1511-
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {
1528+
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size,
1529+
std::string_view dictionary) {
15121530
pledged_src_size_ = pledged_src_size;
15131531
cctx_.reset(ZSTD_createCCtx());
15141532
if (!cctx_) {
15151533
return CompressionError("Could not initialize zstd instance",
15161534
"ERR_ZLIB_INITIALIZATION_FAILED",
15171535
-1);
15181536
}
1537+
1538+
if (!dictionary.empty()) {
1539+
size_t ret = ZSTD_CCtx_loadDictionary(
1540+
cctx_.get(), dictionary.data(), dictionary.size());
1541+
if (ZSTD_isError(ret)) {
1542+
return CompressionError("Failed to load zstd dictionary",
1543+
"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
1544+
-1);
1545+
}
1546+
}
1547+
15191548
size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);
15201549
if (ZSTD_isError(result)) {
15211550
return CompressionError(
@@ -1548,13 +1577,24 @@ CompressionError ZstdDecompressContext::SetParameter(int key, int value) {
15481577
return {};
15491578
}
15501579

1551-
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {
1580+
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size,
1581+
std::string_view dictionary) {
15521582
dctx_.reset(ZSTD_createDCtx());
15531583
if (!dctx_) {
15541584
return CompressionError("Could not initialize zstd instance",
15551585
"ERR_ZLIB_INITIALIZATION_FAILED",
15561586
-1);
15571587
}
1588+
1589+
if (!dictionary.empty()) {
1590+
size_t ret = ZSTD_DCtx_loadDictionary(
1591+
dctx_.get(), dictionary.data(), dictionary.size());
1592+
if (ZSTD_isError(ret)) {
1593+
return CompressionError("Failed to load zstd dictionary",
1594+
"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
1595+
-1);
1596+
}
1597+
}
15581598
return {};
15591599
}
15601600

‎test/parallel/test-zlib-zstd-dictionary.js‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
'use strict';
2+
3+
const common = require('../common');
4+
const assert = require('assert');
5+
const zlib = require('zlib');
6+
7+
const dictionary = Buffer.from(
8+
`Lorem ipsum dolor sit amet, consectetur adipiscing elit.
9+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
10+
Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.`
11+
);
12+
13+
const input = Buffer.from(
14+
`Lorem ipsum dolor sit amet, consectetur adipiscing elit.
15+
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
16+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
17+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
18+
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`
19+
);
20+
21+
zlib.zstdCompress(input, { dictionary }, common.mustSucceed((compressed) => {
22+
assert(compressed.length < input.length);
23+
zlib.zstdDecompress(compressed, { dictionary }, common.mustSucceed((decompressed) => {
24+
assert.strictEqual(decompressed.toString(), input.toString());
25+
}));
26+
}));

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL