FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

zlib: add dictionary support to zstdCompress and zstdDecompress · nodejs/node@2013045 · GitHub

/ node Public

Commit 2013045

Browse files
lluisemper
authored and
RafaelGSS
committed
zlib: add dictionary support to zstdCompress and zstdDecompress
Adds optional dictionary support to zlib’s zstdCompress and zstdDecompress APIs. This enables better compression ratios when the dictionary matches expected input structure or content patterns. The implementation allows passing a `dictionary` buffer through the options object. Support was added to both streaming and convenience methods. Tests and documentation were also updated to reflect this new capability. Fixes: #59105 PR-URL: #59240 Reviewed-By: Anna Henningsen <anna@addaleax.net>
1 parent e79c93a commit 2013045

4 files changed

Lines changed: 79 additions & 7 deletions

File tree

‎doc/api/zlib.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,9 @@ Each Zstd-based class takes an `options` object. All options are optional.
10691069
* `maxOutputLength` {integer} Limits output size when using
10701070
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
10711071
* `info` {boolean} If `true`, returns an object with `buffer` and `engine`. **Default:** `false`
1072+
* `dictionary` {Buffer} Optional dictionary used to
1073+
improve compression efficiency when compressing or decompressing data that
1074+
shares common patterns with the dictionary.
10721075

10731076
For example:
10741077

‎lib/zlib.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -893,12 +893,15 @@ class Zstd extends ZlibBase {
893893
const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;
894894

895895
const writeState = new Uint32Array(2);
896+
896897
handle.init(
897898
initParamsArray,
898899
pledgedSrcSize,
899900
writeState,
900901
processCallback,
902+
opts?.dictionary && isArrayBufferView(opts.dictionary) ? opts.dictionary : undefined,
901903
);
904+
902905
super(opts, mode, handle, zstdDefaultOpts);
903906
this._writeState = writeState;
904907
}

‎src/node_zlib.cc‎

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,8 @@ class ZstdCompressContext final : public ZstdContext {
324324
CompressionError ResetStream();
325325

326326
// Zstd specific:
327-
CompressionError Init(uint64_t pledged_src_size);
327+
CompressionError Init(uint64_t pledged_src_size,
328+
std::string_view dictionary = {});
328329
CompressionError SetParameter(int key, int value);
329330

330331
// Wrap ZSTD_freeCCtx to remove the return type.
@@ -349,7 +350,9 @@ class ZstdDecompressContext final : public ZstdContext {
349350
CompressionError ResetStream();
350351

351352
// Zstd specific:
352-
CompressionError Init(uint64_t pledged_src_size);
353+
CompressionError Init(uint64_t pledged_src_size,
354+
std::string_view dictionary = {});
355+
353356
CompressionError SetParameter(int key, int value);
354357

355358
// Wrap ZSTD_freeDCtx to remove the return type.
@@ -875,8 +878,10 @@ class ZstdStream final : public CompressionStream<CompressionContext> {
875878
Environment* env = Environment::GetCurrent(args);
876879
Local<Context> context = env->context();
877880

878-
CHECK(args.Length() == 4 &&
879-
"init(params, pledgedSrcSize, writeResult, writeCallback)");
881+
CHECK((args.Length() == 4 || args.Length() == 5) &&
882+
"init(params, pledgedSrcSize, writeResult, writeCallback[, "
883+
"dictionary])");
884+
880885
ZstdStream* wrap;
881886
ASSIGN_OR_RETURN_UNWRAP(&wrap, args.This());
882887

@@ -904,7 +909,19 @@ class ZstdStream final : public CompressionStream<CompressionContext> {
904909
}
905910

906911
AllocScope alloc_scope(wrap);
907-
CompressionError err = wrap->context()->Init(pledged_src_size);
912+
std::string_view dictionary;
913+
ArrayBufferViewContents<char> contents;
914+
if (args.Length() == 5 && !args[4]->IsUndefined()) {
915+
if (!args[4]->IsArrayBufferView()) {
916+
THROW_ERR_INVALID_ARG_TYPE(
917+
wrap->env(), "dictionary must be an ArrayBufferView if provided");
918+
return;
919+
}
920+
contents.ReadValue(args[4]);
921+
dictionary = std::string_view(contents.data(), contents.length());
922+
}
923+
924+
CompressionError err = wrap->context()->Init(pledged_src_size, dictionary);
908925
if (err.IsError()) {
909926
wrap->EmitError(err);
910927
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
@@ -1509,14 +1526,26 @@ CompressionError ZstdCompressContext::SetParameter(int key, int value) {
15091526
return {};
15101527
}
15111528

1512-
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {
1529+
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size,
1530+
std::string_view dictionary) {
15131531
pledged_src_size_ = pledged_src_size;
15141532
cctx_.reset(ZSTD_createCCtx());
15151533
if (!cctx_) {
15161534
return CompressionError("Could not initialize zstd instance",
15171535
"ERR_ZLIB_INITIALIZATION_FAILED",
15181536
-1);
15191537
}
1538+
1539+
if (!dictionary.empty()) {
1540+
size_t ret = ZSTD_CCtx_loadDictionary(
1541+
cctx_.get(), dictionary.data(), dictionary.size());
1542+
if (ZSTD_isError(ret)) {
1543+
return CompressionError("Failed to load zstd dictionary",
1544+
"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
1545+
-1);
1546+
}
1547+
}
1548+
15201549
size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);
15211550
if (ZSTD_isError(result)) {
15221551
return CompressionError(
@@ -1549,13 +1578,24 @@ CompressionError ZstdDecompressContext::SetParameter(int key, int value) {
15491578
return {};
15501579
}
15511580

1552-
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {
1581+
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size,
1582+
std::string_view dictionary) {
15531583
dctx_.reset(ZSTD_createDCtx());
15541584
if (!dctx_) {
15551585
return CompressionError("Could not initialize zstd instance",
15561586
"ERR_ZLIB_INITIALIZATION_FAILED",
15571587
-1);
15581588
}
1589+
1590+
if (!dictionary.empty()) {
1591+
size_t ret = ZSTD_DCtx_loadDictionary(
1592+
dctx_.get(), dictionary.data(), dictionary.size());
1593+
if (ZSTD_isError(ret)) {
1594+
return CompressionError("Failed to load zstd dictionary",
1595+
"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
1596+
-1);
1597+
}
1598+
}
15591599
return {};
15601600
}
15611601

‎test/parallel/test-zlib-zstd-dictionary.js‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
'use strict';
2+
3+
const common = require('../common');
4+
const assert = require('assert');
5+
const zlib = require('zlib');
6+
7+
const dictionary = Buffer.from(
8+
`Lorem ipsum dolor sit amet, consectetur adipiscing elit.
9+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
10+
Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.`
11+
);
12+
13+
const input = Buffer.from(
14+
`Lorem ipsum dolor sit amet, consectetur adipiscing elit.
15+
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
16+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
17+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
18+
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`
19+
);
20+
21+
zlib.zstdCompress(input, { dictionary }, common.mustSucceed((compressed) => {
22+
assert(compressed.length < input.length);
23+
zlib.zstdDecompress(compressed, { dictionary }, common.mustSucceed((decompressed) => {
24+
assert.strictEqual(decompressed.toString(), input.toString());
25+
}));
26+
}));

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL