FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

src: track allocations made by zstd streams · nodejs/node@b572492 · GitHub

/ node Public

Commit b572492

Browse files
addaleax
authored and
aduh95
committed
src: track allocations made by zstd streams
This is both valuable as a diagnostic tool and as a way to inform the JS runtime about external allocations. Currently, this is a feature only enabled in the statically linked builds of zstd, so with `--shared-zstd`, we fall back to the non-tracking variant. PR-URL: #61717 Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com> Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
1 parent 3d1d152 commit b572492

3 files changed

Lines changed: 72 additions & 0 deletions

File tree

‎node.gypi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@
248248

249249
[ 'node_shared_zstd=="false"', {
250250
'dependencies': [ 'deps/zstd/zstd.gyp:zstd' ],
251+
'defines': [ 'NODE_BUNDLED_ZSTD' ],
251252
}],
252253

253254
[ 'OS=="mac"', {

‎src/node_zlib.cc‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
2020
// USE OR OTHER DEALINGS IN THE SOFTWARE.
2121

22+
#ifdef NODE_BUNDLED_ZSTD
23+
#define ZSTD_STATIC_LINKING_ONLY
24+
#endif
25+
2226
#include "memory_tracker-inl.h"
2327
#include "node.h"
2428
#include "node_buffer.h"
@@ -1549,7 +1553,16 @@ void ZstdCompressContext::Close() {
15491553
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size,
15501554
std::string_view dictionary) {
15511555
pledged_src_size_ = pledged_src_size;
1556+
#ifdef NODE_BUNDLED_ZSTD
1557+
ZSTD_customMem custom_mem = {
1558+
CompressionStreamMemoryOwner::AllocForBrotli,
1559+
CompressionStreamMemoryOwner::FreeForZlib,
1560+
CompressionStream<ZstdCompressContext>::AllocatorOpaquePointerForContext(
1561+
this)};
1562+
cctx_.reset(ZSTD_createCCtx_advanced(custom_mem));
1563+
#else
15521564
cctx_.reset(ZSTD_createCCtx());
1565+
#endif
15531566
if (!cctx_) {
15541567
return CompressionError("Could not initialize zstd instance",
15551568
"ERR_ZLIB_INITIALIZATION_FAILED",
@@ -1604,7 +1617,16 @@ void ZstdDecompressContext::Close() {
16041617

16051618
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size,
16061619
std::string_view dictionary) {
1620+
#ifdef NODE_BUNDLED_ZSTD
1621+
ZSTD_customMem custom_mem = {
1622+
CompressionStreamMemoryOwner::AllocForBrotli,
1623+
CompressionStreamMemoryOwner::FreeForZlib,
1624+
CompressionStream<
1625+
ZstdDecompressContext>::AllocatorOpaquePointerForContext(this)};
1626+
dctx_.reset(ZSTD_createDCtx_advanced(custom_mem));
1627+
#else
16071628
dctx_.reset(ZSTD_createDCtx());
1629+
#endif
16081630
if (!dctx_) {
16091631
return CompressionError("Could not initialize zstd instance",
16101632
"ERR_ZLIB_INITIALIZATION_FAILED",

‎test/pummel/test-heapdump-zstd.js‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
'use strict';
2+
// This tests heap snapshot integration of zlib stream.
3+
4+
const common = require('../common');
5+
const assert = require('assert');
6+
const { validateByRetainingPath, validateByRetainingPathFromNodes } = require('../common/heap');
7+
const zlib = require('zlib');
8+
9+
// Before zstd stream is created, no ZstdStream should be created.
10+
{
11+
const nodes = validateByRetainingPath('Node / ZstdStream', []);
12+
assert.strictEqual(nodes.length, 0);
13+
}
14+
15+
const compress = zlib.createZstdCompress();
16+
17+
// After zstd stream is created, a ZstdStream should be created.
18+
{
19+
const streams = validateByRetainingPath('Node / ZstdStream', []);
20+
validateByRetainingPathFromNodes(streams, 'Node / ZstdStream', [
21+
{ node_name: 'ZstdCompress', edge_name: 'native_to_javascript' },
22+
]);
23+
const withMemory = validateByRetainingPathFromNodes(streams, 'Node / ZstdStream', [
24+
{ node_name: 'Node / zlib_memory', edge_name: 'zlib_memory' },
25+
], true);
26+
if (process.config.variables.node_shared_zstd === true) {
27+
assert.strictEqual(withMemory.length, 0);
28+
} else {
29+
assert.strictEqual(withMemory.length, 1);
30+
// Between 1KB and 1MB (measured value was around ~5kB)
31+
assert.ok(withMemory[0].self_size > 1024);
32+
assert.ok(withMemory[0].self_size < 1024 * 1024);
33+
}
34+
}
35+
36+
// After zstd stream is written, zlib_memory is significantly larger.
37+
compress.write('hello world', common.mustCall(() => {
38+
const streams = validateByRetainingPath('Node / ZstdStream', []);
39+
const withMemory = validateByRetainingPathFromNodes(streams, 'Node / ZstdStream', [
40+
{ node_name: 'Node / zlib_memory', edge_name: 'zlib_memory' },
41+
], true);
42+
if (process.config.variables.node_shared_zstd === true) {
43+
assert.strictEqual(withMemory.length, 0);
44+
} else {
45+
assert.strictEqual(withMemory.length, 1);
46+
// More than 1MB
47+
assert.ok(withMemory[0].self_size > 1024 * 1024);
48+
}
49+
}));

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL