| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 2548f75 commit d91742a
2 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -27,9 +27,18 @@ function lazyFs() { | |||
| 27 | 27 | const kMinPoolSpace = 128; | |
| 28 | 28 | ||
| 29 | 29 | let pool; | |
| 30 | + // It can happen that we expect to read a large chunk of data, and reserve | ||
| 31 | + // a large chunk of the pool accordingly, but the read() call only filled | ||
| 32 | + // a portion of it. If a concurrently executing read() then uses the same pool, | ||
| 33 | + // the "reserved" portion cannot be used, so we allow it to be re-used as a | ||
| 34 | + // new pool later. | ||
| 35 | + const poolFragments = []; | ||
| 30 | 36 | ||
| 31 | 37 | function allocNewPool(poolSize) { | |
| 32 | - pool = Buffer.allocUnsafe(poolSize); | ||
| 38 | + if (poolFragments.length > 0) | ||
| 39 | + pool = poolFragments.pop(); | ||
| 40 | + else | ||
| 41 | + pool = Buffer.allocUnsafe(poolSize); | ||
| 33 | 42 | pool.used = 0; | |
| 34 | 43 | } | |
| 35 | 44 | ||
@@ -156,6 +165,14 @@ ReadStream.prototype._read = function(n) { | |||
| 156 | 165 | this.emit('error', er); | |
| 157 | 166 | } else { | |
| 158 | 167 | let b = null; | |
| 168 | + // Now that we know how much data we have actually read, re-wind the | ||
| 169 | + // 'used' field if we can, and otherwise allow the remainder of our | ||
| 170 | + // reservation to be used as a new pool later. | ||
| 171 | + if (start + toRead === thisPool.used && thisPool === pool) | ||
| 172 | + thisPool.used += bytesRead - toRead; | ||
| 173 | + else if (toRead - bytesRead > kMinPoolSpace) | ||
| 174 | + poolFragments.push(thisPool.slice(start + bytesRead, start + toRead)); | ||
| 175 | + | ||
| 159 | 176 | if (bytesRead > 0) { | |
| 160 | 177 | this.bytesRead += bytesRead; | |
| 161 | 178 | b = thisPool.slice(start, start + bytesRead); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,47 @@ | |||
| 1 | + 'use strict'; | ||
| 2 | + const common = require('../common'); | ||
| 3 | + const fixtures = require('../common/fixtures'); | ||
| 4 | + const assert = require('assert'); | ||
| 5 | + const fs = require('fs'); | ||
| 6 | + | ||
| 7 | + // Test that concurrent file read streams don’t interfere with each other’s | ||
| 8 | + // contents, and that the chunks generated by the reads only retain a | ||
| 9 | + // 'reasonable' amount of memory. | ||
| 10 | + | ||
| 11 | + // Refs: https://github.com/nodejs/node/issues/21967 | ||
| 12 | + | ||
| 13 | + const filename = fixtures.path('loop.js'); // Some small non-homogeneous file. | ||
| 14 | + const content = fs.readFileSync(filename); | ||
| 15 | + | ||
| 16 | + const N = 1000; | ||
| 17 | + let started = 0; | ||
| 18 | + let done = 0; | ||
| 19 | + | ||
| 20 | + const arrayBuffers = new Set(); | ||
| 21 | + | ||
| 22 | + function startRead() { | ||
| 23 | + ++started; | ||
| 24 | + const chunks = []; | ||
| 25 | + fs.createReadStream(filename) | ||
| 26 | + .on('data', (chunk) => { | ||
| 27 | + chunks.push(chunk); | ||
| 28 | + arrayBuffers.add(chunk.buffer); | ||
| 29 | + if (started < N) | ||
| 30 | + startRead(); | ||
| 31 | + }) | ||
| 32 | + .on('end', common.mustCall(() => { | ||
| 33 | + assert.deepStrictEqual(Buffer.concat(chunks), content); | ||
| 34 | + if (++done === N) { | ||
| 35 | + const retainedMemory = | ||
| 36 | + [...arrayBuffers].map((ab) => ab.byteLength).reduce((a, b) => a + b); | ||
| 37 | + assert(retainedMemory / (N * content.length) <= 3, | ||
| 38 | + `Retaining ${retainedMemory} bytes in ABs for ${N} ` + | ||
| 39 | + `chunks of size ${content.length}`); | ||
| 40 | + } | ||
| 41 | + })); | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + // Don’t start the reads all at once – that way we would have to allocate | ||
| 45 | + // a large amount of memory upfront. | ||
| 46 | + for (let i = 0; i < 4; ++i) | ||
| 47 | + startRead(); | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments