FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fs: add support for async iterators to `fsPromises.writeFile` · nodejs/node@cad9d20 · GitHub

/ node Public

Commit cad9d20

Browse files
HiroyukiYagihashi
authored and
targos
committed
fs: add support for async iterators to fsPromises.writeFile
Fixes: #37391 PR-URL: #37490 Backport-PR-URL: #39973 Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent 32a5b8f commit cad9d20

4 files changed

Lines changed: 144 additions & 16 deletions

File tree

‎doc/api/fs.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1234,6 +1234,9 @@ All the [caveats][] for `fs.watch()` also apply to `fsPromises.watch()`.
12341234
<!-- YAML
12351235
added: v10.0.0
12361236
changes:
1237+
- version: REPLACEME
1238+
pr-url: https://github.com/nodejs/node/pull/37490
1239+
description: The `data` argument supports `AsyncIterable`, `Iterable` & `Stream`.
12371240
- version: v14.17.0
12381241
pr-url: https://github.com/nodejs/node/pull/35993
12391242
description: The options argument may include an AbortSignal to abort an
@@ -1249,7 +1252,8 @@ changes:
12491252
-->
12501253
12511254
* `file` {string|Buffer|URL|FileHandle} filename or `FileHandle`
1252-
* `data` {string|Buffer|Uint8Array|Object}
1255+
* `data` {string|Buffer|Uint8Array|Object|AsyncIterable|Iterable
1256+
|Stream}
12531257
* `options` {Object|string}
12541258
* `encoding` {string|null} **Default:** `'utf8'`
12551259
* `mode` {integer} **Default:** `0o666`

‎lib/internal/fs/promises.js‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ const {
2525
const binding = internalBinding('fs');
2626
const { Buffer } = require('buffer');
2727

28-
const { codes, hideStackFrames } = require('internal/errors');
28+
const { AbortError, codes, hideStackFrames } = require('internal/errors');
2929
const {
3030
ERR_FS_FILE_TOO_LARGE,
3131
ERR_INVALID_ARG_TYPE,
@@ -70,6 +70,7 @@ const {
7070
const pathModule = require('path');
7171
const { promisify } = require('internal/util');
7272
const { watch } = require('internal/fs/watchers');
73+
const { isIterable } = require('internal/streams/utils');
7374

7475
const kHandle = Symbol('kHandle');
7576
const kFd = Symbol('kFd');
@@ -251,8 +252,23 @@ async function fsCall(fn, handle, ...args) {
251252
}
252253
}
253254

254-
async function writeFileHandle(filehandle, data, signal) {
255-
// `data` could be any kind of typed array.
255+
function checkAborted(signal) {
256+
if (signal && signal.aborted)
257+
throw new AbortError();
258+
}
259+
260+
async function writeFileHandle(filehandle, data, signal, encoding) {
261+
checkAborted(signal);
262+
if (isCustomIterable(data)) {
263+
for await (const buf of data) {
264+
checkAborted(signal);
265+
await write(
266+
filehandle, buf, undefined,
267+
isArrayBufferView(buf) ? buf.length : encoding);
268+
checkAborted(signal);
269+
}
270+
return;
271+
}
256272
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
257273
let remaining = data.length;
258274
if (remaining === 0) return;
@@ -422,7 +438,7 @@ async function readv(handle, buffers, position) {
422438
}
423439

424440
async function write(handle, buffer, offset, length, position) {
425-
if (buffer.length === 0)
441+
if (buffer && buffer.length === 0)
426442
return { bytesWritten: 0, buffer };
427443

428444
if (isArrayBufferView(buffer)) {
@@ -664,22 +680,26 @@ async function writeFile(path, data, options) {
664680
options = getOptions(options, { encoding: 'utf8', mode: 0o666, flag: 'w' });
665681
const flag = options.flag || 'w';
666682

667-
if (!isArrayBufferView(data)) {
683+
if (!isArrayBufferView(data) && !isCustomIterable(data)) {
668684
validateStringAfterArrayBufferView(data, 'data');
669685
data = Buffer.from(data, options.encoding || 'utf8');
670686
}
671687

672688
validateAbortSignal(options.signal);
673689
if (path instanceof FileHandle)
674-
return writeFileHandle(path, data, options.signal);
690+
return writeFileHandle(path, data, options.signal, options.encoding);
675691

676692
if (options.signal?.aborted) {
677693
throw lazyDOMException('The operation was aborted', 'AbortError');
678694
}
679695

680696
const fd = await open(path, flag, options.mode);
681-
const { signal } = options;
682-
return PromisePrototypeFinally(writeFileHandle(fd, data, signal), fd.close);
697+
return PromisePrototypeFinally(
698+
writeFileHandle(fd, data, options.signal, options.encoding), fd.close);
699+
}
700+
701+
function isCustomIterable(obj) {
702+
return isIterable(obj) && !isArrayBufferView(obj) && typeof obj !== 'string';
683703
}
684704

685705
async function appendFile(path, data, options) {

‎test/parallel/test-fs-append-file.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ const throwNextTick = (e) => { process.nextTick(() => { throw e; }); };
121121
}
122122

123123
// Test that appendFile does not accept invalid data type (callback API).
124-
[false, 5, {}, [], null, undefined].forEach(async (data) => {
124+
[false, 5, {}, null, undefined].forEach(async (data) => {
125125
const errObj = {
126126
code: 'ERR_INVALID_ARG_TYPE',
127127
message: /"data"|"buffer"/

‎test/parallel/test-fs-promises-writefile.js‎

Lines changed: 110 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,115 @@ const path = require('path');
88
const tmpdir = require('../common/tmpdir');
99
const assert = require('assert');
1010
const tmpDir = tmpdir.path;
11+
const { Readable } = require('stream');
1112

1213
tmpdir.refresh();
1314

1415
const dest = path.resolve(tmpDir, 'tmp.txt');
1516
const otherDest = path.resolve(tmpDir, 'tmp-2.txt');
1617
const buffer = Buffer.from('abc'.repeat(1000));
1718
const buffer2 = Buffer.from('xyz'.repeat(1000));
19+
const stream = Readable.from(['a', 'b', 'c']);
20+
const stream2 = Readable.from(['ümlaut', ' ', 'sechzig']);
21+
const iterable = {
22+
expected: 'abc',
23+
*[Symbol.iterator]() {
24+
yield 'a';
25+
yield 'b';
26+
yield 'c';
27+
}
28+
};
29+
function iterableWith(value) {
30+
return {
31+
*[Symbol.iterator]() {
32+
yield value;
33+
}
34+
};
35+
}
36+
const bufferIterable = {
37+
expected: 'abc',
38+
*[Symbol.iterator]() {
39+
yield Buffer.from('a');
40+
yield Buffer.from('b');
41+
yield Buffer.from('c');
42+
}
43+
};
44+
const asyncIterable = {
45+
expected: 'abc',
46+
async* [Symbol.asyncIterator]() {
47+
yield 'a';
48+
yield 'b';
49+
yield 'c';
50+
}
51+
};
1852

1953
async function doWrite() {
2054
await fsPromises.writeFile(dest, buffer);
2155
const data = fs.readFileSync(dest);
2256
assert.deepStrictEqual(data, buffer);
2357
}
2458

59+
async function doWriteStream() {
60+
await fsPromises.writeFile(dest, stream);
61+
const expected = 'abc';
62+
const data = fs.readFileSync(dest, 'utf-8');
63+
assert.deepStrictEqual(data, expected);
64+
}
65+
66+
async function doWriteStreamWithCancel() {
67+
const controller = new AbortController();
68+
const { signal } = controller;
69+
process.nextTick(() => controller.abort());
70+
assert.rejects(fsPromises.writeFile(otherDest, stream, { signal }), {
71+
name: 'AbortError'
72+
});
73+
}
74+
75+
async function doWriteIterable() {
76+
await fsPromises.writeFile(dest, iterable);
77+
const data = fs.readFileSync(dest, 'utf-8');
78+
assert.deepStrictEqual(data, iterable.expected);
79+
}
80+
81+
async function doWriteInvalidIterable() {
82+
await Promise.all(
83+
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
84+
assert.rejects(fsPromises.writeFile(dest, iterableWith(value)), {
85+
code: 'ERR_INVALID_ARG_TYPE',
86+
})
87+
)
88+
);
89+
}
90+
91+
async function doWriteIterableWithEncoding() {
92+
await fsPromises.writeFile(dest, stream2, 'latin1');
93+
const expected = 'ümlaut sechzig';
94+
const data = fs.readFileSync(dest, 'latin1');
95+
assert.deepStrictEqual(data, expected);
96+
}
97+
98+
async function doWriteBufferIterable() {
99+
await fsPromises.writeFile(dest, bufferIterable);
100+
const data = fs.readFileSync(dest, 'utf-8');
101+
assert.deepStrictEqual(data, bufferIterable.expected);
102+
}
103+
104+
async function doWriteAsyncIterable() {
105+
await fsPromises.writeFile(dest, asyncIterable);
106+
const data = fs.readFileSync(dest, 'utf-8');
107+
assert.deepStrictEqual(data, asyncIterable.expected);
108+
}
109+
110+
async function doWriteInvalidValues() {
111+
await Promise.all(
112+
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
113+
assert.rejects(fsPromises.writeFile(dest, value), {
114+
code: 'ERR_INVALID_ARG_TYPE',
115+
})
116+
)
117+
);
118+
}
119+
25120
async function doWriteWithCancel() {
26121
const controller = new AbortController();
27122
const { signal } = controller;
@@ -51,9 +146,18 @@ async function doReadWithEncoding() {
51146
assert.deepStrictEqual(data, syncData);
52147
}
53148

54-
doWrite()
55-
.then(doWriteWithCancel)
56-
.then(doAppend)
57-
.then(doRead)
58-
.then(doReadWithEncoding)
59-
.then(common.mustCall());
149+
(async () => {
150+
await doWrite();
151+
await doWriteWithCancel();
152+
await doAppend();
153+
await doRead();
154+
await doReadWithEncoding();
155+
await doWriteStream();
156+
await doWriteStreamWithCancel();
157+
await doWriteIterable();
158+
await doWriteInvalidIterable();
159+
await doWriteIterableWithEncoding();
160+
await doWriteBufferIterable();
161+
await doWriteAsyncIterable();
162+
await doWriteInvalidValues();
163+
})().then(common.mustCall());

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL