FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fs: add recursive option to readdir and opendir · nodejs/node@7273ef5 · GitHub

/ node Public

Commit 7273ef5

Browse files
Ethan Arrowood
authored and
MoLow
committed
fs: add recursive option to readdir and opendir
Adds a naive, linear recursive algorithm for the following methods: readdir, readdirSync, opendir, opendirSync, and the promise based equivalents. Fixes: #34992 PR-URL: #41439 Refs: nodejs/tooling#130 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
1 parent cc7e5dd commit 7273ef5

7 files changed

Lines changed: 659 additions & 31 deletions

File tree

‎doc/api/fs.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1214,6 +1214,9 @@ a colon, Node.js will open a file system stream, as described by
12141214
<!-- YAML
12151215
added: v12.12.0
12161216
changes:
1217+
- version: REPLACEME
1218+
pr-url: https://github.com/nodejs/node/pull/41439
1219+
description: Added `recursive` option.
12171220
- version:
12181221
- v13.1.0
12191222
- v12.16.0
@@ -1227,6 +1230,8 @@ changes:
12271230
* `bufferSize` {number} Number of directory entries that are buffered
12281231
internally when reading from the directory. Higher values lead to better
12291232
performance but higher memory usage. **Default:** `32`
1233+
* `recursive` {boolean} Resolved `Dir` will be an {AsyncIterable}
1234+
containing all sub files and directories. **Default:** `false`
12301235
* Returns: {Promise} Fulfills with an {fs.Dir}.
12311236
12321237
Asynchronously open a directory for iterative scanning. See the POSIX
@@ -1260,6 +1265,9 @@ closed after the iterator exits.
12601265
<!-- YAML
12611266
added: v10.0.0
12621267
changes:
1268+
- version: REPLACEME
1269+
pr-url: https://github.com/nodejs/node/pull/41439
1270+
description: Added `recursive` option.
12631271
- version: v10.11.0
12641272
pr-url: https://github.com/nodejs/node/pull/22020
12651273
description: New option `withFileTypes` was added.
@@ -1269,6 +1277,7 @@ changes:
12691277
* `options` {string|Object}
12701278
* `encoding` {string} **Default:** `'utf8'`
12711279
* `withFileTypes` {boolean} **Default:** `false`
1280+
* `recursive` {boolean} **Default:** `false`
12721281
* Returns: {Promise} Fulfills with an array of the names of the files in
12731282
the directory excluding `'.'` and `'..'`.
12741283
@@ -3344,6 +3353,9 @@ Functions based on `fs.open()` exhibit this behavior as well:
33443353
<!-- YAML
33453354
added: v12.12.0
33463355
changes:
3356+
- version: REPLACEME
3357+
pr-url: https://github.com/nodejs/node/pull/41439
3358+
description: Added `recursive` option.
33473359
- version: v18.0.0
33483360
pr-url: https://github.com/nodejs/node/pull/41678
33493361
description: Passing an invalid callback to the `callback` argument
@@ -3362,6 +3374,7 @@ changes:
33623374
* `bufferSize` {number} Number of directory entries that are buffered
33633375
internally when reading from the directory. Higher values lead to better
33643376
performance but higher memory usage. **Default:** `32`
3377+
* `recursive` {boolean} **Default:** `false`
33653378
* `callback` {Function}
33663379
* `err` {Error}
33673380
* `dir` {fs.Dir}
@@ -3478,6 +3491,9 @@ above values.
34783491
<!-- YAML
34793492
added: v0.1.8
34803493
changes:
3494+
- version: REPLACEME
3495+
pr-url: https://github.com/nodejs/node/pull/41439
3496+
description: Added `recursive` option.
34813497
- version: v18.0.0
34823498
pr-url: https://github.com/nodejs/node/pull/41678
34833499
description: Passing an invalid callback to the `callback` argument
@@ -3507,6 +3523,7 @@ changes:
35073523
* `options` {string|Object}
35083524
* `encoding` {string} **Default:** `'utf8'`
35093525
* `withFileTypes` {boolean} **Default:** `false`
3526+
* `recursive` {boolean} **Default:** `false`
35103527
* `callback` {Function}
35113528
* `err` {Error}
35123529
* `files` {string\[]|Buffer\[]|fs.Dirent\[]}
@@ -5470,6 +5487,9 @@ object with an `encoding` property specifying the character encoding to use.
54705487
<!-- YAML
54715488
added: v12.12.0
54725489
changes:
5490+
- version: REPLACEME
5491+
pr-url: https://github.com/nodejs/node/pull/41439
5492+
description: Added `recursive` option.
54735493
- version:
54745494
- v13.1.0
54755495
- v12.16.0
@@ -5483,6 +5503,7 @@ changes:
54835503
* `bufferSize` {number} Number of directory entries that are buffered
54845504
internally when reading from the directory. Higher values lead to better
54855505
performance but higher memory usage. **Default:** `32`
5506+
* `recursive` {boolean} **Default:** `false`
54865507
* Returns: {fs.Dir}
54875508
54885509
Synchronously open a directory. See opendir(3).
@@ -5526,6 +5547,9 @@ this API: [`fs.open()`][].
55265547
<!-- YAML
55275548
added: v0.1.21
55285549
changes:
5550+
- version: REPLACEME
5551+
pr-url: https://github.com/nodejs/node/pull/41439
5552+
description: Added `recursive` option.
55295553
- version: v10.10.0
55305554
pr-url: https://github.com/nodejs/node/pull/22020
55315555
description: New option `withFileTypes` was added.
@@ -5539,6 +5563,7 @@ changes:
55395563
* `options` {string|Object}
55405564
* `encoding` {string} **Default:** `'utf8'`
55415565
* `withFileTypes` {boolean} **Default:** `false`
5566+
* `recursive` {boolean} **Default:** `false`
55425567
* Returns: {string\[]|Buffer\[]|fs.Dirent\[]}
55435568
55445569
Reads the contents of the directory.
@@ -6384,6 +6409,16 @@ The file name that this {fs.Dirent} object refers to. The type of this
63846409
value is determined by the `options.encoding` passed to [`fs.readdir()`][] or
63856410
[`fs.readdirSync()`][].
63866411
6412+
#### `dirent.path`
6413+
6414+
<!-- YAML
6415+
added: REPLACEME
6416+
-->
6417+
6418+
* {string}
6419+
6420+
The base path that this {fs.Dirent} object refers to.
6421+
63876422
### Class: `fs.FSWatcher`
63886423
63896424
<!-- YAML

‎lib/fs.js‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1399,6 +1399,36 @@ function mkdirSync(path, options) {
13991399
}
14001400
}
14011401

1402+
// TODO(Ethan-Arrowood): Make this iterative too
1403+
function readdirSyncRecursive(path, origPath, options) {
1404+
nullCheck(path, 'path', true);
1405+
const ctx = { path };
1406+
const result = binding.readdir(pathModule.toNamespacedPath(path),
1407+
options.encoding, !!options.withFileTypes, undefined, ctx);
1408+
handleErrorFromBinding(ctx);
1409+
return options.withFileTypes ?
1410+
getDirents(path, result).flatMap((dirent) => {
1411+
return [
1412+
dirent,
1413+
...(dirent.isDirectory() ?
1414+
readdirSyncRecursive(
1415+
pathModule.join(path, dirent.name),
1416+
origPath,
1417+
options,
1418+
) : []),
1419+
];
1420+
}) :
1421+
result.flatMap((ent) => {
1422+
const innerPath = pathModule.join(path, ent);
1423+
const relativePath = pathModule.relative(origPath, innerPath);
1424+
const stat = binding.internalModuleStat(innerPath);
1425+
return [
1426+
relativePath,
1427+
...(stat === 1 ? readdirSyncRecursive(innerPath, origPath, options) : []),
1428+
];
1429+
});
1430+
}
1431+
14021432
/**
14031433
* Reads the contents of a directory.
14041434
* @param {string | Buffer | URL} path
@@ -1416,6 +1446,14 @@ function readdir(path, options, callback) {
14161446
callback = makeCallback(typeof options === 'function' ? options : callback);
14171447
options = getOptions(options);
14181448
path = getValidatedPath(path);
1449+
if (options.recursive != null) {
1450+
validateBoolean(options.recursive, 'options.recursive');
1451+
}
1452+
1453+
if (options.recursive) {
1454+
callback(null, readdirSyncRecursive(path, path, options));
1455+
return;
1456+
}
14191457

14201458
const req = new FSReqCallback();
14211459
if (!options.withFileTypes) {
@@ -1439,12 +1477,21 @@ function readdir(path, options, callback) {
14391477
* @param {string | {
14401478
* encoding?: string;
14411479
* withFileTypes?: boolean;
1480+
* recursive?: boolean;
14421481
* }} [options]
14431482
* @returns {string | Buffer[] | Dirent[]}
14441483
*/
14451484
function readdirSync(path, options) {
14461485
options = getOptions(options);
14471486
path = getValidatedPath(path);
1487+
if (options.recursive != null) {
1488+
validateBoolean(options.recursive, 'options.recursive');
1489+
}
1490+
1491+
if (options.recursive) {
1492+
return readdirSyncRecursive(path, path, options);
1493+
}
1494+
14481495
const ctx = { path };
14491496
const result = binding.readdir(pathModule.toNamespacedPath(path),
14501497
options.encoding, !!options.withFileTypes,

‎lib/internal/fs/dir.js‎

Lines changed: 77 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22

33
const {
44
ArrayPrototypePush,
5-
ArrayPrototypeSlice,
6-
ArrayPrototypeSplice,
5+
ArrayPrototypeShift,
76
FunctionPrototypeBind,
87
ObjectDefineProperty,
98
PromiseReject,
@@ -99,13 +98,21 @@ class Dir {
9998
}
10099

101100
if (this[kDirBufferedEntries].length > 0) {
102-
const { 0: name, 1: type } =
103-
ArrayPrototypeSplice(this[kDirBufferedEntries], 0, 2);
104-
if (maybeSync)
105-
process.nextTick(getDirent, this[kDirPath], name, type, callback);
106-
else
107-
getDirent(this[kDirPath], name, type, callback);
108-
return;
101+
try {
102+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
103+
104+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
105+
this.readSyncRecursive(dirent);
106+
}
107+
108+
if (maybeSync)
109+
process.nextTick(callback, null, dirent);
110+
else
111+
callback(null, dirent);
112+
return;
113+
} catch (error) {
114+
return callback(error);
115+
}
109116
}
110117

111118
const req = new FSReqCallback();
@@ -120,8 +127,16 @@ class Dir {
120127
return callback(err, result);
121128
}
122129

123-
this[kDirBufferedEntries] = ArrayPrototypeSlice(result, 2);
124-
getDirent(this[kDirPath], result[0], result[1], callback);
130+
try {
131+
this.processReadResult(this[kDirPath], result);
132+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
133+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
134+
this.readSyncRecursive(dirent);
135+
}
136+
callback(null, dirent);
137+
} catch (error) {
138+
callback(error);
139+
}
125140
};
126141

127142
this[kDirOperationQueue] = [];
@@ -132,6 +147,45 @@ class Dir {
132147
);
133148
}
134149

150+
processReadResult(path, result) {
151+
for (let i = 0; i < result.length; i += 2) {
152+
ArrayPrototypePush(
153+
this[kDirBufferedEntries],
154+
getDirent(
155+
pathModule.join(path, result[i]),
156+
result[i],
157+
result[i + 1],
158+
),
159+
);
160+
}
161+
}
162+
163+
// TODO(Ethan-Arrowood): Review this implementation. Make it iterative.
164+
// Can we better leverage the `kDirOperationQueue`?
165+
readSyncRecursive(dirent) {
166+
const ctx = { path: dirent.path };
167+
const handle = dirBinding.opendir(
168+
pathModule.toNamespacedPath(dirent.path),
169+
this[kDirOptions].encoding,
170+
undefined,
171+
ctx,
172+
);
173+
handleErrorFromBinding(ctx);
174+
const result = handle.read(
175+
this[kDirOptions].encoding,
176+
this[kDirOptions].bufferSize,
177+
undefined,
178+
ctx,
179+
);
180+
181+
if (result) {
182+
this.processReadResult(dirent.path, result);
183+
}
184+
185+
handle.close(undefined, ctx);
186+
handleErrorFromBinding(ctx);
187+
}
188+
135189
readSync() {
136190
if (this[kDirClosed] === true) {
137191
throw new ERR_DIR_CLOSED();
@@ -142,9 +196,11 @@ class Dir {
142196
}
143197

144198
if (this[kDirBufferedEntries].length > 0) {
145-
const { 0: name, 1: type } =
146-
ArrayPrototypeSplice(this[kDirBufferedEntries], 0, 2);
147-
return getDirent(this[kDirPath], name, type);
199+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
200+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
201+
this.readSyncRecursive(dirent);
202+
}
203+
return dirent;
148204
}
149205

150206
const ctx = { path: this[kDirPath] };
@@ -160,8 +216,13 @@ class Dir {
160216
return result;
161217
}
162218

163-
this[kDirBufferedEntries] = ArrayPrototypeSlice(result, 2);
164-
return getDirent(this[kDirPath], result[0], result[1]);
219+
this.processReadResult(this[kDirPath], result);
220+
221+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
222+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
223+
this.readSyncRecursive(dirent);
224+
}
225+
return dirent;
165226
}
166227

167228
close(callback) {

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL