FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fs: add recursive option to readdir and opendir · nodejs/node@439ea47 · GitHub

/ node Public

Commit 439ea47

Browse files
Ethan Arrowood
authored and
targos
committed
fs: add recursive option to readdir and opendir
Adds a naive, linear recursive algorithm for the following methods: readdir, readdirSync, opendir, opendirSync, and the promise based equivalents. Fixes: #34992 PR-URL: #41439 Refs: nodejs/tooling#130 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
1 parent 872e670 commit 439ea47

7 files changed

Lines changed: 659 additions & 31 deletions

File tree

‎doc/api/fs.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1220,6 +1220,9 @@ a colon, Node.js will open a file system stream, as described by
12201220
<!-- YAML
12211221
added: v12.12.0
12221222
changes:
1223+
- version: REPLACEME
1224+
pr-url: https://github.com/nodejs/node/pull/41439
1225+
description: Added `recursive` option.
12231226
- version:
12241227
- v13.1.0
12251228
- v12.16.0
@@ -1233,6 +1236,8 @@ changes:
12331236
* `bufferSize` {number} Number of directory entries that are buffered
12341237
internally when reading from the directory. Higher values lead to better
12351238
performance but higher memory usage. **Default:** `32`
1239+
* `recursive` {boolean} Resolved `Dir` will be an {AsyncIterable}
1240+
containing all sub files and directories. **Default:** `false`
12361241
* Returns: {Promise} Fulfills with an {fs.Dir}.
12371242
12381243
Asynchronously open a directory for iterative scanning. See the POSIX
@@ -1266,6 +1271,9 @@ closed after the iterator exits.
12661271
<!-- YAML
12671272
added: v10.0.0
12681273
changes:
1274+
- version: REPLACEME
1275+
pr-url: https://github.com/nodejs/node/pull/41439
1276+
description: Added `recursive` option.
12691277
- version: v10.11.0
12701278
pr-url: https://github.com/nodejs/node/pull/22020
12711279
description: New option `withFileTypes` was added.
@@ -1275,6 +1283,7 @@ changes:
12751283
* `options` {string|Object}
12761284
* `encoding` {string} **Default:** `'utf8'`
12771285
* `withFileTypes` {boolean} **Default:** `false`
1286+
* `recursive` {boolean} **Default:** `false`
12781287
* Returns: {Promise} Fulfills with an array of the names of the files in
12791288
the directory excluding `'.'` and `'..'`.
12801289
@@ -3402,6 +3411,9 @@ const { openAsBlob } = require('node:fs');
34023411
<!-- YAML
34033412
added: v12.12.0
34043413
changes:
3414+
- version: REPLACEME
3415+
pr-url: https://github.com/nodejs/node/pull/41439
3416+
description: Added `recursive` option.
34053417
- version: v18.0.0
34063418
pr-url: https://github.com/nodejs/node/pull/41678
34073419
description: Passing an invalid callback to the `callback` argument
@@ -3420,6 +3432,7 @@ changes:
34203432
* `bufferSize` {number} Number of directory entries that are buffered
34213433
internally when reading from the directory. Higher values lead to better
34223434
performance but higher memory usage. **Default:** `32`
3435+
* `recursive` {boolean} **Default:** `false`
34233436
* `callback` {Function}
34243437
* `err` {Error}
34253438
* `dir` {fs.Dir}
@@ -3538,6 +3551,9 @@ above values.
35383551
<!-- YAML
35393552
added: v0.1.8
35403553
changes:
3554+
- version: REPLACEME
3555+
pr-url: https://github.com/nodejs/node/pull/41439
3556+
description: Added `recursive` option.
35413557
- version: v18.0.0
35423558
pr-url: https://github.com/nodejs/node/pull/41678
35433559
description: Passing an invalid callback to the `callback` argument
@@ -3567,6 +3583,7 @@ changes:
35673583
* `options` {string|Object}
35683584
* `encoding` {string} **Default:** `'utf8'`
35693585
* `withFileTypes` {boolean} **Default:** `false`
3586+
* `recursive` {boolean} **Default:** `false`
35703587
* `callback` {Function}
35713588
* `err` {Error}
35723589
* `files` {string\[]|Buffer\[]|fs.Dirent\[]}
@@ -5543,6 +5560,9 @@ object with an `encoding` property specifying the character encoding to use.
55435560
<!-- YAML
55445561
added: v12.12.0
55455562
changes:
5563+
- version: REPLACEME
5564+
pr-url: https://github.com/nodejs/node/pull/41439
5565+
description: Added `recursive` option.
55465566
- version:
55475567
- v13.1.0
55485568
- v12.16.0
@@ -5556,6 +5576,7 @@ changes:
55565576
* `bufferSize` {number} Number of directory entries that are buffered
55575577
internally when reading from the directory. Higher values lead to better
55585578
performance but higher memory usage. **Default:** `32`
5579+
* `recursive` {boolean} **Default:** `false`
55595580
* Returns: {fs.Dir}
55605581
55615582
Synchronously open a directory. See opendir(3).
@@ -5599,6 +5620,9 @@ this API: [`fs.open()`][].
55995620
<!-- YAML
56005621
added: v0.1.21
56015622
changes:
5623+
- version: REPLACEME
5624+
pr-url: https://github.com/nodejs/node/pull/41439
5625+
description: Added `recursive` option.
56025626
- version: v10.10.0
56035627
pr-url: https://github.com/nodejs/node/pull/22020
56045628
description: New option `withFileTypes` was added.
@@ -5612,6 +5636,7 @@ changes:
56125636
* `options` {string|Object}
56135637
* `encoding` {string} **Default:** `'utf8'`
56145638
* `withFileTypes` {boolean} **Default:** `false`
5639+
* `recursive` {boolean} **Default:** `false`
56155640
* Returns: {string\[]|Buffer\[]|fs.Dirent\[]}
56165641
56175642
Reads the contents of the directory.
@@ -6465,6 +6490,16 @@ The file name that this {fs.Dirent} object refers to. The type of this
64656490
value is determined by the `options.encoding` passed to [`fs.readdir()`][] or
64666491
[`fs.readdirSync()`][].
64676492
6493+
#### `dirent.path`
6494+
6495+
<!-- YAML
6496+
added: REPLACEME
6497+
-->
6498+
6499+
* {string}
6500+
6501+
The base path that this {fs.Dirent} object refers to.
6502+
64686503
### Class: `fs.FSWatcher`
64696504
64706505
<!-- YAML

‎lib/fs.js‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1404,6 +1404,36 @@ function mkdirSync(path, options) {
14041404
}
14051405
}
14061406

1407+
// TODO(Ethan-Arrowood): Make this iterative too
1408+
function readdirSyncRecursive(path, origPath, options) {
1409+
nullCheck(path, 'path', true);
1410+
const ctx = { path };
1411+
const result = binding.readdir(pathModule.toNamespacedPath(path),
1412+
options.encoding, !!options.withFileTypes, undefined, ctx);
1413+
handleErrorFromBinding(ctx);
1414+
return options.withFileTypes ?
1415+
getDirents(path, result).flatMap((dirent) => {
1416+
return [
1417+
dirent,
1418+
...(dirent.isDirectory() ?
1419+
readdirSyncRecursive(
1420+
pathModule.join(path, dirent.name),
1421+
origPath,
1422+
options,
1423+
) : []),
1424+
];
1425+
}) :
1426+
result.flatMap((ent) => {
1427+
const innerPath = pathModule.join(path, ent);
1428+
const relativePath = pathModule.relative(origPath, innerPath);
1429+
const stat = binding.internalModuleStat(innerPath);
1430+
return [
1431+
relativePath,
1432+
...(stat === 1 ? readdirSyncRecursive(innerPath, origPath, options) : []),
1433+
];
1434+
});
1435+
}
1436+
14071437
/**
14081438
* Reads the contents of a directory.
14091439
* @param {string | Buffer | URL} path
@@ -1421,6 +1451,14 @@ function readdir(path, options, callback) {
14211451
callback = makeCallback(typeof options === 'function' ? options : callback);
14221452
options = getOptions(options);
14231453
path = getValidatedPath(path);
1454+
if (options.recursive != null) {
1455+
validateBoolean(options.recursive, 'options.recursive');
1456+
}
1457+
1458+
if (options.recursive) {
1459+
callback(null, readdirSyncRecursive(path, path, options));
1460+
return;
1461+
}
14241462

14251463
const req = new FSReqCallback();
14261464
if (!options.withFileTypes) {
@@ -1444,12 +1482,21 @@ function readdir(path, options, callback) {
14441482
* @param {string | {
14451483
* encoding?: string;
14461484
* withFileTypes?: boolean;
1485+
* recursive?: boolean;
14471486
* }} [options]
14481487
* @returns {string | Buffer[] | Dirent[]}
14491488
*/
14501489
function readdirSync(path, options) {
14511490
options = getOptions(options);
14521491
path = getValidatedPath(path);
1492+
if (options.recursive != null) {
1493+
validateBoolean(options.recursive, 'options.recursive');
1494+
}
1495+
1496+
if (options.recursive) {
1497+
return readdirSyncRecursive(path, path, options);
1498+
}
1499+
14531500
const ctx = { path };
14541501
const result = binding.readdir(pathModule.toNamespacedPath(path),
14551502
options.encoding, !!options.withFileTypes,

‎lib/internal/fs/dir.js‎

Lines changed: 77 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22

33
const {
44
ArrayPrototypePush,
5-
ArrayPrototypeSlice,
6-
ArrayPrototypeSplice,
5+
ArrayPrototypeShift,
76
FunctionPrototypeBind,
87
ObjectDefineProperty,
98
PromiseReject,
@@ -99,13 +98,21 @@ class Dir {
9998
}
10099

101100
if (this[kDirBufferedEntries].length > 0) {
102-
const { 0: name, 1: type } =
103-
ArrayPrototypeSplice(this[kDirBufferedEntries], 0, 2);
104-
if (maybeSync)
105-
process.nextTick(getDirent, this[kDirPath], name, type, callback);
106-
else
107-
getDirent(this[kDirPath], name, type, callback);
108-
return;
101+
try {
102+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
103+
104+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
105+
this.readSyncRecursive(dirent);
106+
}
107+
108+
if (maybeSync)
109+
process.nextTick(callback, null, dirent);
110+
else
111+
callback(null, dirent);
112+
return;
113+
} catch (error) {
114+
return callback(error);
115+
}
109116
}
110117

111118
const req = new FSReqCallback();
@@ -120,8 +127,16 @@ class Dir {
120127
return callback(err, result);
121128
}
122129

123-
this[kDirBufferedEntries] = ArrayPrototypeSlice(result, 2);
124-
getDirent(this[kDirPath], result[0], result[1], callback);
130+
try {
131+
this.processReadResult(this[kDirPath], result);
132+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
133+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
134+
this.readSyncRecursive(dirent);
135+
}
136+
callback(null, dirent);
137+
} catch (error) {
138+
callback(error);
139+
}
125140
};
126141

127142
this[kDirOperationQueue] = [];
@@ -132,6 +147,45 @@ class Dir {
132147
);
133148
}
134149

150+
processReadResult(path, result) {
151+
for (let i = 0; i < result.length; i += 2) {
152+
ArrayPrototypePush(
153+
this[kDirBufferedEntries],
154+
getDirent(
155+
pathModule.join(path, result[i]),
156+
result[i],
157+
result[i + 1],
158+
),
159+
);
160+
}
161+
}
162+
163+
// TODO(Ethan-Arrowood): Review this implementation. Make it iterative.
164+
// Can we better leverage the `kDirOperationQueue`?
165+
readSyncRecursive(dirent) {
166+
const ctx = { path: dirent.path };
167+
const handle = dirBinding.opendir(
168+
pathModule.toNamespacedPath(dirent.path),
169+
this[kDirOptions].encoding,
170+
undefined,
171+
ctx,
172+
);
173+
handleErrorFromBinding(ctx);
174+
const result = handle.read(
175+
this[kDirOptions].encoding,
176+
this[kDirOptions].bufferSize,
177+
undefined,
178+
ctx,
179+
);
180+
181+
if (result) {
182+
this.processReadResult(dirent.path, result);
183+
}
184+
185+
handle.close(undefined, ctx);
186+
handleErrorFromBinding(ctx);
187+
}
188+
135189
readSync() {
136190
if (this[kDirClosed] === true) {
137191
throw new ERR_DIR_CLOSED();
@@ -142,9 +196,11 @@ class Dir {
142196
}
143197

144198
if (this[kDirBufferedEntries].length > 0) {
145-
const { 0: name, 1: type } =
146-
ArrayPrototypeSplice(this[kDirBufferedEntries], 0, 2);
147-
return getDirent(this[kDirPath], name, type);
199+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
200+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
201+
this.readSyncRecursive(dirent);
202+
}
203+
return dirent;
148204
}
149205

150206
const ctx = { path: this[kDirPath] };
@@ -160,8 +216,13 @@ class Dir {
160216
return result;
161217
}
162218

163-
this[kDirBufferedEntries] = ArrayPrototypeSlice(result, 2);
164-
return getDirent(this[kDirPath], result[0], result[1]);
219+
this.processReadResult(this[kDirPath], result);
220+
221+
const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
222+
if (this[kDirOptions].recursive && dirent.isDirectory()) {
223+
this.readSyncRecursive(dirent);
224+
}
225+
return dirent;
165226
}
166227

167228
close(callback) {

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL