FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fs: copy directory trees for fs.cp() on the thread pool · nodejs/node@45406cc · GitHub

/ node Public

Commit 45406cc

Browse files
committed
fs: copy directory trees for fs.cp() on the thread pool
fs.cp() and fs.promises.cp() walked the tree in JavaScript with several thread pool round trips per entry (opendir batches, two stat()s, the copyFile(), a chmod()), all awaited in sequence: a 2 100-file tree took ~215 ms with ~110 ms of that on the main thread, against ~36 ms for fs.cpSync(), which copies the tree in C++ when no filter is given. Factor that C++ walk into CopyDirRecursive(), which records the error instead of throwing so that it can run on any thread, and run it as one ThreadPoolWork request (CpDirJob) for fs.cp()/fs.promises.cp() when the destination directory does not exist yet and nothing has to run per entry (no filter, no dereference, permission model off). Copying into an existing tree keeps the JavaScript walk and its rules for what may already be there. The same tree now takes ~30 ms with under 1 ms on the main thread. For that job the walk follows the JavaScript walk's rules rather than cpSync's: it creates every directory with mkdir() and every file with an exclusive uv_fs_copyfile() (honouring the copyFile() mode flags) and fails with EEXIST if anything has appeared in their place since the JavaScript check, so it never opens or follows something it did not create; sockets, FIFOs and unknown entries are reported back to JavaScript, which rejects them with the same SystemErrors as before; relative link targets are made absolute lexically as path.resolve() does. cpSync keeps merging into existing directories, skipping special files and canonicalizing link targets. The walk now uses the error_code overloads of std::filesystem throughout (directory iteration included), so an unreadable directory inside the tree is reported as EACCES by both cp() and cpSync() instead of terminating the process, which cpSync() has done since the walk moved to C++. Filesystem errors raised inside the walk keep their codes, with 'cp' or 'copyfile'/'mkdir' as the syscall. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com>
1 parent b533509 commit 45406cc

8 files changed

Lines changed: 709 additions & 212 deletions

‎benchmark/fs/bench-cp.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
'use strict';
2+
3+
// fs.promises.cp() of a directory tree.
4+
5+
const common = require('../common');
6+
const fs = require('fs');
7+
const path = require('path');
8+
const tmpdir = require('../../test/common/tmpdir');
9+
10+
const bench = common.createBenchmark(main, {
11+
files: [500],
12+
n: [3],
13+
});
14+
15+
function prepareSource(files) {
16+
const src = tmpdir.resolve('cp-src');
17+
for (let i = 0; i < files; i++) {
18+
const dir = path.join(src, `dir-${i % 10}`, `sub-${i % 7}`);
19+
fs.mkdirSync(dir, { recursive: true });
20+
fs.writeFileSync(path.join(dir, `file-${i}.js`), 'x'.repeat(1024 + (i % 512)));
21+
}
22+
return src;
23+
}
24+
25+
async function main({ files, n }) {
26+
tmpdir.refresh();
27+
const src = prepareSource(files);
28+
bench.start();
29+
for (let i = 0; i < n; i++) {
30+
await fs.promises.cp(src, tmpdir.resolve(`cp-dest-${i}`), { recursive: true });
31+
}
32+
bench.end(n);
33+
}

‎lib/internal/fs/cp/cp.js‎

Lines changed: 45 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ const {
66
ArrayPrototypeEvery,
77
ArrayPrototypeFilter,
88
Boolean,
9+
ErrorCaptureStackTrace,
10+
Promise,
911
PromisePrototypeThen,
1012
PromiseReject,
1113
SafePromiseAll,
@@ -55,6 +57,7 @@ const {
5557
sep,
5658
} = require('path');
5759
const fsBinding = internalBinding('fs');
60+
const permission = require('internal/process/permission');
5861

5962
async function cpFn(src, dest, opts) {
6063
// Warn about using preserveTimestamps on 32-bit node
@@ -211,30 +214,19 @@ async function getStatsForCopy(destStat, src, dest, opts) {
211214
return onFile(srcStat, destStat, src, dest, opts);
212215
} else if (srcStat.isSymbolicLink()) {
213216
return onLink(destStat, src, dest, opts);
214-
} else if (srcStat.isSocket()) {
215-
throw new ERR_FS_CP_SOCKET({
216-
message: `cannot copy a socket file: ${dest}`,
217-
path: dest,
218-
syscall: 'cp',
219-
errno: EINVAL,
220-
code: 'EINVAL',
221-
});
222-
} else if (srcStat.isFIFO()) {
223-
throw new ERR_FS_CP_FIFO_PIPE({
224-
message: `cannot copy a FIFO pipe: ${dest}`,
225-
path: dest,
226-
syscall: 'cp',
227-
errno: EINVAL,
228-
code: 'EINVAL',
229-
});
230217
}
231-
throw new ERR_FS_CP_UNKNOWN({
232-
message: `cannot copy an unknown file type: ${dest}`,
233-
path: dest,
234-
syscall: 'cp',
235-
errno: EINVAL,
236-
code: 'EINVAL',
237-
});
218+
throw errorForSpecialFile(srcStat.isSocket() ? 'socket' : srcStat.isFIFO() ? 'fifo' : 'unknown', dest);
219+
}
220+
221+
function errorForSpecialFile(kind, dest) {
222+
const info = { path: dest, syscall: 'cp', errno: EINVAL, code: 'EINVAL' };
223+
if (kind === 'socket') {
224+
return new ERR_FS_CP_SOCKET({ message: `cannot copy a socket file: ${dest}`, ...info });
225+
}
226+
if (kind === 'fifo') {
227+
return new ERR_FS_CP_FIFO_PIPE({ message: `cannot copy a FIFO pipe: ${dest}`, ...info });
228+
}
229+
return new ERR_FS_CP_UNKNOWN({ message: `cannot copy an unknown file type: ${dest}`, ...info });
238230
}
239231

240232
function onFile(srcStat, destStat, src, dest, opts) {
@@ -315,11 +307,41 @@ async function onDir(srcStat, destStat, src, dest, opts) {
315307
}
316308

317309
async function mkDirAndCopy(srcMode, src, dest, opts) {
310+
// A destination directory that does not exist yet is filled in one thread
311+
// pool request by the walk fs.cpSync() uses, unless a filter has to run per
312+
// entry, links inside the tree must be dereferenced, or the permission model
313+
// has to check each path. Copying into an existing tree keeps the per-entry
314+
// walk below and its rules for what may already be there.
315+
if (!opts.filter && !opts.dereference && !permission.isEnabled()) {
316+
// Creates dest itself, with the mode of src.
317+
return copyDirNative(src, dest, opts);
318+
}
318319
await mkdir(dest);
319320
await copyDir(src, dest, opts);
320321
return setDestMode(dest, srcMode);
321322
}
322323

324+
function copyDirNative(src, dest, opts) {
325+
return new Promise((resolve, reject) => {
326+
const job = new fsBinding.CpDirJob(src, dest, opts.force, opts.dereference, opts.errorOnExist,
327+
opts.verbatimSymlinks, opts.preserveTimestamps, opts.mode);
328+
// Sockets, FIFOs and unknown entries come back as (kind, path) so that
329+
// they reject with the same errors as the walk above.
330+
job.ondone = (err, specialFile, specialFilePath) => {
331+
if (specialFile !== undefined) {
332+
err = errorForSpecialFile(specialFile, specialFilePath);
333+
}
334+
if (err != null) {
335+
ErrorCaptureStackTrace(err, copyDirNative);
336+
reject(err);
337+
} else {
338+
resolve();
339+
}
340+
};
341+
job.run();
342+
});
343+
}
344+
323345
async function copyDir(src, dest, opts) {
324346
const dir = await opendir(src);
325347

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL