FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

build: build addon tests in parallel · nodejs/node@896017b · GitHub

/ node Public

Commit 896017b

Browse files
addaleax
authored and
targos
committed
build: build addon tests in parallel
Use a JS script to build addons rather than a shell command embedded in the Makefile, because parallelizing is hard in sh and easy in JS. PR-URL: #21155 Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent a5c386d commit 896017b

2 files changed

Lines changed: 67 additions & 30 deletions

File tree

‎Makefile‎

Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -315,25 +315,14 @@ ADDONS_BINDING_SOURCES := \
315315
# Depends on node-gyp package.json so that build-addons is (re)executed when
316316
# node-gyp is updated as part of an npm update.
317317
test/addons/.buildstamp: config.gypi \
318-
deps/npm/node_modules/node-gyp/package.json \
318+
deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \
319319
$(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \
320320
deps/uv/include/*.h deps/v8/include/*.h \
321321
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
322322
test/addons/.docbuildstamp
323-
# Cannot use $(wildcard test/addons/*/) here, it's evaluated before
324-
# embedded addons have been generated from the documentation.
325-
# Ignore folders without binding.gyp
326-
# (https://github.com/nodejs/node/issues/14843)
327-
@for dirname in test/addons/*/; do \
328-
if [ ! -f "$$PWD/$${dirname}binding.gyp" ]; then \
329-
continue; fi ; \
330-
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
331-
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
332-
--loglevel=$(LOGLEVEL) rebuild \
333-
--python="$(PYTHON)" \
334-
--directory="$$PWD/$$dirname" \
335-
--nodedir="$$PWD" || exit 1 ; \
336-
done
323+
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
324+
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \
325+
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" "$$PWD/test/addons"
337326
touch $@
338327

339328
.PHONY: build-addons
@@ -355,25 +344,15 @@ ADDONS_NAPI_BINDING_SOURCES := \
355344

356345
# Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale.
357346
test/addons-napi/.buildstamp: config.gypi \
358-
deps/npm/node_modules/node-gyp/package.json \
347+
deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \
359348
$(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \
360349
deps/uv/include/*.h deps/v8/include/*.h \
361350
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
362351
src/node_api.h src/node_api_types.h
363-
# Cannot use $(wildcard test/addons-napi/*/) here, it's evaluated before
364-
# embedded addons have been generated from the documentation.
365-
# Ignore folders without binding.gyp
366-
# (https://github.com/nodejs/node/issues/14843)
367-
@for dirname in test/addons-napi/*/; do \
368-
if [ ! -f "$$PWD/$${dirname}binding.gyp" ]; then \
369-
continue; fi ; \
370-
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
371-
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
372-
--loglevel=$(LOGLEVEL) rebuild \
373-
--python="$(PYTHON)" \
374-
--directory="$$PWD/$$dirname" \
375-
--nodedir="$$PWD" || exit 1 ; \
376-
done
352+
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
353+
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \
354+
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \
355+
"$$PWD/test/addons-napi"
377356
touch $@
378357

379358
.PHONY: build-addons-napi

‎tools/build-addons.js‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
'use strict';
2+
3+
// Usage: e.g. node build-addons.js <path to node-gyp> <directory>
4+
5+
const child_process = require('child_process');
6+
const path = require('path');
7+
const fs = require('fs').promises;
8+
const util = require('util');
9+
10+
const execFile = util.promisify(child_process.execFile);
11+
12+
const parallelization = +process.env.JOBS || require('os').cpus().length;
13+
const nodeGyp = process.argv[2];
14+
15+
async function runner(directoryQueue) {
16+
if (directoryQueue.length === 0)
17+
return;
18+
19+
const dir = directoryQueue.shift();
20+
const next = () => runner(directoryQueue);
21+
22+
try {
23+
// Only run for directories that have a `binding.gyp`.
24+
// (https://github.com/nodejs/node/issues/14843)
25+
await fs.stat(path.join(dir, 'binding.gyp'));
26+
} catch (err) {
27+
if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
28+
return next();
29+
throw err;
30+
}
31+
32+
console.log(`Building addon in ${dir}`);
33+
const { stdout, stderr } =
34+
await execFile(process.execPath, [nodeGyp, 'rebuild', `--directory=${dir}`],
35+
{
36+
stdio: 'inherit',
37+
env: { ...process.env, MAKEFLAGS: '-j1' }
38+
});
39+
40+
// We buffer the output and print it out once the process is done in order
41+
// to avoid interleaved output from multiple builds running at once.
42+
process.stdout.write(stdout);
43+
process.stderr.write(stderr);
44+
45+
return next();
46+
}
47+
48+
async function main(directory) {
49+
const directoryQueue = (await fs.readdir(directory))
50+
.map((subdir) => path.join(directory, subdir));
51+
52+
const runners = [];
53+
for (let i = 0; i < parallelization; ++i)
54+
runners.push(runner(directoryQueue));
55+
return Promise.all(runners);
56+
}
57+
58+
main(process.argv[3]).catch((err) => setImmediate(() => { throw err; }));

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL