FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

tools: run CI with shared libs on GHA · nodejs/node@5d5c848 · GitHub

/ node Public

Commit 5d5c848

Browse files
aduh95
committed
tools: run CI with shared libs on GHA
PR-URL: #60121 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
1 parent 7a6542c commit 5d5c848

10 files changed

Lines changed: 359 additions & 1 deletion

File tree

‎.github/workflows/linters.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,22 @@ jobs:
140140
NODE=$(command -v node) make lint-md
141141
env:
142142
NODE_RELEASED_VERSIONS: ${{ steps.get-released-versions.outputs.NODE_RELEASED_VERSIONS }}
143+
lint-nix:
144+
if: github.event.pull_request.draft == false
145+
runs-on: ubuntu-latest
146+
steps:
147+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
148+
with:
149+
persist-credentials: false
150+
sparse-checkout: '*.nix'
151+
sparse-checkout-cone-mode: false
152+
- uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
153+
- name: Lint Nix files
154+
run: nix-shell -I nixpkgs=./tools/nix/pkgs.nix -p 'nixfmt-tree' --run 'treefmt --quiet --fail-on-change'
155+
- if: ${{ failure() }}
156+
name: Show diff
157+
run: git --no-pager diff
158+
143159
lint-py:
144160
if: github.event.pull_request.draft == false
145161
runs-on: ubuntu-latest

‎.github/workflows/test-linux.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on:
55
paths-ignore:
66
- .mailmap
77
- README.md
8+
- '*.nix'
89
- .github/**
910
- '!.github/workflows/test-linux.yml'
1011
types: [opened, synchronize, reopened, ready_for_review]
@@ -17,6 +18,7 @@ on:
1718
paths-ignore:
1819
- .mailmap
1920
- README.md
21+
- '*.nix'
2022
- .github/**
2123
- '!.github/workflows/test-linux.yml'
2224

‎.github/workflows/test-macos.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ on:
66
paths-ignore:
77
- .mailmap
88
- '**.md'
9+
- '*.nix'
910
- AUTHORS
1011
- doc/**
1112
- .github/**
@@ -19,6 +20,7 @@ on:
1920
paths-ignore:
2021
- .mailmap
2122
- '**.md'
23+
- '*.nix'
2224
- AUTHORS
2325
- doc/**
2426
- .github/**

‎.github/workflows/test-shared.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Test Shared librairies
2+
3+
on:
4+
pull_request:
5+
paths-ignore:
6+
- .mailmap
7+
- '**.md'
8+
- AUTHORS
9+
- doc/**
10+
- .github/**
11+
- '!.github/workflows/test-shared.yml'
12+
types: [opened, synchronize, reopened, ready_for_review]
13+
push:
14+
branches:
15+
- main
16+
- canary
17+
- v[0-9]+.x-staging
18+
- v[0-9]+.x
19+
paths-ignore:
20+
- .mailmap
21+
- '**.md'
22+
- AUTHORS
23+
- doc/**
24+
- .github/**
25+
- '!.github/workflows/test-shared.yml'
26+
27+
concurrency:
28+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
29+
cancel-in-progress: true
30+
31+
env:
32+
FLAKY_TESTS: keep_retrying
33+
34+
permissions:
35+
contents: read
36+
37+
jobs:
38+
build:
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
include:
43+
- runner: ubuntu-24.04
44+
system: x86_64-linux
45+
- runner: ubuntu-24.04-arm
46+
system: aarch64-linux
47+
- runner: macos-13
48+
system: x86_64-darwin
49+
- runner: macos-latest
50+
system: aarch64-darwin
51+
name: '${{ matrix.system }}: with shared libraries'
52+
runs-on: ${{ matrix.runner }}
53+
steps:
54+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
55+
with:
56+
persist-credentials: false
57+
58+
- uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
59+
with:
60+
extra_nix_config: sandbox = true
61+
62+
- name: Configure sccache
63+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
64+
with:
65+
script: |
66+
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
67+
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
68+
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
69+
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
70+
71+
- name: Build Node.js and run tests
72+
run: |
73+
nix-shell \
74+
-I nixpkgs=./tools/nix/pkgs.nix \
75+
--pure --keep FLAKY_TESTS \
76+
--keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
77+
--arg loadJSBuiltinsDynamically false \
78+
--arg ccache '(import <nixpkgs> {}).sccache' \
79+
--arg devTools '[]' \
80+
--arg benchmarkTools '[]' \
81+
--run '
82+
make run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
83+
'

‎.github/workflows/tools.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ on:
3030
- llhttp
3131
- minimatch
3232
- nbytes
33+
- nixpkgs-unstable
3334
- nghttp2
3435
- nghttp3
3536
- ngtcp2
@@ -190,6 +191,14 @@ jobs:
190191
cat temp-output
191192
tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true
192193
rm temp-output
194+
- id: nixpkgs-unstable
195+
subsystem: tools
196+
label: tools
197+
run: |
198+
./tools/dep_updaters/update-nixpkgs-pin.sh > temp-output
199+
cat temp-output
200+
tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true
201+
rm temp-output
193202
- id: nghttp2
194203
subsystem: deps
195204
label: dependencies
@@ -292,6 +301,9 @@ jobs:
292301
with:
293302
python-version: ${{ env.PYTHON_VERSION }}
294303
allow-prereleases: true
304+
- name: Set up Nix
305+
if: matrix.id == 'nixpkgs-unstable' && (github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id)
306+
uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
295307
- run: ${{ matrix.run }}
296308
if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id
297309
env:

‎BUILDING.md‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ Consult previous versions of this document for older versions of Node.js:
239239

240240
Installation via Linux package manager can be achieved with:
241241

242+
* Nix, NixOS: `nix-shell`
242243
* Ubuntu, Debian: `sudo apt-get install python3 g++-12 gcc-12 make python3-pip`
243244
* Fedora: `sudo dnf install python3 gcc-c++ make python3-pip`
244245
* CentOS and RHEL: `sudo yum install python3 gcc-c++ make python3-pip`
@@ -259,6 +260,75 @@ installed, you can find them under the menu `Xcode -> Open Developer Tool ->
259260
More Developer Tools...`. This step will install `clang`, `clang++`, and
260261
`make`.
261262

263+
#### Nix integration
264+
265+
If you are using Nix and direnv, you can use the following to get started:
266+
267+
```bash
268+
echo 'use_nix --arg sharedLibDeps {} --argstr icu small' > .envrc
269+
direnv allow .
270+
make build-ci -j12
271+
```
272+
273+
The use of `make build-ci` is to ensure you are using the `CONFIG_FLAGS`
274+
environment variable. You can also specify it manually:
275+
276+
```bash
277+
./configure $CONFIG_FLAGS
278+
make -j12
279+
```
280+
281+
Passing the `--arg sharedLibDeps {}` instructs direnv and Nix to generate an
282+
environment that uses the vendored-in native dependencies. Using the vendored-in
283+
dependencies result in a result closer to the official binaries, the tradeoff
284+
being the build will take longer to complete as you'd have to build those
285+
dependencies instead of using the cached ones from the Nix cache. You can omit
286+
that flag to use all the shared dependencies, or specify only some dependencies:
287+
288+
```bash
289+
cat -> .envrc <<'EOF'
290+
use nix --arg sharedLibDeps '{
291+
inherit (import <nixpkgs> {})
292+
openssl
293+
zlib
294+
;
295+
}'
296+
EOF
297+
```
298+
299+
Passing the `--argstr icu small` instructs direnv and Nix to pass `--with-intl=small` in
300+
the `CONFIG_FLAGS` environment variable. If you omit this, the prebuilt ICU from Nix cache
301+
will be used, which should speed up greatly compilation time.
302+
303+
The use of `direnv` is completely optional, you can also use `nix-shell` directly,
304+
e.g. here's a command you can use to build a binary for benchmarking purposes:
305+
306+
```bash
307+
# Passing `--arg loadJSBuiltinsDynamically false` to instruct the compiler to
308+
# embed the JS core files so it is no longer affected by local changes
309+
# (necessary for getting useful benchmark results).
310+
# Passing `--arg devTools '[]' --arg benchmarkTools '[]'` since we don't need
311+
# those to build node.
312+
nix-shell \
313+
--arg loadJSBuiltinsDynamically false \
314+
--arg devTools '[]' --arg benchmarkTools '[]' \
315+
--run 'make build-ci -j12'
316+
317+
mv out/Release/node ./node_old
318+
319+
# ...
320+
# Make your local changes, and re-build node
321+
322+
nix-shell \
323+
--arg loadJSBuiltinsDynamically false \
324+
--arg devTools '[]' --arg benchmarkTools '[]' \
325+
--run 'make build-ci -j12'
326+
327+
nix-shell --pure --run './node benchmark/compare.js --old ./node_old --new ./node http | Rscript benchmark/compare.R'
328+
```
329+
330+
There are additional attributes you can pass, see `shell.nix` file for more details.
331+
262332
#### Building Node.js
263333

264334
If the path to your build directory contains a space, the build will likely
@@ -267,7 +337,6 @@ fail.
267337
To build Node.js:
268338

269339
```bash
270-
export CXX=g++-12
271340
./configure
272341
make -j4
273342
```

‎doc/contributing/writing-and-running-benchmarks.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ Basic Unix tools are required for some benchmarks.
2626
[Git for Windows][git-for-windows] includes Git Bash and the necessary tools,
2727
which need to be included in the global Windows `PATH`.
2828

29+
If you are using Nix, all the required tools are already listed in the
30+
`benchmarkTools` argument of the `shell.nix` file, so you can skip those
31+
prerequesites.
32+
2933
### HTTP benchmark requirements
3034

3135
Most of the HTTP benchmarks require a benchmarker to be installed. This can be

‎shell.nix‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
{
2+
pkgs ? import "${./tools/nix/pkgs.nix}" { },
3+
loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding
4+
ncu-path ? null, # Provide this if you want to use a local version of NCU
5+
icu ? pkgs.icu,
6+
sharedLibDeps ? {
7+
inherit (pkgs)
8+
ada
9+
brotli
10+
c-ares
11+
libuv
12+
nghttp2
13+
nghttp3
14+
ngtcp2
15+
openssl
16+
simdjson
17+
simdutf
18+
sqlite
19+
uvwasi
20+
zlib
21+
zstd
22+
;
23+
http-parser = pkgs.llhttp;
24+
},
25+
ccache ? pkgs.ccache,
26+
ninja ? pkgs.ninja,
27+
devTools ? [
28+
pkgs.curl
29+
pkgs.gh
30+
pkgs.git
31+
pkgs.jq
32+
pkgs.shellcheck
33+
]
34+
++ (
35+
if (ncu-path == null) then
36+
[ pkgs.node-core-utils ]
37+
else
38+
[
39+
(pkgs.writeShellScriptBin "git-node" "exec \"${ncu-path}/bin/git-node.js\" \"$@\"")
40+
(pkgs.writeShellScriptBin "ncu-ci" "exec \"${ncu-path}/bin/ncu-ci.js\" \"$@\"")
41+
(pkgs.writeShellScriptBin "ncu-config" "exec \"${ncu-path}/bin/ncu-config.js\" \"$@\"")
42+
]
43+
),
44+
benchmarkTools ? [
45+
pkgs.R
46+
pkgs.rPackages.ggplot2
47+
pkgs.rPackages.plyr
48+
pkgs.wrk
49+
],
50+
extraConfigFlags ? [
51+
"--without-npm"
52+
"--debug-node"
53+
],
54+
}:
55+
56+
let
57+
useSharedICU = if builtins.isString icu then icu == "system" else icu != null;
58+
useSharedAda = builtins.hasAttr "ada" sharedLibDeps;
59+
useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps;
60+
in
61+
pkgs.mkShell {
62+
inherit (pkgs.nodejs_latest) nativeBuildInputs;
63+
64+
buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optionals useSharedICU [ icu ];
65+
66+
packages = [
67+
ccache
68+
]
69+
++ devTools
70+
++ benchmarkTools;
71+
72+
shellHook =
73+
if (ccache != null) then
74+
''
75+
export CC="${pkgs.lib.getExe ccache} $CC"
76+
export CXX="${pkgs.lib.getExe ccache} $CXX"
77+
''
78+
else
79+
"";
80+
81+
BUILD_WITH = if (ninja != null) then "ninja" else "make";
82+
NINJA = if (ninja != null) then "${pkgs.lib.getExe ninja}" else "";
83+
CI_SKIP_TESTS = pkgs.lib.concatStringsSep "," (
84+
[ ]
85+
++ pkgs.lib.optionals useSharedAda [
86+
# Different versions of Ada affect the WPT tests
87+
"test-url"
88+
]
89+
++ pkgs.lib.optionals useSharedOpenSSL [
90+
# Path to the openssl.cnf is different from the expected one
91+
"test-strace-openat-openssl"
92+
]
93+
);
94+
CONFIG_FLAGS = builtins.toString (
95+
[
96+
(
97+
if icu == null then
98+
"--without-intl"
99+
else
100+
"--with-intl=${if useSharedICU then "system" else icu}-icu"
101+
)
102+
]
103+
++ extraConfigFlags
104+
++ pkgs.lib.optionals (ninja != null) [
105+
"--ninja"
106+
]
107+
++ pkgs.lib.optionals loadJSBuiltinsDynamically [
108+
"--node-builtin-modules-path=${builtins.toString ./.}"
109+
]
110+
++ pkgs.lib.concatMap (name: [
111+
"--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}"
112+
"--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-libpath=${
113+
pkgs.lib.getLib sharedLibDeps.${name}
114+
}/lib"
115+
"--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-include=${
116+
pkgs.lib.getInclude sharedLibDeps.${name}
117+
}/include"
118+
]) (builtins.attrNames sharedLibDeps)
119+
);
120+
}

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL