|
# Benchmark libgit2 against the git reference implementation. |
|
name: Benchmark |
|
|
|
on: |
|
workflow_dispatch: |
|
inputs: |
|
suite: |
|
description: Benchmark suite to run |
|
debug: |
|
type: boolean |
|
description: Debugging output |
|
deploy: |
|
type: boolean |
|
description: Deploy the benchmark site |
|
schedule: |
|
- cron: '15 4 * * *' |
|
|
|
permissions: |
|
contents: read |
|
|
|
jobs: |
|
# Run our benchmarks. We build a matrix with the various build |
|
# targets and their details. Unlike our CI builds, we run these |
|
# directly on the VM instead of in containers since we do not |
|
# need the breadth of platform diversity. |
|
build: |
|
# Only run scheduled workflows on the main repository; prevents people |
|
# from using build minutes on their forks. |
|
if: github.repository == 'libgit2/libgit2' |
|
|
|
strategy: |
|
matrix: |
|
platform: |
|
- name: "Linux (clang, OpenSSL)" |
|
id: linux |
|
os: ubuntu-latest |
|
setup-script: ubuntu |
|
env: |
|
CC: clang |
|
CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DUSE_GSSAPI=ON |
|
- name: "macOS" |
|
id: macos |
|
os: macos-latest |
|
setup-script: osx |
|
env: |
|
CC: clang |
|
CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DUSE_GSSAPI=ON |
|
PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig |
|
- name: "Windows (amd64, Visual Studio)" |
|
id: windows |
|
os: windows-2022 |
|
setup-script: win32 |
|
env: |
|
ARCH: amd64 |
|
CMAKE_GENERATOR: Visual Studio 17 2022 |
|
CMAKE_OPTIONS: -A x64 |
|
fail-fast: false |
|
name: "Benchmark ${{ matrix.platform.name }}" |
|
env: ${{ matrix.platform.env }} |
|
runs-on: ${{ matrix.platform.os }} |
|
steps: |
|
- name: Check out repository |
|
uses: actions/checkout@v4 |
|
with: |
|
path: source |
|
fetch-depth: 0 |
|
- name: Set up benchmark environment |
|
run: source/ci/setup-${{ matrix.platform.setup-script }}-benchmark.sh |
|
shell: bash |
|
if: matrix.platform.setup-script != '' |
|
- name: Clone resource repositories |
|
run: | |
|
# TODO: |
|
# we need a superior way to package the benchmark resources; lfs |
|
# is too expensive |
|
# git lfs install |
|
# git clone https://github.com/libgit2/benchmark-resources resources |
|
|
|
git clone --bare https://github.com/git/git resources/git |
|
|
|
# TODO: |
|
# avoid linux temporarily; the linux blame benchmarks are simply |
|
# too slow to use |
|
# git clone --bare https://github.com/torvalds/linux resources/linux |
|
- name: Build |
|
run: | |
|
mkdir build && cd build |
|
../source/ci/build.sh |
|
env: |
|
CMAKE_GLOBAL_OPTIONS: -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo |
|
CMAKE_BUILD_OPTIONS: --config RelWithDebInfo |
|
shell: bash |
|
- name: Benchmark |
|
run: | |
|
# TODO: |
|
# avoid benchmark resource path currently |
|
#export BENCHMARK_RESOURCES_PATH="$(pwd)/resources" |
|
export BENCHMARK_GIT_PATH="$(pwd)/resources/git" |
|
# avoid linux temporarily; the linux blame benchmarks are simply |
|
# too slow to use |
|
# export BENCHMARK_LINUX_PATH="$(pwd)/resources/linux" |
|
|
|
if [[ "$(uname -s)" == MINGW* ]]; then |
|
GIT2_CLI="$(cygpath -w $(pwd))\\build\\RelWithDebInfo\\git2" |
|
else |
|
GIT2_CLI="$(pwd)/build/git2" |
|
fi |
|
|
|
if [ "${{ github.event.inputs.suite }}" != "" ]; then |
|
SUITE_FLAG="--suite ${{ github.event.inputs.suite }}" |
|
fi |
|
|
|
if [ "${{ github.event.inputs.debug }}" = "true" ]; then |
|
DEBUG_FLAG="--debug" |
|
fi |
|
|
|
mkdir benchmark && cd benchmark |
|
../source/benchmarks/cli/benchmark.sh \ |
|
${SUITE_FLAG} ${DEBUG_FLAG} \ |
|
--admin \ |
|
--baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \ |
|
--json benchmarks.json --flamegraph --zip benchmarks.zip |
|
shell: bash |
|
- name: Upload results |
|
uses: actions/upload-artifact@v4 |
|
with: |
|
name: benchmark-${{ matrix.platform.id }} |
|
path: benchmark |
|
if: always() |
|
|
|
# Publish the results |
|
publish: |
|
name: Publish results |
|
needs: [ build ] |
|
if: always() && github.repository == 'libgit2/libgit2' |
|
runs-on: ubuntu-latest |
|
steps: |
|
- name: Check out benchmark repository |
|
uses: actions/checkout@v4 |
|
with: |
|
repository: libgit2/benchmarks |
|
path: site |
|
fetch-depth: 0 |
|
ssh-key: ${{ secrets.BENCHMARKS_PUBLISH_KEY }} |
|
- name: Download test results |
|
uses: actions/download-artifact@v4 |
|
- name: Generate API |
|
run: | |
|
# Move today's benchmark run into the right place |
|
for platform in linux macos windows; do |
|
TIMESTAMP=$(jq .time.start < "benchmark-${platform}/benchmarks.json") |
|
TIMESTAMP_LEN=$(echo -n ${TIMESTAMP} | wc -c | xargs) |
|
DENOMINATOR=1 |
|
if [ "${TIMESTAMP_LEN}" = "19" ]; then |
|
DENOMINATOR="1000000000" |
|
elif [ "${TIMESTAMP_LEN}" = "13" ]; then |
|
DENOMINATOR="1000" |
|
else |
|
echo "unknown timestamp" |
|
exit 1 |
|
fi |
|
|
|
if [[ "$(uname -s)" == "Darwin" ]]; then |
|
DATE=$(date -R -r $(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d") |
|
else |
|
DATE=$(date -d @$(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d") |
|
fi |
|
|
|
# move the complete results in |
|
mkdir -p "site/public/api/runs/${DATE}" |
|
cp "benchmark-${platform}/benchmarks.json" "site/public/api/runs/${DATE}/${platform}.json" |
|
|
|
# unzip the individual results |
|
PLATFORM_TEMP=$(mktemp -d) |
|
unzip "benchmark-${platform}/benchmarks.zip" -d "${PLATFORM_TEMP}" |
|
|
|
mkdir -p "site/public/api/runs/${DATE}/${platform}" |
|
find "${PLATFORM_TEMP}" -name \*\.svg -exec cp {} "site/public/api/runs/${DATE}/${platform}" \; |
|
done |
|
|
|
(cd site && node scripts/aggregate.js) |
|
shell: bash |
|
|
|
# in debug mode, don't deploy the site; only create a zip file and |
|
# upload it for debugging |
|
- name: Upload site |
|
uses: actions/upload-artifact@v4 |
|
with: |
|
name: site |
|
path: site |
|
if: github.event_name == 'workflow_dispatch' |
|
- name: Publish API |
|
run: | |
|
git config user.name 'Benchmark Site Generation' && |
|
git config user.email 'libgit2@users.noreply.github.com' && |
|
git add . && |
|
git commit --allow-empty -m"benchmark update ${DATE}" && |
|
git push origin main |
|
shell: bash |
|
working-directory: site |
|
if: github.event_name == 'schedule' || github.event.inputs.deploy == 'true' |