.github/workflows/libcxx-pr-benchmark.yml - llvm-project - Git at Google


            
            # This file defines a workflow that runs the libc++ benchmarks when a comment is added to the PR.

            
            #

            
            # The comment is of the form:

            
            #

            
            #   /libcxx-bot benchmark <path-to-benchmarks-to-run>

            
            #

            
            # That will cause the specified benchmarks to be run on the PR and on the pull-request target, and

            
            # their results to be compared.


            name: "[libc++] Benchmark PR"


            permissions:

            
              contents: read


            on:

            
              issue_comment:

            
                types:

            
                  - created

            
                  - edited


            jobs:

            
              permission-check:

            
                if: >-

            
                  github.event.issue.pull_request &&

            
                  contains(github.event.comment.body, '/libcxx-bot benchmark')

            
                runs-on: ubuntu-26.04

            
                environment:

            
                  name: main-branch-only

            
                  deployment: false

            
                steps:

            
                  - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

            
                    with:

            
                      persist-credentials: false

            
                      sparse-checkout: |

            
                        .github/workflows/


                  - name: Check Permissions

            
                    uses: ./.github/workflows/require-team-membership

            
                    with:

            
                      team-slug: llvm-committers

            
                      LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }}

            
                      LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }}


              extract-info:

            
                runs-on: ubuntu-26.04

            
                needs:

            
                  - permission-check

            
                permissions:

            
                  pull-requests: write

            
                steps:

            
                  - name: Checkout the LNT configurations

            
                    uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

            
                    with:

            
                      persist-credentials: false

            
                      # Disabling cone mode allows checking out exactly the single file we need.

            
                      sparse-checkout: libcxx/utils/ci/lnt/machines.json

            
                      sparse-checkout-cone-mode: false


                  - name: Extract information from the PR

            
                    id: vars

            
                    uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

            
                    with:

            
                      script: |

            
                        const pr = await github.rest.pulls.get({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          pull_number: context.issue.number,

            
                        });

            
                        core.setOutput('pr_base', pr.data.base.sha);

            
                        core.setOutput('pr_head', pr.data.head.sha);

            
                        const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/);

            
                        core.setOutput('benchmarks', match ? match[1] : '');


                        // Benchmark on the same configurations that we track performance on.

            
                        const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));

            
                        core.setOutput('matrix', JSON.stringify(config.map(({coverage, ...machine}) => machine)));


                  - name: Update comment with link to the run

            
                    uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

            
                    with:

            
                      script: |

            
                        const { data: comment } = await github.rest.issues.getComment({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          comment_id: context.payload.comment.id,

            
                        });

            
                        const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

            
                        const note = `> _Running benchmarks in ${run_url}_`;

            
                        await github.rest.issues.updateComment({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          comment_id: context.payload.comment.id,

            
                          body: [comment.body, note].join('\n\n'),

            
                        });

            
                outputs:

            
                  pr_base: ${{ steps.vars.outputs.pr_base }}

            
                  pr_head: ${{ steps.vars.outputs.pr_head }}

            
                  benchmarks: ${{ steps.vars.outputs.benchmarks }}

            
                  matrix: ${{ steps.vars.outputs.matrix }}


              run-benchmarks:

            
                strategy:

            
                  matrix:

            
                    include: ${{ fromJSON(needs.extract-info.outputs.matrix) }}

            
                  fail-fast: false

            
                permissions:

            
                  pull-requests: write

            
                runs-on: ${{ matrix.runner }}

            
                needs:

            
                  - extract-info

            
                env:

            
                  BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}

            
                  COMPILER: ${{ matrix.cxx }}

            
                  LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}

            
                  LNT_MACHINE: ${{ matrix.lnt-machine }}

            
                  PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}

            
                  PR_BASE: ${{ needs.extract-info.outputs.pr_base }}

            
                  TOOLING: ${{ github.workspace }}/tooling

            
                steps:

            
                  - name: Checkout the PR

            
                    uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

            
                    with:

            
                      persist-credentials: false

            
                      ref: ${{ needs.extract-info.outputs.pr_head }}

            
                      fetch-depth: 0

            
                      fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main


                  # This workflow is always taken from the default branch. Check out the tooling from the corresponding

            
                  # commit so that we're always running tooling that's in sync with the workflow's definition. Only the

            
                  # code and the test suite (including its testing configuration) are taken from the PR head.

            
                  - name: Checkout the tooling from ${{ github.sha }}

            
                    uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

            
                    with:

            
                      persist-credentials: false

            
                      ref: ${{ github.sha }}

            
                      path: tooling

            
                      sparse-checkout: |

            
                        .github/workflows/libcxx

            
                        libcxx/utils


                  - name: Install Python

            
                    if: runner.os == 'Linux' # installed via Homebrew on macOS

            
                    uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0

            
                    with:

            
                      python-version: '3.14'


                  - name: Select Xcode

            
                    if: runner.os == 'macOS'

            
                    run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV


                  - name: Install dependencies via Homebrew

            
                    if: runner.os == 'macOS'

            
                    run: |

            
                      brew update

            
                      brew install ninja cmake python@3.14

            
                      echo "$(brew --prefix python@3.14)/bin" >> "$GITHUB_PATH"


                  - name: Diagnose tools in use

            
                    run: |

            
                      cmake --version

            
                      ninja --version

            
                      "${COMPILER}" --version

            
                      python3 --version


                  - name: Setup virtual environment

            
                    run: |

            
                      python3 -m venv .venv

            
                      source .venv/bin/activate

            
                      pip install -r "${TOOLING}/libcxx/utils/requirements.txt"


                  - name: Determine the baseline commit

            
                    id: baseline

            
                    run: echo "commit=$(git merge-base "${PR_BASE}" "${PR_HEAD}")" >> "${GITHUB_OUTPUT}"


                  - name: Build the baseline

            
                    uses: ./tooling/.github/workflows/libcxx/build-at-commit

            
                    with:

            
                      git-repo: ${{ github.workspace }}

            
                      commit: ${{ steps.baseline.outputs.commit }}

            
                      install-dir: ${{ github.workspace }}/install/baseline

            
                      compiler: ${{ matrix.cxx }}

            
                      cmake-cache: ${{ matrix.cmake-cache }}


                  - name: Build the candidate

            
                    uses: ./tooling/.github/workflows/libcxx/build-at-commit

            
                    with:

            
                      git-repo: ${{ github.workspace }}

            
                      commit: ${{ needs.extract-info.outputs.pr_head }}

            
                      install-dir: ${{ github.workspace }}/install/candidate

            
                      compiler: ${{ matrix.cxx }}

            
                      cmake-cache: ${{ matrix.cmake-cache }}


                  - name: Run baseline and candidate interleaved

            
                    run: |

            
                      source .venv/bin/activate


                      read -ra configured_params <<< "${LIT_PARAMS}"

            
                      lit_params=()

            
                      for param in "${configured_params[@]}"; do

            
                        lit_params+=(--param "${param}")

            
                      done


                      # Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of

            
                      # environmental noise

            
                      for _ in $(seq 1 5); do

            
                        "${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"

            
                        "${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/baseline | tee -a baseline.lnt

            
                        "${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"

            
                        "${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/candidate | tee -a candidate.lnt

            
                      done


                  - name: Compare baseline and candidate runs

            
                    run: |

            
                      source .venv/bin/activate

            
                      "${TOOLING}/libcxx/utils/compare-benchmarks" baseline.lnt candidate.lnt | tee results.txt


                  - name: Update comment with results

            
                    uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

            
                    with:

            
                      script: |

            
                        const fs = require('fs');

            
                        const results = fs.readFileSync('results.txt', 'utf8');

            
                        const { data: comment } = await github.rest.issues.getComment({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          comment_id: context.payload.comment.id,

            
                        });

            
                        const details = [

            
                          '<details>',

            
                          '<summary>',

            
                          `Benchmark results for ${process.env.LNT_MACHINE}:`,

            
                          '</summary>',

            
                          '',

            
                          '```',

            
                          results,

            
                          '```',

            
                          '',

            
                          '</details>',

            
                        ].join('\n');

            
                        await github.rest.issues.updateComment({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          comment_id: context.payload.comment.id,

            
                          body: [comment.body, details].join('\n\n'),

            
                        });


                  - name: Report failure in the comment

            
                    if: failure()

            
                    uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

            
                    with:

            
                      script: |

            
                        const { data: comment } = await github.rest.issues.getComment({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          comment_id: context.payload.comment.id,

            
                        });

            
                        const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

            
                        const note = `> _:x: Benchmarks for ${process.env.LNT_MACHINE} failed. See ${run_url} for details._`;

            
                        await github.rest.issues.updateComment({

            
                          owner: context.repo.owner,

            
                          repo: context.repo.repo,

            
                          comment_id: context.payload.comment.id,

            
                          body: [comment.body, note].join('\n\n'),

            
                        });
	# This file defines a workflow that runs the libc++ benchmarks when a comment is added to the PR.
	#
	# The comment is of the form:
	#
	# /libcxx-bot benchmark <path-to-benchmarks-to-run>
	#
	# That will cause the specified benchmarks to be run on the PR and on the pull-request target, and
	# their results to be compared.

	name: "[libc++] Benchmark PR"

	permissions:
	contents: read

	on:
	issue_comment:
	types:
	- created
	- edited

	jobs:
	permission-check:
	if: >-
	github.event.issue.pull_request &&
	contains(github.event.comment.body, '/libcxx-bot benchmark')
	runs-on: ubuntu-26.04
	environment:
	name: main-branch-only
	deployment: false
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	sparse-checkout: \|
	.github/workflows/

	- name: Check Permissions
	uses: ./.github/workflows/require-team-membership
	with:
	team-slug: llvm-committers
	LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }}
	LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }}

	extract-info:
	runs-on: ubuntu-26.04
	needs:
	- permission-check
	permissions:
	pull-requests: write
	steps:
	- name: Checkout the LNT configurations
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	# Disabling cone mode allows checking out exactly the single file we need.
	sparse-checkout: libcxx/utils/ci/lnt/machines.json
	sparse-checkout-cone-mode: false

	- name: Extract information from the PR
	id: vars
	uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	with:
	script: \|
	const pr = await github.rest.pulls.get({
	owner: context.repo.owner,
	repo: context.repo.repo,
	pull_number: context.issue.number,
	});
	core.setOutput('pr_base', pr.data.base.sha);
	core.setOutput('pr_head', pr.data.head.sha);
	const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/);
	core.setOutput('benchmarks', match ? match[1] : '');

	// Benchmark on the same configurations that we track performance on.
	const config = JSON.parse(require('fs').readFileSync('libcxx/utils/ci/lnt/machines.json', 'utf8'));
	core.setOutput('matrix', JSON.stringify(config.map(({coverage, ...machine}) => machine)));

	- name: Update comment with link to the run
	uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	with:
	script: \|
	const { data: comment } = await github.rest.issues.getComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: context.payload.comment.id,
	});
	const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
	const note = `> _Running benchmarks in ${run_url}_`;
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: context.payload.comment.id,
	body: [comment.body, note].join('\n\n'),
	});
	outputs:
	pr_base: ${{ steps.vars.outputs.pr_base }}
	pr_head: ${{ steps.vars.outputs.pr_head }}
	benchmarks: ${{ steps.vars.outputs.benchmarks }}
	matrix: ${{ steps.vars.outputs.matrix }}

	run-benchmarks:
	strategy:
	matrix:
	include: ${{ fromJSON(needs.extract-info.outputs.matrix) }}
	fail-fast: false
	permissions:
	pull-requests: write
	runs-on: ${{ matrix.runner }}
	needs:
	- extract-info
	env:
	BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}
	COMPILER: ${{ matrix.cxx }}
	LIT_PARAMS: ${{ join(matrix.lit-params, ' ') }}
	LNT_MACHINE: ${{ matrix.lnt-machine }}
	PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
	PR_BASE: ${{ needs.extract-info.outputs.pr_base }}
	TOOLING: ${{ github.workspace }}/tooling
	steps:
	- name: Checkout the PR
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	ref: ${{ needs.extract-info.outputs.pr_head }}
	fetch-depth: 0
	fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main

	# This workflow is always taken from the default branch. Check out the tooling from the corresponding
	# commit so that we're always running tooling that's in sync with the workflow's definition. Only the
	# code and the test suite (including its testing configuration) are taken from the PR head.
	- name: Checkout the tooling from ${{ github.sha }}
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	ref: ${{ github.sha }}
	path: tooling
	sparse-checkout: \|
	.github/workflows/libcxx
	libcxx/utils

	- name: Install Python
	if: runner.os == 'Linux' # installed via Homebrew on macOS
	uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
	with:
	python-version: '3.14'

	- name: Select Xcode
	if: runner.os == 'macOS'
	run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV

	- name: Install dependencies via Homebrew
	if: runner.os == 'macOS'
	run: \|
	brew update
	brew install ninja cmake python@3.14
	echo "$(brew --prefix python@3.14)/bin" >> "$GITHUB_PATH"

	- name: Diagnose tools in use
	run: \|
	cmake --version
	ninja --version
	"${COMPILER}" --version
	python3 --version

	- name: Setup virtual environment
	run: \|
	python3 -m venv .venv
	source .venv/bin/activate
	pip install -r "${TOOLING}/libcxx/utils/requirements.txt"

	- name: Determine the baseline commit
	id: baseline
	run: echo "commit=$(git merge-base "${PR_BASE}" "${PR_HEAD}")" >> "${GITHUB_OUTPUT}"

	- name: Build the baseline
	uses: ./tooling/.github/workflows/libcxx/build-at-commit
	with:
	git-repo: ${{ github.workspace }}
	commit: ${{ steps.baseline.outputs.commit }}
	install-dir: ${{ github.workspace }}/install/baseline
	compiler: ${{ matrix.cxx }}
	cmake-cache: ${{ matrix.cmake-cache }}

	- name: Build the candidate
	uses: ./tooling/.github/workflows/libcxx/build-at-commit
	with:
	git-repo: ${{ github.workspace }}
	commit: ${{ needs.extract-info.outputs.pr_head }}
	install-dir: ${{ github.workspace }}/install/candidate
	compiler: ${{ matrix.cxx }}
	cmake-cache: ${{ matrix.cmake-cache }}

	- name: Run baseline and candidate interleaved
	run: \|
	source .venv/bin/activate

	read -ra configured_params <<< "${LIT_PARAMS}"
	lit_params=()
	for param in "${configured_params[@]}"; do
	lit_params+=(--param "${param}")
	done

	# Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of
	# environmental noise
	for _ in $(seq 1 5); do
	"${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS"
	"${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/baseline \| tee -a baseline.lnt
	"${TOOLING}/libcxx/utils/test-at-commit" --test-config installed-libc++.cfg.in -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 "${lit_params[@]}" --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS"
	"${TOOLING}/libcxx/utils/consolidate-benchmarks" benchmarks/candidate \| tee -a candidate.lnt
	done

	- name: Compare baseline and candidate runs
	run: \|
	source .venv/bin/activate
	"${TOOLING}/libcxx/utils/compare-benchmarks" baseline.lnt candidate.lnt \| tee results.txt

	- name: Update comment with results
	uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	with:
	script: \|
	const fs = require('fs');
	const results = fs.readFileSync('results.txt', 'utf8');
	const { data: comment } = await github.rest.issues.getComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: context.payload.comment.id,
	});
	const details = [
	'<details>',
	'<summary>',
	`Benchmark results for ${process.env.LNT_MACHINE}:`,
	'</summary>',
	'',
	'```',
	results,
	'```',
	'',
	'</details>',
	].join('\n');
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: context.payload.comment.id,
	body: [comment.body, details].join('\n\n'),
	});

	- name: Report failure in the comment
	if: failure()
	uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	with:
	script: \|
	const { data: comment } = await github.rest.issues.getComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: context.payload.comment.id,
	});
	const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
	const note = `> _:x: Benchmarks for ${process.env.LNT_MACHINE} failed. See ${run_url} for details._`;
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: context.payload.comment.id,
	body: [comment.body, note].join('\n\n'),
	});