From 009e6adb85ecf7ce019e05d8d30817f93dd036d8 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 26 Jun 2025 14:18:53 -0400 Subject: [PATCH 01/17] Allow for bisecting by pystats --- bench_runner/scripts/bisect.py | 116 ++++++++++++++++++++----- bench_runner/scripts/run_benchmarks.py | 11 +-- 2 files changed, 98 insertions(+), 29 deletions(-) diff --git a/bench_runner/scripts/bisect.py b/bench_runner/scripts/bisect.py index 45220dbe..5cd4e9bd 100644 --- a/bench_runner/scripts/bisect.py +++ b/bench_runner/scripts/bisect.py @@ -1,5 +1,6 @@ import argparse import contextlib +import json from pathlib import Path import subprocess import sys @@ -12,18 +13,27 @@ from bench_runner import flags as mflags from bench_runner import git -from bench_runner import result +from bench_runner import result as mresult from bench_runner.scripts import run_benchmarks from bench_runner.scripts import workflow from bench_runner.util import PathLike, format_seconds +def format_result(result: float | int, pystats: str | None = None) -> str: + if pystats is None: + return format_seconds(result) + else: + return f"{result:,}" + + def _get_result_commandline( benchmark: str, good_val: float, bad_val: float, pgo: bool, flags: str, + pystats: str | None, + invert: bool, repo: PathLike, ) -> list[str]: repo = Path(repo) @@ -37,16 +47,18 @@ def _get_result_commandline( str(bad_val), str(pgo), str(flags), + str(pystats), + str(invert), str(repo.absolute()), ] -def parse_result(benchmark_json: PathLike) -> float: +def parse_timing_result(benchmark_json: PathLike) -> float: # The name of the benchmark in the JSON file may be different from the one # used to select the benchmark. Therefore, just take the mean of all the # benchmarks in the JSON file. - result.clear_contents_cache() - r = result.Result.from_arbitrary_filename(benchmark_json) + mresult.clear_contents_cache() + r = mresult.Result.from_arbitrary_filename(benchmark_json) timing_data = r.get_timing_data() return float(np.mean([x.mean() for x in timing_data.values()])) @@ -55,10 +67,13 @@ def get_result( benchmark: str, pgo: bool = False, flags: str = "", + pystats: str | None = None, cpython: PathLike = Path("cpython"), reconfigure: bool = False, ) -> float: cpython = Path(cpython) + python = cpython / "python" + parsed_flags = mflags.parse_flags(flags) if pgo or reconfigure: # Jumping around through commits with PGO can leave stale PGO data @@ -66,11 +81,20 @@ def get_result( # first time in case the *last* bisect run used pgo. subprocess.run(["make", "clean"], cwd=cpython) - workflow.compile_unix(cpython, mflags.parse_flags(flags), pgo, False, reconfigure) - run_benchmarks.run_benchmarks(cpython / "python", benchmark) - timing = parse_result(run_benchmarks.BENCHMARK_JSON) + workflow.compile_unix(cpython, parsed_flags, pgo, pystats is not None, reconfigure) - return timing + if pystats is None: + run_benchmarks.run_benchmarks(python, benchmark) + return parse_timing_result(run_benchmarks.BENCHMARK_JSON) + else: + run_benchmarks.collect_pystats(python, benchmark) + summarize_stats_path = cpython / "Tools" / "scripts" / "summarize_stats.py" + subprocess.check_output( + [str(python), str(summarize_stats_path), "--json-output", "pystats.json"] + ) + with open("pystats.json", "r") as f: + contents = json.load(f) + return int(contents[pystats]) def get_log_file() -> Path: @@ -103,11 +127,14 @@ def _main( bad: str, pgo: bool = False, flags: str = "", + pystats: str | None = None, + invert: bool = False, repo: PathLike = Path("."), cpython: PathLike = Path("cpython"), ): repo = Path(repo).absolute() cpython = Path(cpython).absolute() + im = -1 if invert else 1 delete_log() @@ -117,17 +144,21 @@ def _main( ) git.checkout(cpython, good) - good_timing = get_result(benchmark, pgo, flags, cpython=cpython, reconfigure=True) - log(f"KNOWN GOOD ({good[:7]}): {format_seconds(good_timing)}") + good_result = get_result( + benchmark, pgo, flags, pystats, cpython=cpython, reconfigure=True + ) + log(f"KNOWN GOOD ({good[:7]}): {format_result(good_result, pystats)}") git.checkout(cpython, bad) - bad_timing = get_result(benchmark, pgo, flags, cpython=cpython) - log(f"KNOWN BAD ({bad[:7]}): {format_seconds(bad_timing)}") + bad_result = get_result(benchmark, pgo, flags, pystats, cpython=cpython) + log(f"KNOWN BAD ({bad[:7]}): {format_result(bad_result, pystats)}") - if good_timing >= bad_timing: + if im * good_result >= im * bad_result: show_log() raise ValueError( - f"Good timing ({good_timing}) must be less than bad timing ({bad_timing})." + f"Good result ({format_result(good_result, pystats)}) " + f"must be {'more' if invert else 'less'} than " + f"bad result ({format_result(bad_result, pystats)})." ) try: @@ -138,7 +169,14 @@ def _main( subprocess.run( ["git", "bisect", "run"] + _get_result_commandline( - benchmark, good_timing, bad_timing, pgo, flags, repo + benchmark, + good_result, + bad_result, + pgo, + flags, + pystats, + invert, + repo, ) ) finally: @@ -182,10 +220,28 @@ def main(): type=str, default="", ) + parser.add_argument( + "--pystats", + type=str, + help="Bisect using pystats. Should be the key in the pystats.json file.", + ) + parser.add_argument( + "--invert", + action="store_true", + help="Invert the values, i.e. expect the bad value to be lower than the good value.", + ) args = parser.parse_args() - _main(args.benchmark, args.good, args.bad, args.pgo, args.flags) + _main( + args.benchmark, + args.good, + args.bad, + args.pgo, + args.flags, + args.pystats, + args.invert, + ) if __name__ == "__main__": @@ -197,9 +253,17 @@ def main(): parser.add_argument("bad_val", type=float) parser.add_argument("pgo", type=str) parser.add_argument("flags", type=str) + parser.add_argument("pystats", type=str) + parser.add_argument("invert", type=str, choices=["True", "False"]) parser.add_argument("repo", type=str) args = parser.parse_args() + if args.pystats == "None": + args.pystats = None + + invert = args.invert == "True" + im = -1 if invert else 1 + mid_point = (args.good_val + args.bad_val) / 2.0 repo = Path(args.repo) @@ -207,8 +271,12 @@ def main(): try: with contextlib.chdir(repo): - timing = get_result( - args.benchmark, args.pgo == "True", args.flags, cpython=cpython + result = get_result( + args.benchmark, + args.pgo == "True", + args.flags, + args.pystats, + cpython=cpython, ) except Exception as e: # If there was any exception, display that exception and traceback and @@ -218,20 +286,20 @@ def main(): # The confidence is 0.0 at the mid-point, 1.0 at the good and bad values, # and > 1.0 outside of that. - confidence = abs((timing - mid_point) / ((args.bad_val - args.good_val) / 2.0)) + confidence = abs((result - mid_point) / ((args.bad_val - args.good_val) / 2.0)) with contextlib.chdir(repo): - if timing > mid_point: + if im * result > im * mid_point: log( f"BISECT BAD ({git.get_git_hash(cpython)[:7]}): " - f"{format_seconds(timing)} (confidence {confidence:.02f})" + f"{format_result(result, args.pystats)} (confidence {confidence:.02f})" ) - print(f"BAD: {timing} vs. ({args.good_val}, {args.bad_val})") + print(f"BAD: {result} vs. ({args.good_val}, {args.bad_val})") sys.exit(1) else: log( f"BISECT GOOD ({git.get_git_hash(cpython)[:7]}): " - f"{format_seconds(timing)} (confidence {confidence:.02f})" + f"{format_result(result, args.pystats)} (confidence {confidence:.02f})" ) - print(f"GOOD: {timing} vs. ({args.good_val}, {args.bad_val})") + print(f"GOOD: {result} vs. ({args.good_val}, {args.bad_val})") sys.exit(0) diff --git a/bench_runner/scripts/run_benchmarks.py b/bench_runner/scripts/run_benchmarks.py index 293e50cc..36ab6025 100644 --- a/bench_runner/scripts/run_benchmarks.py +++ b/bench_runner/scripts/run_benchmarks.py @@ -131,9 +131,9 @@ def run_benchmarks( def collect_pystats( python: PathLike, benchmarks: str, - fork: str, - ref: str, - individual: bool, + fork: str | None = None, + ref: str | None = None, + individual: bool = True, flags: Iterable[str] | None = None, ) -> None: pystats_dir = Path("/tmp/py_stats") @@ -167,7 +167,7 @@ def collect_pystats( except NoBenchmarkError: pass else: - if individual: + if individual and fork is not None and ref is not None: run_summarize_stats(python, fork, ref, benchmark, flags=flags) for filename in pystats_dir.iterdir(): @@ -181,7 +181,8 @@ def collect_pystats( else: benchmark_links = [] - run_summarize_stats(python, fork, ref, "all", benchmark_links, flags=flags) + if fork is not None and ref is not None: + run_summarize_stats(python, fork, ref, "all", benchmark_links, flags=flags) def get_perf_lines(files: Iterable[PathLike]) -> Iterable[str]: From 29596550b8f7d244f27d332fe5d36f1eed89ebc3 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 26 Jun 2025 14:23:28 -0400 Subject: [PATCH 02/17] Lint --- bench_runner/scripts/bisect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench_runner/scripts/bisect.py b/bench_runner/scripts/bisect.py index 5cd4e9bd..d0dca21a 100644 --- a/bench_runner/scripts/bisect.py +++ b/bench_runner/scripts/bisect.py @@ -228,7 +228,8 @@ def main(): parser.add_argument( "--invert", action="store_true", - help="Invert the values, i.e. expect the bad value to be lower than the good value.", + help="Invert the values, i.e. expect the bad value to be lower than " + "the good value.", ) args = parser.parse_args() From d281743d8e11317d4d6b02978e62f7f4930dc13f Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 10 Dec 2025 11:24:51 -0800 Subject: [PATCH 03/17] Update to pyperformance 1.13 --- bench_runner/benchmark_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench_runner/benchmark_definitions.py b/bench_runner/benchmark_definitions.py index 577be504..6ab39760 100644 --- a/bench_runner/benchmark_definitions.py +++ b/bench_runner/benchmark_definitions.py @@ -18,7 +18,7 @@ class BenchmarkRepo: BENCHMARK_REPOS = [ BenchmarkRepo( - "29cd683acc3eddfd878aa87ef52c15aa4a53641e", + "cdbf33bca8216d4636322a7637917a03b0fba945", "https://github.com/python/pyperformance.git", "pyperformance", ), From 28d41ad8619d027b47de8692a1223529e5a5727e Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 11 Feb 2026 08:41:41 -0800 Subject: [PATCH 04/17] Update to 1.14 --- bench_runner/benchmark_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench_runner/benchmark_definitions.py b/bench_runner/benchmark_definitions.py index 6ab39760..78a342e4 100644 --- a/bench_runner/benchmark_definitions.py +++ b/bench_runner/benchmark_definitions.py @@ -18,7 +18,7 @@ class BenchmarkRepo: BENCHMARK_REPOS = [ BenchmarkRepo( - "cdbf33bca8216d4636322a7637917a03b0fba945", + "2ab96f98f98274d07ebfdfd66e4f021723179706", "https://github.com/python/pyperformance.git", "pyperformance", ), From c551dac4eb46f9e5420f37c759d7f60a04f5c1e8 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Fri, 20 Feb 2026 21:16:32 +0100 Subject: [PATCH 05/17] "--pgo" and "-c Release" are mutually exclusive --- bench_runner/scripts/workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py index a84355bd..a6312bb7 100644 --- a/bench_runner/scripts/workflow.py +++ b/bench_runner/scripts/workflow.py @@ -185,9 +185,10 @@ def compile_windows( args = ["--%"] # This is the PowerShell "stop parsing" flag if force_32bit: args.extend(["-p", "win32"]) - args.extend(["-c", "Release"]) if pgo: args.append("--pgo") + else: + args.extend(["-c", "Release"]) if "JIT" in flags: args.append("--experimental-jit") if "PYTHON_UOPS" in flags: From 364144d8c93098e62a629bf4ef738941f2dfd872 Mon Sep 17 00:00:00 2001 From: Diego Russo Date: Wed, 22 Apr 2026 10:15:06 +0100 Subject: [PATCH 06/17] Fix CPU_AFFINITY flag --- bench_runner/scripts/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py index a6312bb7..92728c04 100644 --- a/bench_runner/scripts/workflow.py +++ b/bench_runner/scripts/workflow.py @@ -226,7 +226,7 @@ def tune_system(venv: PathLike, perf: bool) -> None: args = ["system", perf and "reset" or "tune"] if cpu_affinity := os.environ.get("CPU_AFFINITY"): - args.append(f'--affinity="{cpu_affinity}"') + args.append(f"--affinity={cpu_affinity}") run_in_venv(venv, "pyperf", args, sudo=True) From 2de11067df67afe65618a7add728da13fc0a04f8 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 8 May 2026 11:21:24 -0700 Subject: [PATCH 07/17] Remove extra files --- .github/workflows/ci.yml | 16 ++++++------- .github/workflows/pypi.yml | 14 +++++------ bench_runner/templates/_benchmark.src.yml | 24 +++++++++---------- bench_runner/templates/_find_failures.src.yml | 6 ++--- bench_runner/templates/_generate.src.yml | 8 +++---- bench_runner/templates/_notify.src.yml | 6 ++--- bench_runner/templates/_publish.src.yml | 2 +- bench_runner/templates/_pystats.src.yml | 6 ++--- bench_runner/templates/_weekly.src.yml | 2 +- bench_runner/templates/benchmark.src.yml | 6 ++--- bench_runner/templates/ci.src.yml | 4 ++-- 11 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4f93614..db65e57a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" cache: pip @@ -27,8 +27,8 @@ jobs: matrix: python_version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python_version }} cache: pip @@ -42,8 +42,8 @@ jobs: long_test_linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.13 cache: pip @@ -57,8 +57,8 @@ jobs: long_test_windows: runs-on: windows-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.13 cache: pip diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index be28340a..df42e84a 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -13,11 +13,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" - name: Install pypa/build @@ -27,7 +27,7 @@ jobs: run: | python3 -m build - name: Store the distribution packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: python-package-distributions path: dist/ @@ -46,12 +46,12 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 with: name: python-package-distributions path: dist/ - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b #v1.14.0 github-release: name: >- @@ -67,12 +67,12 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 with: name: python-package-distributions path: dist/ - name: Sign the dists with Sigstore - uses: sigstore/gh-action-sigstore-python@v3.0.0 + uses: sigstore/gh-action-sigstore-python@04cffa1d795717b140764e8b640de88853c92acc # v3.3.0 with: inputs: >- ./dist/*.tar.gz diff --git a/bench_runner/templates/_benchmark.src.yml b/bench_runner/templates/_benchmark.src.yml index 6f7cc1a6..f97dee05 100644 --- a/bench_runner/templates/_benchmark.src.yml +++ b/bench_runner/templates/_benchmark.src.yml @@ -67,7 +67,7 @@ jobs: run: | git config --global core.symlinks true - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: git gc run: | git gc @@ -83,11 +83,11 @@ jobs: # just pulled in in that case. git pull -s recursive -X ours --autostash --rebase - name: Add data to repo - uses: EndBug/add-and-commit@v9 + uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 with: add: results - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: benchmark path: | @@ -100,14 +100,14 @@ jobs: steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: git gc run: | git gc - - uses: fregante/setup-git-user@v2 + - uses: fregante/setup-git-user@024bc0b8e177d7e77203b48dab6fb45666854b35 # v2.0.2 - name: Setup system Python if: ${{ runner.arch == 'X64' }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" - name: Building Python and running pyperformance @@ -124,12 +124,12 @@ jobs: git pull -s recursive -X ours --autostash --rebase - name: Adding data to repo if: ${{ !inputs.perf }} - uses: EndBug/add-and-commit@v9 + uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 with: add: results - name: Upload benchmark artifacts if: ${{ !inputs.perf }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: benchmark path: | @@ -137,7 +137,7 @@ jobs: overwrite: true - name: Upload perf artifacts if: ${{ inputs.perf }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: perf path: | @@ -148,7 +148,7 @@ jobs: steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: git gc run: | git gc @@ -164,11 +164,11 @@ jobs: # just pulled in in that case. git pull -s recursive -X ours --autostash --rebase - name: Add data to repo - uses: EndBug/add-and-commit@v9 + uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 with: add: results - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: benchmark path: | diff --git a/bench_runner/templates/_find_failures.src.yml b/bench_runner/templates/_find_failures.src.yml index 616679ac..b2346454 100644 --- a/bench_runner/templates/_find_failures.src.yml +++ b/bench_runner/templates/_find_failures.src.yml @@ -11,9 +11,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup system Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" cache: pip @@ -24,7 +24,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} - name: Add to repo - uses: EndBug/add-and-commit@v9 + uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 with: add: "['failures.md']" message: Benchmarking failures \ No newline at end of file diff --git a/bench_runner/templates/_generate.src.yml b/bench_runner/templates/_generate.src.yml index ad8e13df..2f4e4c8c 100644 --- a/bench_runner/templates/_generate.src.yml +++ b/bench_runner/templates/_generate.src.yml @@ -27,17 +27,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: main - name: Checkout CPython - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false repository: python/cpython path: cpython - name: Setup system Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" cache: pip @@ -46,7 +46,7 @@ jobs: - name: Regenerate derived data run: python -m bench_runner generate_results ${{ inputs.force == true && '--force' || '' }} - name: Add to repo - uses: EndBug/add-and-commit@v9 + uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 if: ${{ !inputs.dry_run }} with: add: "['results', 'README.md', 'RESULTS.md', 'longitudinal.svg', 'longitudinal.json', 'configs.svg', 'configs.json', 'memory_long.svg', 'memory_long.json', 'memory_configs.svg', 'memory_configs.json']" diff --git a/bench_runner/templates/_notify.src.yml b/bench_runner/templates/_notify.src.yml index fe53a558..7aa68421 100644 --- a/bench_runner/templates/_notify.src.yml +++ b/bench_runner/templates/_notify.src.yml @@ -19,13 +19,13 @@ jobs: notify: runs-on: ubuntu-latest steps: - - uses: technote-space/workflow-conclusion-action@v3 + - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 #v3.0.3 - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Setup system Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" cache: pip diff --git a/bench_runner/templates/_publish.src.yml b/bench_runner/templates/_publish.src.yml index 21b41478..52064d7d 100644 --- a/bench_runner/templates/_publish.src.yml +++ b/bench_runner/templates/_publish.src.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: main # This has to be enough depth to handle since the last time we published. diff --git a/bench_runner/templates/_pystats.src.yml b/bench_runner/templates/_pystats.src.yml index 8ef1b143..8a91278d 100644 --- a/bench_runner/templates/_pystats.src.yml +++ b/bench_runner/templates/_pystats.src.yml @@ -42,12 +42,12 @@ jobs: runs-on: [self-hosted, linux, cloud] steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: git gc run: | git gc - name: Setup system Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" - name: Build CPython and run pyperformance benchmarks @@ -61,6 +61,6 @@ jobs: # just pulled in in that case. git pull -s recursive -X ours --autostash --rebase - name: Add data to repo - uses: EndBug/add-and-commit@v9 + uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 with: add: results diff --git a/bench_runner/templates/_weekly.src.yml b/bench_runner/templates/_weekly.src.yml index 4ed39e62..9f53a42d 100644 --- a/bench_runner/templates/_weekly.src.yml +++ b/bench_runner/templates/_weekly.src.yml @@ -16,7 +16,7 @@ jobs: commit: ${{ steps.head.outputs.commit }} steps: - name: Checkout CPython - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false repository: python/cpython diff --git a/bench_runner/templates/benchmark.src.yml b/bench_runner/templates/benchmark.src.yml index 67ddc038..9ee61f9d 100644 --- a/bench_runner/templates/benchmark.src.yml +++ b/bench_runner/templates/benchmark.src.yml @@ -46,16 +46,16 @@ jobs: need_to_run: ${{ steps.base.outputs.need_to_run }} steps: - name: Checkout benchmarking - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Setup system Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" cache: pip - name: Checkout CPython - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false repository: ${{ inputs.fork }}/cpython diff --git a/bench_runner/templates/ci.src.yml b/bench_runner/templates/ci.src.yml index 3944945f..9531f946 100644 --- a/bench_runner/templates/ci.src.yml +++ b/bench_runner/templates/ci.src.yml @@ -11,8 +11,8 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" cache: pip From 668fe2af5fc8ebf6825cd079aac192a94a0650f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 16:27:05 +0000 Subject: [PATCH 08/17] Bump pyright from 1.1.402 to 1.1.409 Bumps [pyright](https://github.com/RobertCraigie/pyright-python) from 1.1.402 to 1.1.409. - [Release notes](https://github.com/RobertCraigie/pyright-python/releases) - [Commits](https://github.com/RobertCraigie/pyright-python/compare/v1.1.402...v1.1.409) --- updated-dependencies: - dependency-name: pyright dependency-version: 1.1.407 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 90b92bf6..3372b18b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ test = [ "black==25.1.0", "filelock==3.18.0", "flake8==7.3.0", - "pyright==1.1.402", + "pyright==1.1.409", "pytest==8.4.1", "pytest-cov==6.2.1", "pytest-xdist==3.7.0", From 1fddec4ce99ff3a5b3a20fb094cf8ff73944edf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 16:57:43 +0000 Subject: [PATCH 09/17] Bump filelock from 3.18.0 to 3.29.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.18.0 to 3.29.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.18.0...3.29.0) --- updated-dependencies: - dependency-name: filelock dependency-version: 3.20.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3372b18b..89f1d950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dynamic = ["version"] [project.optional-dependencies] test = [ "black==25.1.0", - "filelock==3.18.0", + "filelock==3.29.0", "flake8==7.3.0", "pyright==1.1.409", "pytest==8.4.1", From 9a0c39e4a4c29509c1b365db7db5db7407b77c29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 16:58:09 +0000 Subject: [PATCH 10/17] Bump pytest-xdist from 3.7.0 to 3.8.0 Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.7.0 to 3.8.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.7.0...v3.8.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-version: 3.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3372b18b..b981075f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ test = [ "pyright==1.1.409", "pytest==8.4.1", "pytest-cov==6.2.1", - "pytest-xdist==3.7.0", + "pytest-xdist==3.8.0", ] [tool.setuptools_scm] From f87655488524aeb5fffc9ce96048843b0588d942 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 19:59:16 +0000 Subject: [PATCH 11/17] Bump pyperf from 2.9.0 to 2.10.0 Bumps [pyperf](https://github.com/psf/pyperf) from 2.9.0 to 2.10.0. - [Release notes](https://github.com/psf/pyperf/releases) - [Changelog](https://github.com/psf/pyperf/blob/main/doc/changelog.rst) - [Commits](https://github.com/psf/pyperf/compare/2.9.0...2.10.0) --- updated-dependencies: - dependency-name: pyperf dependency-version: 2.10.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8f62945..f293dd85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ ] dependencies = [ "matplotlib==3.10.3", - "pyperf==2.9.0", + "pyperf==2.10.0", "rich==14.0.0", "rich-argparse==1.7.1", "ruamel.yaml==0.18.14", From a0df0385192afc522dad19911737795bedf247a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 19:59:20 +0000 Subject: [PATCH 12/17] Bump ruamel-yaml from 0.18.14 to 0.19.1 Bumps ruamel-yaml from 0.18.14 to 0.19.1. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-version: 0.19.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8f62945..7dfebc0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "pyperf==2.9.0", "rich==14.0.0", "rich-argparse==1.7.1", - "ruamel.yaml==0.18.14", + "ruamel.yaml==0.19.1", "scour==0.38.2", "wheel", ] From f9a978e76d1eae3df62bd4e06b42ee4660c71a8f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:58:21 +0000 Subject: [PATCH 13/17] Bump matplotlib from 3.10.3 to 3.11.2 Bumps [matplotlib](https://github.com/matplotlib/matplotlib) from 3.10.3 to 3.11.2. - [Release notes](https://github.com/matplotlib/matplotlib/releases) - [Commits](https://github.com/matplotlib/matplotlib/compare/v3.10.3...v3.11.2) --- updated-dependencies: - dependency-name: matplotlib dependency-version: 3.10.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f293dd85..67ae1700 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ "Programming Language :: Python :: 3", ] dependencies = [ - "matplotlib==3.10.3", + "matplotlib==3.11.2", "pyperf==2.10.0", "rich==14.0.0", "rich-argparse==1.7.1", From c64ec6575ed9f468901b55104e02c0c92e9918ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:59:58 +0000 Subject: [PATCH 14/17] Bump rich from 14.0.0 to 15.0.0 Bumps [rich](https://github.com/Textualize/rich) from 14.0.0 to 15.0.0. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/main/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v14.0.0...v15.0.0) --- updated-dependencies: - dependency-name: rich dependency-version: 14.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f293dd85..08bbf2b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ dependencies = [ "matplotlib==3.10.3", "pyperf==2.10.0", - "rich==14.0.0", + "rich==15.0.0", "rich-argparse==1.7.1", "ruamel.yaml==0.18.14", "scour==0.38.2", From e372ef5dc9a4245d7e96d01c47b6e3982a8c7765 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 6 Nov 2026 10:16:20 -0800 Subject: [PATCH 15/17] Update repository URLs after move to Python organization --- bench_runner/scripts/run_benchmarks.py | 2 +- tests/test_run_benchmarks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench_runner/scripts/run_benchmarks.py b/bench_runner/scripts/run_benchmarks.py index 36ab6025..b005c3b3 100644 --- a/bench_runner/scripts/run_benchmarks.py +++ b/bench_runner/scripts/run_benchmarks.py @@ -32,7 +32,7 @@ BENCHMARK_JSON = REPO_ROOT / "benchmark.json" PROFILING_RESULTS = REPO_ROOT / "profiling" / "results" GITHUB_URL = "https://github.com/" + os.environ.get( - "GITHUB_REPOSITORY", "faster-cpython/bench_runner" + "GITHUB_REPOSITORY", "python/bench_runner" ) # Environment variables that control the execution of CPython ENV_VARS = ["PYTHON_JIT", "PYPERF_PERF_RECORD_EXTRA_OPTS"] diff --git a/tests/test_run_benchmarks.py b/tests/test_run_benchmarks.py index 74e7c1e2..2ff273d4 100644 --- a/tests/test_run_benchmarks.py +++ b/tests/test_run_benchmarks.py @@ -60,7 +60,7 @@ def test_update_metadata(benchmarks_checkout, monkeypatch): assert metadata["benchmark_hash"] == "215d35" assert ( metadata["github_action_url"] - == "https://github.com/faster-cpython/bench_runner/actions/runs/12345" + == "https://github.com/python/bench_runner/actions/runs/12345" ) @@ -119,7 +119,7 @@ def test_run_benchmarks(benchmarks_checkout, monkeypatch): assert metadata["benchmark_hash"] == "215d35" assert ( metadata["github_action_url"] - == "https://github.com/faster-cpython/bench_runner/actions/runs/12345" + == "https://github.com/python/bench_runner/actions/runs/12345" ) assert len(benchmarks) == 3 From 7e12b8b80311fe6badebb381c32e66e9e1bfb7c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:42:34 +0000 Subject: [PATCH 16/17] Bump pyright from 1.1.409 to 1.1.414 Bumps [pyright](https://github.com/RobertCraigie/pyright-python) from 1.1.409 to 1.1.414. - [Release notes](https://github.com/RobertCraigie/pyright-python/releases) - [Commits](https://github.com/RobertCraigie/pyright-python/compare/v1.1.409...v1.1.414) --- updated-dependencies: - dependency-name: pyright dependency-version: 1.1.414 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 67ae1700..1a90cee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ test = [ "black==25.1.0", "filelock==3.29.0", "flake8==7.3.0", - "pyright==1.1.409", + "pyright==1.1.414", "pytest==8.4.1", "pytest-cov==6.2.1", "pytest-xdist==3.8.0", From facd6ebe2f20792144ca9b6bc5ece544a489f2a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:13:47 +0000 Subject: [PATCH 17/17] Bump rich-argparse from 1.7.1 to 1.8.0 Bumps [rich-argparse](https://github.com/hamdanal/rich-argparse) from 1.7.1 to 1.8.0. - [Release notes](https://github.com/hamdanal/rich-argparse/releases) - [Changelog](https://github.com/hamdanal/rich-argparse/blob/main/CHANGELOG.md) - [Commits](https://github.com/hamdanal/rich-argparse/compare/v1.7.1...v1.8.0) --- updated-dependencies: - dependency-name: rich-argparse dependency-version: 1.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cd8dc15c..ec632344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "matplotlib==3.11.2", "pyperf==2.10.0", "rich==15.0.0", - "rich-argparse==1.7.1", + "rich-argparse==1.8.0", "ruamel.yaml==0.19.1", "scour==0.38.2", "wheel",