Merge remote-tracking branch 'upstream/main' into config-handling

Author: mdboom

README.md

@@ -149,6 +149,7 @@ In `bench_runner.toml`, the `longitudinal_plot` table has a `subplots` key which
 - `base`: The base version to compare to. Should be a fully-specified version, e.g. "3.13.0".
 - `version`: The version series to use as a head. Should be a 2-part version, e.g. "3.14"
 - `flags`: (optional) A list of flags to match to for the head versions
+- `runners`: (optional) A list of nicknames of runners to plot. Defaults to all runners.
 
 For example:
 
@@ -157,7 +158,8 @@ For example:
 subplots = [
     { base = "3.10.4", version = "3.11" },
     { base = "3.12.0", version = "3.13" },
-    { base = "3.13.0", version = "3.14" },
+    { base = "3.13.0", version = "3.14", runners = ["linux1", "linux2"] },
+    { base = "3.13.0", version = "3.14", runners = ["windows1", "macos1"] },
     { base = "3.13.0", version = "3.14", flags = ["JIT"] }
 ]
 ```
@@ -172,6 +174,7 @@ In `bench_runner.toml`, the `flag_effect_plot` table has a `subplots` key which
 - `version`: The version series to compare. Should be a 2-part version, e.g. "3.14"
 - `head_flags`: A list of flags to use as the head.
 - `base_flags`: (optional) A list of flags to use as the base. By default, this is a default build, i.e. no flags.
+- `runners`: (optional) A list of nicknames of runners to plot. Defaults to all runners.
 - `runner_map`: (optional) If you need to map a runner to a base in a
   different runner, you can provide that mapping here. For example, with
   tail-calling, you may want to compare runners configured to use clang
@@ -192,6 +195,7 @@ head_flags = ["JIT"]
 name = "Tail calling interpreter"
 version = "3.14"
 head_flags = ["TAILCALL"]
+runners = ["linux_clang"]
 runner_map = { linux_clang = "linux" }
 ```
 

bench_runner/plot.py

@@ -222,6 +222,7 @@ class Subplot:
         base: str
         version: str
         flags: list[str] = dataclasses.field(default_factory=list)
+        runners: list[str] = dataclasses.field(default_factory=list)
 
         def __post_init__(self):
             if not util.valid_version(self.base):
@@ -291,23 +292,29 @@ def get_comparison_value(ref, r, base):
         axs = _axs
 
     results = [r for r in results if r.fork == "python"]
+    runners = cfg.runners.values()
 
     for subcfg, ax in zip(all_cfg, axs):
         version = [int(x) for x in subcfg.version.split(".")]
         ver_results = [
             r for r in results if list(r.parsed_version.release[0:2]) == version
         ]
+        if subcfg.runners:
+            cfg_runners = [r for r in runners if r.nickname in subcfg.runners]
+        else:
+            cfg_runners = runners
 
-        subtitle = f"Python {subcfg.version}.x vs. {subcfg.base}"
         if len(subcfg.flags):
-            subtitle += f" ({','.join(subcfg.flags)})"
+            titleflags = f" ({','.join(subcfg.flags)})"
+        else:
+            titleflags = ""
+        subtitle = f"Python {subcfg.version}.x{titleflags} vs. {subcfg.base}"
         ax.set_title(subtitle)
 
         first_runner = True
 
-        for runner in cfg.runners.values():
-            assert runner.plot is not None
-
+        for runner in cfg_runners:
+            assert runner.plot is not None  # typing
             runner_results = [
                 r
                 for r in ver_results
@@ -410,6 +417,7 @@ class Subplot:
         head_flags: list[str] = dataclasses.field(default_factory=list)
         base_flags: list[str] = dataclasses.field(default_factory=list)
         runner_map: dict[str, str] = dataclasses.field(default_factory=dict)
+        runners: list[str] = dataclasses.field(default_factory=list)
 
         def __post_init__(self):
             if (
@@ -495,7 +503,8 @@ def get_comparison_value(ref, r, force_valid):
         )
 
         for runner in cfg.runners.values():
-            assert runner.plot is not None
+            if subplot.runners and runner.nickname not in subplot.runners:
+                continue
             runner_is_mapped = runner.nickname in subplot.runner_map
             if subplot.runner_map and not runner_is_mapped:
                 continue

bench_runner/result.py

@@ -34,6 +34,12 @@
 CombinedData = list[tuple[str, np.ndarray | None, float]]
 
 
+@functools.lru_cache(maxsize=100)
+def _load_contents(filename: Path) -> dict[str, Any]:
+    with filename.open("rb") as fd:
+        return json.load(fd)
+
+
 def _clean(string: str) -> str:
     """
     Clean an arbitrary string to be suitable for a filename.
@@ -599,16 +605,15 @@ def result_info(self) -> tuple[str | None, str | None, str | None]:
             f"Unknown result type (extra={self.extra} suffix={self.suffix})"
         )
 
-    @functools.cached_property
+    @property
     def contents(self) -> dict[str, Any]:
-        with self.filename.open("rb") as fd:
-            return json.load(fd)
+        return _load_contents(self.filename)
 
-    @property
+    @functools.cached_property
     def metadata(self) -> dict[str, Any]:
         return self.contents.get("metadata", {})
 
-    @property
+    @functools.cached_property
     def commit_datetime(self) -> str:
         if self._commit_datetime is not None:
             return self._commit_datetime
@@ -618,7 +623,7 @@ def commit_datetime(self) -> str:
     def commit_date(self) -> str:
         return self.commit_datetime[:10]
 
-    @property
+    @functools.cached_property
     def run_datetime(self) -> str:
         return self.contents["benchmarks"][0]["runs"][0]["metadata"]["date"]
 

bench_runner/templates/workflow_bootstrap.src.py

@@ -59,12 +59,12 @@ def install_requirements(venv: Path) -> None:
 
 def main():
     venv = Path("venv")
-    print("::group::Creating venv")
+    print("::group::Creating venv", file=sys.stderr)
     create_venv(venv)
-    print("::endgroup::")
-    print("::group::Installing requirements")
+    print("::endgroup::", file=sys.stderr)
+    print("::group::Installing requirements", file=sys.stderr)
     install_requirements(venv)
-    print("::endgroup::")
+    print("::endgroup::", file=sys.stderr)
 
     # Now that we've installed the full bench_runner library,
     # continue on in a new process...

bench_runner/util.py

@@ -109,11 +109,11 @@ def valid_version(version: str) -> bool:
 
     @contextlib.contextmanager
     def log_group(text: str) -> Iterator:
-        print(f"::group::{text}")
+        print(f"::group::{text}", file=sys.stderr)
         try:
             yield
         finally:
-            print("::endgroup::")
+            print("::endgroup::", file=sys.stderr)
 
     def track(iterable: Iterable, name: str) -> Iterable:
         with log_group(name):

tests/data/bench_runner.toml

@@ -12,13 +12,14 @@ plot = { name = "Linux", color = "C0" }
 subplots = [
     { base = "3.10.4", version = "3.11" },
     { base = "3.12.0", version = "3.13" },
-    { base = "3.13.0b2", version = "3.14" },
+    { base = "3.13.0b2", version = "3.14", runners = ["linux"] },
     { base = "3.13.0b2", version = "3.14", flags = ["JIT"] }
 ]
 
 [flag_effect_plot]
 subplots = [
     { name = "JIT", version = "3.14", head_flags = ["JIT"] },
+    { name = "NOGIL", version = "3.14", head_flags = ["NOGIL"], runners = ["linux"] },
 ]
 
 [benchmark_longitudinal_plot]