feat(experiment): tighten RunnerContext + RegressionError public surface

- RunnerContext no longer carries `name` or `run_name` as context-level
  defaults. `name` is now required on every `run_experiment` call
  (supports the action's directory-of-experiments mode where each
  script must name itself). `run_name` passes straight through to
  `Langfuse.run_experiment`.
- RegressionError gains three typed `@overload` signatures (minimal,
  free-form message, structured metric/value/threshold) so type
  checkers enforce that `metric` and `value` are supplied together.
  At runtime, partial structured input falls back to the default
  message instead of rendering misleading `None` placeholders in PR
  comments.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: wochinge

langfuse/experiment.py

@@ -17,6 +17,7 @@
     Protocol,
     TypedDict,
     Union,
+    overload,
 )
 
 from langfuse.api import DatasetItem
@@ -1063,10 +1064,9 @@ class RunnerContext:
     Intended for use with the ``langfuse/experiment-action`` GitHub Action
     (https://github.com/langfuse/experiment-action). The action builds a
     ``RunnerContext`` before invoking the user's ``experiment(context)``
-    function. Defaults set here (dataset, name, run name, metadata tags) are
-    applied when the user omits them on the :meth:`run_experiment` call;
-    users can override any default by passing the corresponding argument
-    explicitly.
+    function. Defaults set here (dataset, metadata tags) are applied when
+    the user omits them on the :meth:`run_experiment` call; users can
+    override any default by passing the corresponding argument explicitly.
     """
 
     def __init__(
@@ -1075,8 +1075,6 @@ def __init__(
         client: "Langfuse",
         data: Optional[ExperimentData] = None,
         dataset_version: Optional[datetime] = None,
-        name: Optional[str] = None,
-        run_name: Optional[str] = None,
         metadata: Optional[Dict[str, str]] = None,
     ):
         """Build a ``RunnerContext`` populated with defaults for ``run_experiment``.
@@ -1098,12 +1096,6 @@ def __init__(
                 :meth:`run_experiment`.
             dataset_version: Optional pinned dataset version. Injected by the
                 action when ``dataset_version`` is configured.
-            name: Default human-readable experiment name (e.g. the action's
-                ``experiment_name`` input). If ``None``, the user must pass
-                ``name=`` to :meth:`run_experiment`.
-            run_name: Default exact run name. The action typically derives
-                this from the commit SHA / PR number so that reruns produce
-                distinct runs in Langfuse.
             metadata: Default metadata attached to every experiment trace and
                 the dataset run. The action injects GitHub-sourced tags (SHA,
                 PR link, workflow run link, branch, GH user, etc.). Merged
@@ -1113,14 +1105,12 @@ def __init__(
         self.client = client
         self.data = data
         self.dataset_version = dataset_version
-        self.name = name
-        self.run_name = run_name
         self.metadata = metadata
 
     def run_experiment(
         self,
         *,
-        name: Optional[str] = None,
+        name: str,
         run_name: Optional[str] = None,
         description: Optional[str] = None,
         data: Optional[ExperimentData] = None,
@@ -1132,19 +1122,12 @@ def run_experiment(
         metadata: Optional[Dict[str, str]] = None,
         _dataset_version: Optional[datetime] = None,
     ) -> ExperimentResult:
-        resolved_name = name if name is not None else self.name
-        if resolved_name is None:
-            raise ValueError(
-                "`name` must be provided either on the RunnerContext or the run_experiment call"
-            )
-
         resolved_data = data if data is not None else self.data
         if resolved_data is None:
             raise ValueError(
                 "`data` must be provided either on the RunnerContext or the run_experiment call"
             )
 
-        resolved_run_name = run_name if run_name is not None else self.run_name
         resolved_dataset_version = (
             _dataset_version if _dataset_version is not None else self.dataset_version
         )
@@ -1156,8 +1139,8 @@ def run_experiment(
             merged_metadata = {**(self.metadata or {}), **(metadata or {})}
 
         return self.client.run_experiment(
-            name=resolved_name,
-            run_name=resolved_run_name,
+            name=name,
+            run_name=run_name,
             description=description,
             data=resolved_data,
             task=task,
@@ -1178,8 +1161,30 @@ class RegressionError(Exception):
     exception and, when ``should_fail_on_error`` is enabled, fails the
     workflow run and renders a callout in the PR comment using
     ``metric``/``value``/``threshold`` if supplied, otherwise ``str(exc)``.
+
+    Callers choose one of three forms:
+
+    - ``RegressionError(result=r)`` — minimal, generic message.
+    - ``RegressionError(result=r, message="...")`` — free-form message.
+    - ``RegressionError(result=r, metric="acc", value=0.7, threshold=0.9)`` —
+      structured; ``metric`` and ``value`` must be provided together so the
+      action can render a targeted callout without ``None`` placeholders.
     """
 
+    @overload
+    def __init__(self, *, result: ExperimentResult) -> None: ...
+    @overload
+    def __init__(self, *, result: ExperimentResult, message: str) -> None: ...
+    @overload
+    def __init__(
+        self,
+        *,
+        result: ExperimentResult,
+        metric: str,
+        value: float,
+        threshold: Optional[float] = None,
+        message: Optional[str] = None,
+    ) -> None: ...
     def __init__(
         self,
         *,
@@ -1195,7 +1200,7 @@ def __init__(
         self.threshold = threshold
         if message is not None:
             formatted = message
-        elif metric is not None:
+        elif metric is not None and value is not None:
             formatted = f"Regression on `{metric}`: {value} (threshold {threshold})"
         else:
             formatted = "Experiment regression detected"

tests/unit/test_experiment.py

@@ -30,18 +30,15 @@ def test_context_defaults_flow_through(self):
         ctx = _make_ctx(
             data=ctx_data,
             dataset_version=ctx_version,
-            name="ctx-name",
-            run_name="ctx-run",
             metadata={"sha": "abc123"},
         )
 
-        result = ctx.run_experiment(task=_noop_task)
+        result = ctx.run_experiment(name="exp", task=_noop_task)
 
         assert result == "result-sentinel"
         ctx.client.run_experiment.assert_called_once()
         kwargs = ctx.client.run_experiment.call_args.kwargs
-        assert kwargs["name"] == "ctx-name"
-        assert kwargs["run_name"] == "ctx-run"
+        assert kwargs["name"] == "exp"
         assert kwargs["data"] is ctx_data
         assert kwargs["metadata"] == {"sha": "abc123"}
         assert kwargs["_dataset_version"] == ctx_version
@@ -51,22 +48,20 @@ def test_call_overrides_win(self):
         ctx = _make_ctx(
             data=[{"input": "ctx"}],
             dataset_version=datetime(2026, 1, 1),
-            name="ctx-name",
-            run_name="ctx-run",
         )
 
         override_data = [{"input": "override"}]
         override_version = datetime(2026, 6, 6)
         ctx.run_experiment(
+            name="exp",
             task=_noop_task,
-            name="call-name",
             run_name="call-run",
             data=override_data,
             _dataset_version=override_version,
         )
 
         kwargs = ctx.client.run_experiment.call_args.kwargs
-        assert kwargs["name"] == "call-name"
+        assert kwargs["name"] == "exp"
         assert kwargs["run_name"] == "call-run"
         assert kwargs["data"] is override_data
         assert kwargs["_dataset_version"] == override_version
@@ -76,58 +71,52 @@ class TestRunnerContextMetadataMerge:
     def test_user_keys_win_on_collision(self):
         ctx = _make_ctx(
             data=[{"input": "a"}],
-            name="n",
             metadata={"sha": "abc", "branch": "main"},
         )
-        ctx.run_experiment(task=_noop_task, metadata={"sha": "def", "pr": "42"})
+        ctx.run_experiment(
+            name="exp", task=_noop_task, metadata={"sha": "def", "pr": "42"}
+        )
         assert ctx.client.run_experiment.call_args.kwargs["metadata"] == {
             "sha": "def",
             "branch": "main",
             "pr": "42",
         }
 
     def test_context_metadata_only(self):
-        ctx = _make_ctx(
-            data=[{"input": "a"}], name="n", metadata={"sha": "abc"}
-        )
-        ctx.run_experiment(task=_noop_task)
+        ctx = _make_ctx(data=[{"input": "a"}], metadata={"sha": "abc"})
+        ctx.run_experiment(name="exp", task=_noop_task)
         assert ctx.client.run_experiment.call_args.kwargs["metadata"] == {"sha": "abc"}
 
     def test_call_metadata_only(self):
-        ctx = _make_ctx(data=[{"input": "a"}], name="n")
-        ctx.run_experiment(task=_noop_task, metadata={"pr": "1"})
+        ctx = _make_ctx(data=[{"input": "a"}])
+        ctx.run_experiment(name="exp", task=_noop_task, metadata={"pr": "1"})
         assert ctx.client.run_experiment.call_args.kwargs["metadata"] == {"pr": "1"}
 
     def test_both_none_stays_none(self):
-        ctx = _make_ctx(data=[{"input": "a"}], name="n")
-        ctx.run_experiment(task=_noop_task)
+        ctx = _make_ctx(data=[{"input": "a"}])
+        ctx.run_experiment(name="exp", task=_noop_task)
         assert ctx.client.run_experiment.call_args.kwargs["metadata"] is None
 
 
 class TestRunnerContextLocalItems:
     def test_local_items_pass_through_as_context_default(self):
         items = [{"input": "x", "expected_output": "y"}]
-        ctx = _make_ctx(data=items, name="n")
-        ctx.run_experiment(task=_noop_task)
+        ctx = _make_ctx(data=items)
+        ctx.run_experiment(name="exp", task=_noop_task)
         assert ctx.client.run_experiment.call_args.kwargs["data"] is items
 
     def test_local_items_pass_through_as_call_override(self):
-        ctx = _make_ctx(name="n")
+        ctx = _make_ctx()
         items = [{"input": "x"}]
-        ctx.run_experiment(task=_noop_task, data=items)
+        ctx.run_experiment(name="exp", task=_noop_task, data=items)
         assert ctx.client.run_experiment.call_args.kwargs["data"] is items
 
 
 class TestRunnerContextValidation:
-    def test_missing_name_raises(self):
-        ctx = _make_ctx(data=[{"input": "a"}])
-        with pytest.raises(ValueError, match="name"):
-            ctx.run_experiment(task=_noop_task)
-
     def test_missing_data_raises(self):
-        ctx = _make_ctx(name="n")
+        ctx = _make_ctx()
         with pytest.raises(ValueError, match="data"):
-            ctx.run_experiment(task=_noop_task)
+            ctx.run_experiment(name="exp", task=_noop_task)
 
 
 class TestRegressionError:
@@ -155,7 +144,14 @@ def test_structured_message(self):
         assert "0.78" in str(exc)
         assert "0.9" in str(exc)
 
-    def test_user_message_wins(self):
+    def test_free_form_message(self):
+        exc = RegressionError(
+            result=MagicMock(),
+            message="custom explanation",
+        )
+        assert str(exc) == "custom explanation"
+
+    def test_message_wins_over_structured(self):
         exc = RegressionError(
             result=MagicMock(),
             metric="avg_accuracy",
@@ -164,19 +160,33 @@ def test_user_message_wins(self):
             message="custom explanation",
         )
         assert str(exc) == "custom explanation"
+        assert exc.metric == "avg_accuracy"
+        assert exc.value == 0.5
+        assert exc.threshold == 0.9
+
+    def test_partial_structured_falls_back_to_default(self):
+        """The structured overload requires ``metric`` and ``value`` together.
+
+        If a caller bypasses the type checker and passes only one, we fall
+        back to the default message rather than rendering misleading
+        ``None`` placeholders in the PR comment.
+        """
+        exc = RegressionError(result=MagicMock(), metric="avg_accuracy")  # type: ignore[call-overload]
+        assert str(exc) == "Experiment regression detected"
 
 
 class TestSignatureDriftGuard:
     """Fails loudly if ``Langfuse.run_experiment`` grows a parameter that is
     not threaded through ``RunnerContext.run_experiment``.
 
-    The four action-relaxed params (``name``, ``run_name``, ``data``,
-    ``_dataset_version``) are allowed to diverge: the RunnerContext variant
-    must be the ``Optional[...]`` of the client annotation so the action can
-    inject them.
+    ``data`` is the only genuinely relaxed parameter: it is required on the
+    client but optional on the RunnerContext so the action can inject it.
+    ``run_name`` and ``_dataset_version`` are already ``Optional`` on the
+    client and must match as-is. ``name`` is required on both — the action
+    supports a directory of experiments, so each script must name itself.
     """
 
-    RELAXED_PARAMS = {"name", "run_name", "data", "_dataset_version"}
+    RELAXED_PARAMS = {"data"}
 
     # `CompositeEvaluatorFunction` is only imported under TYPE_CHECKING in
     # ``langfuse.experiment`` to break the circular dependency with