feat(evals): add shared case groups and selectable test suites

evals/catalog.py

@@ -0,0 +1,93 @@
+"""Shared case selection for remote API and local LangGraph evaluations."""
+
+import argparse
+from collections import Counter
+from pathlib import Path
+
+import yaml
+
+SUITES_PATH = Path(__file__).with_name("suites.yaml")
+
+
+def load_suites(path: Path = SUITES_PATH) -> dict[str, dict]:
+    suites = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(suites, dict):
+        raise ValueError(f"{path}: suites must be an object")
+    for name, suite in suites.items():
+        if not isinstance(name, str) or not isinstance(suite, dict):
+            raise ValueError(f"{path}: invalid suite {name!r}")
+        patterns = suite.get("include")
+        if not isinstance(patterns, list) or not patterns or not all(
+            isinstance(pattern, str) and pattern for pattern in patterns
+        ):
+            raise ValueError(f"{path}: suite {name!r} needs nonempty include patterns")
+    return suites
+
+
+def add_selection_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--case", action="append", default=[],
+                        help="Name substring; repeat to match any")
+    parser.add_argument("--group", action="append", default=[],
+                        help="Exact case directory group; repeat to match any")
+    parser.add_argument("--suite", action="append", default=[],
+                        help="Named suite from evals/suites.yaml; repeat for union")
+    parser.add_argument("--mode", choices=("direct", "browser_addon"),
+                        help="Only this client mode")
+    parser.add_argument("--exclude-case", action="append", default=[],
+                        help="Exclude names containing this value; repeatable")
+    listing = parser.add_mutually_exclusive_group()
+    listing.add_argument("--list", action="store_true", dest="list_cases",
+                         help="List selected cases without API/model calls")
+    listing.add_argument("--list-groups", action="store_true",
+                         help="List directory groups and counts without running")
+    listing.add_argument("--list-suites", action="store_true",
+                         help="List named suites and counts without running")
+
+
+def select_cases(
+    cases: list[dict], *, names=(), groups=(), suites=(), mode=None,
+    exclude_names=(), suite_definitions: dict | None = None,
+) -> list[dict]:
+    """Union within each filter; intersection between filters; exclusions last.
+
+    Operate on the loaded catalog once, so overlapping suites never duplicate
+    runs. Client modes and different assertions remain independent coverage.
+    """
+    available_groups = {case["_group"] for case in cases}
+    unknown = set(groups) - available_groups
+    if unknown:
+        raise ValueError(f"Unknown groups: {sorted(unknown)}; available: {sorted(available_groups)}")
+    definitions = load_suites() if suite_definitions is None else suite_definitions
+    unknown = set(suites) - definitions.keys()
+    if unknown:
+        raise ValueError(f"Unknown suites: {sorted(unknown)}; available: {sorted(definitions)}")
+    patterns = [pattern for name in suites for pattern in definitions[name]["include"]]
+    selected = [case for case in cases
+                if (not names or any(value in case["name"] for value in names))
+                and (not groups or case["_group"] in groups)
+                and (not patterns or any(Path(case["_relative_path"]).match(pattern) for pattern in patterns))
+                and (mode is None or case["clientMode"] == mode)
+                and not any(value in case["name"] for value in exclude_names)]
+    if not selected:
+        raise ValueError("No evaluation cases match the selected filters")
+    return selected
+
+
+def listing_requested(args) -> bool:
+    return args.list_cases or args.list_groups or args.list_suites
+
+
+def print_case_listing(args, catalog: list[dict], selected: list[dict]) -> None:
+    if args.list_groups:
+        counts = Counter(case["_group"] for case in selected)
+        for group, count in sorted(counts.items()):
+            print(f"{group}\t{count}")
+    elif args.list_suites:
+        for name, suite in load_suites().items():
+            count = sum(any(Path(case["_relative_path"]).match(pattern)
+                            for pattern in suite["include"]) for case in selected)
+            print(f"{name}\t{count}\t{suite.get('description', '')}")
+    else:
+        for case in selected:
+            print(f"{case['name']}\t{case['clientMode']}\t{case['_relative_path']}")
+    print(f"Selected {len(selected)} of {len(catalog)} cases")