94 lines
4.4 KiB
Python
94 lines
4.4 KiB
Python
|
"""Shared case selection for remote API and local LangGraph evaluations."""
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
from collections import Counter
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import yaml
|
||
|
|
|
||
|
|
SUITES_PATH = Path(__file__).with_name("suites.yaml")
|
||
|
|
|
||
|
|
|
||
|
|
def load_suites(path: Path = SUITES_PATH) -> dict[str, dict]:
|
||
|
|
suites = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||
|
|
if not isinstance(suites, dict):
|
||
|
|
raise ValueError(f"{path}: suites must be an object")
|
||
|
|
for name, suite in suites.items():
|
||
|
|
if not isinstance(name, str) or not isinstance(suite, dict):
|
||
|
|
raise ValueError(f"{path}: invalid suite {name!r}")
|
||
|
|
patterns = suite.get("include")
|
||
|
|
if not isinstance(patterns, list) or not patterns or not all(
|
||
|
|
isinstance(pattern, str) and pattern for pattern in patterns
|
||
|
|
):
|
||
|
|
raise ValueError(f"{path}: suite {name!r} needs nonempty include patterns")
|
||
|
|
return suites
|
||
|
|
|
||
|
|
|
||
|
|
def add_selection_arguments(parser: argparse.ArgumentParser) -> None:
|
||
|
|
parser.add_argument("--case", action="append", default=[],
|
||
|
|
help="Name substring; repeat to match any")
|
||
|
|
parser.add_argument("--group", action="append", default=[],
|
||
|
|
help="Exact case directory group; repeat to match any")
|
||
|
|
parser.add_argument("--suite", action="append", default=[],
|
||
|
|
help="Named suite from evals/suites.yaml; repeat for union")
|
||
|
|
parser.add_argument("--mode", choices=("direct", "browser_addon"),
|
||
|
|
help="Only this client mode")
|
||
|
|
parser.add_argument("--exclude-case", action="append", default=[],
|
||
|
|
help="Exclude names containing this value; repeatable")
|
||
|
|
listing = parser.add_mutually_exclusive_group()
|
||
|
|
listing.add_argument("--list", action="store_true", dest="list_cases",
|
||
|
|
help="List selected cases without API/model calls")
|
||
|
|
listing.add_argument("--list-groups", action="store_true",
|
||
|
|
help="List directory groups and counts without running")
|
||
|
|
listing.add_argument("--list-suites", action="store_true",
|
||
|
|
help="List named suites and counts without running")
|
||
|
|
|
||
|
|
|
||
|
|
def select_cases(
|
||
|
|
cases: list[dict], *, names=(), groups=(), suites=(), mode=None,
|
||
|
|
exclude_names=(), suite_definitions: dict | None = None,
|
||
|
|
) -> list[dict]:
|
||
|
|
"""Union within each filter; intersection between filters; exclusions last.
|
||
|
|
|
||
|
|
Operate on the loaded catalog once, so overlapping suites never duplicate
|
||
|
|
runs. Client modes and different assertions remain independent coverage.
|
||
|
|
"""
|
||
|
|
available_groups = {case["_group"] for case in cases}
|
||
|
|
unknown = set(groups) - available_groups
|
||
|
|
if unknown:
|
||
|
|
raise ValueError(f"Unknown groups: {sorted(unknown)}; available: {sorted(available_groups)}")
|
||
|
|
definitions = load_suites() if suite_definitions is None else suite_definitions
|
||
|
|
unknown = set(suites) - definitions.keys()
|
||
|
|
if unknown:
|
||
|
|
raise ValueError(f"Unknown suites: {sorted(unknown)}; available: {sorted(definitions)}")
|
||
|
|
patterns = [pattern for name in suites for pattern in definitions[name]["include"]]
|
||
|
|
selected = [case for case in cases
|
||
|
|
if (not names or any(value in case["name"] for value in names))
|
||
|
|
and (not groups or case["_group"] in groups)
|
||
|
|
and (not patterns or any(Path(case["_relative_path"]).match(pattern) for pattern in patterns))
|
||
|
|
and (mode is None or case["clientMode"] == mode)
|
||
|
|
and not any(value in case["name"] for value in exclude_names)]
|
||
|
|
if not selected:
|
||
|
|
raise ValueError("No evaluation cases match the selected filters")
|
||
|
|
return selected
|
||
|
|
|
||
|
|
|
||
|
|
def listing_requested(args) -> bool:
|
||
|
|
return args.list_cases or args.list_groups or args.list_suites
|
||
|
|
|
||
|
|
|
||
|
|
def print_case_listing(args, catalog: list[dict], selected: list[dict]) -> None:
|
||
|
|
if args.list_groups:
|
||
|
|
counts = Counter(case["_group"] for case in selected)
|
||
|
|
for group, count in sorted(counts.items()):
|
||
|
|
print(f"{group}\t{count}")
|
||
|
|
elif args.list_suites:
|
||
|
|
for name, suite in load_suites().items():
|
||
|
|
count = sum(any(Path(case["_relative_path"]).match(pattern)
|
||
|
|
for pattern in suite["include"]) for case in selected)
|
||
|
|
print(f"{name}\t{count}\t{suite.get('description', '')}")
|
||
|
|
else:
|
||
|
|
for case in selected:
|
||
|
|
print(f"{case['name']}\t{case['clientMode']}\t{case['_relative_path']}")
|
||
|
|
print(f"Selected {len(selected)} of {len(catalog)} cases")
|