"""Shared case selection for remote API and local LangGraph evaluations.""" import argparse from collections import Counter from pathlib import Path import yaml SUITES_PATH = Path(__file__).with_name("suites.yaml") def load_suites(path: Path = SUITES_PATH) -> dict[str, dict]: suites = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(suites, dict): raise ValueError(f"{path}: suites must be an object") for name, suite in suites.items(): if not isinstance(name, str) or not isinstance(suite, dict): raise ValueError(f"{path}: invalid suite {name!r}") patterns = suite.get("include") if not isinstance(patterns, list) or not patterns or not all( isinstance(pattern, str) and pattern for pattern in patterns ): raise ValueError(f"{path}: suite {name!r} needs nonempty include patterns") return suites def add_selection_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--case", action="append", default=[], help="Name substring; repeat to match any") parser.add_argument("--group", action="append", default=[], help="Exact case directory group; repeat to match any") parser.add_argument("--suite", action="append", default=[], help="Named suite from evals/suites.yaml; repeat for union") parser.add_argument("--mode", choices=("direct", "browser_addon"), help="Only this client mode") parser.add_argument("--exclude-case", action="append", default=[], help="Exclude names containing this value; repeatable") listing = parser.add_mutually_exclusive_group() listing.add_argument("--list", action="store_true", dest="list_cases", help="List selected cases without API/model calls") listing.add_argument("--list-groups", action="store_true", help="List directory groups and counts without running") listing.add_argument("--list-suites", action="store_true", help="List named suites and counts without running") def select_cases( cases: list[dict], *, names=(), groups=(), suites=(), mode=None, exclude_names=(), suite_definitions: dict | None = None, ) -> list[dict]: """Union within each filter; intersection between filters; exclusions last. Operate on the loaded catalog once, so overlapping suites never duplicate runs. Client modes and different assertions remain independent coverage. """ available_groups = {case["_group"] for case in cases} unknown = set(groups) - available_groups if unknown: raise ValueError(f"Unknown groups: {sorted(unknown)}; available: {sorted(available_groups)}") definitions = load_suites() if suite_definitions is None else suite_definitions unknown = set(suites) - definitions.keys() if unknown: raise ValueError(f"Unknown suites: {sorted(unknown)}; available: {sorted(definitions)}") patterns = [pattern for name in suites for pattern in definitions[name]["include"]] selected = [case for case in cases if (not names or any(value in case["name"] for value in names)) and (not groups or case["_group"] in groups) and (not patterns or any(Path(case["_relative_path"]).match(pattern) for pattern in patterns)) and (mode is None or case["clientMode"] == mode) and not any(value in case["name"] for value in exclude_names)] if not selected: raise ValueError("No evaluation cases match the selected filters") return selected def listing_requested(args) -> bool: return args.list_cases or args.list_groups or args.list_suites def print_case_listing(args, catalog: list[dict], selected: list[dict]) -> None: if args.list_groups: counts = Counter(case["_group"] for case in selected) for group, count in sorted(counts.items()): print(f"{group}\t{count}") elif args.list_suites: for name, suite in load_suites().items(): count = sum(any(Path(case["_relative_path"]).match(pattern) for pattern in suite["include"]) for case in selected) print(f"{name}\t{count}\t{suite.get('description', '')}") else: for case in selected: print(f"{case['name']}\t{case['clientMode']}\t{case['_relative_path']}") print(f"Selected {len(selected)} of {len(catalog)} cases")