feat(evals): add shared case groups and selectable test suites

evals/run.py

@@ -114,11 +114,17 @@ def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]:
         if name_filters and not any(value in name for value in name_filters):
             continue
 
-        case["_path"] = str(path.relative_to(EVALS_DIR))
+        relative_path = path.relative_to(cases_dir)
+        case["_relative_path"] = relative_path.as_posix()
+        case["_group"] = relative_path.parent.as_posix()
+        case["_path"] = str(path.relative_to(EVALS_DIR)) if path.is_relative_to(EVALS_DIR) else str(path)
         cases.append(case)
 
     if not cases:
         raise ValueError(f"No evaluation cases found under {cases_dir}")
+    names = [case["name"] for case in cases]
+    if len(set(names)) != len(names):
+        raise ValueError("Evaluation case names must be unique")
     return cases
 
 
@@ -435,12 +441,12 @@ def parse_args() -> argparse.Namespace:
         default=DEFAULT_CASES_DIR,
         help="Directory containing YAML cases",
     )
-    parser.add_argument(
-        "--case",
-        action="append",
-        default=[],
-        help="Run cases whose names contain this value; repeatable",
-    )
+    # Support both `python evals/run.py` and `python -m evals.run`.
+    try:
+        from .catalog import add_selection_arguments
+    except ImportError:
+        from catalog import add_selection_arguments
+    add_selection_arguments(parser)
     parser.add_argument(
         "--output-dir",
         type=Path,
@@ -467,11 +473,21 @@ def parse_args() -> argparse.Namespace:
 def main() -> int:
     args = parse_args()
     try:
-        cases = load_cases(args.cases_dir, args.case)
+        from .catalog import listing_requested, print_case_listing, select_cases
+    except ImportError:
+        from catalog import listing_requested, print_case_listing, select_cases
+    try:
+        catalog = load_cases(args.cases_dir, [])
+        cases = select_cases(catalog, names=args.case, groups=args.group, suites=args.suite,
+                             mode=args.mode, exclude_names=args.exclude_case)
     except (OSError, ValueError, yaml.YAMLError, re.error) as exc:
         print(f"ERROR: {exc}", file=sys.stderr)
         return 2
 
+    if listing_requested(args):
+        print_case_listing(args, catalog, cases)
+        return 0
+
     output_dir: Path | None = None
     if not args.no_save:
         output_dir = args.output_dir or (DEFAULT_OUTPUT_DIR / local_run_stamp())