feat(evals): add shared case groups and selectable test suites

evals/prompt_compare.py

@@ -9,29 +9,47 @@ from contextlib import redirect_stdout
 from hashlib import sha256
 import json
 from pathlib import Path
+import re
 import subprocess
 import time
 
-from fastapi import FastAPI
-from fastapi.testclient import TestClient
-from src.core.config import Config
-from src.api.endpoints import router
-from src.core.fastgpt_client import lifespan
-from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript
+import yaml
 
-# Exercise the production text router and backend lifespan, without loading
-# unrelated voice models or downloading their assets during text evaluations.
-app = FastAPI(lifespan=lifespan)
-app.include_router(router)
+from evals.catalog import add_selection_arguments, listing_requested, print_case_listing, select_cases
+from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript
 
 
 def main():
     parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument('--output-dir', type=Path, required=True)
+    parser.add_argument('--output-dir', type=Path)
     parser.add_argument('--workers', type=int, default=4)
-    parser.add_argument('--case', action='append', default=[])
+    parser.add_argument('--cases-dir', type=Path, default=DEFAULT_CASES_DIR)
+    add_selection_arguments(parser)
     parser.add_argument('--prompt-dir', type=Path, help='Optional prompt snapshot; other prompts use the project version')
     args = parser.parse_args()
+    try:
+        catalog = load_cases(args.cases_dir, [])
+        cases = select_cases(catalog, names=args.case, groups=args.group, suites=args.suite,
+                             mode=args.mode, exclude_names=args.exclude_case)
+    except (OSError, ValueError, yaml.YAMLError, re.error) as exc:
+        parser.error(str(exc))
+    if listing_requested(args):
+        print_case_listing(args, catalog, cases)
+        return 0
+    if args.output_dir is None:
+        parser.error('--output-dir is required when running evaluations')
+    if args.workers < 1:
+        parser.error('--workers must be at least 1')
+
+    # Previewing the catalog never imports or starts the application/model client.
+    from fastapi import FastAPI
+    from fastapi.testclient import TestClient
+    from src.core.config import Config
+    from src.api.endpoints import router
+    from src.core.fastgpt_client import lifespan
+
+    app = FastAPI(lifespan=lifespan)
+    app.include_router(router)
     if args.prompt_dir:
         import src.agent.prompts as registry
         original_loader = registry.load_prompt
@@ -42,14 +60,16 @@ def main():
     if Config.AGENT_BACKEND != 'langgraph' or not Config.LLM_PROVIDER or Config.LLM_PROVIDER == 'fake':
         raise SystemExit('Requires langgraph backend and real openai provider')
     args.output_dir.mkdir(parents=True, exist_ok=True)
-    cases = load_cases(DEFAULT_CASES_DIR, args.case)
     from src.agent.prompts import load_prompt
     metadata = {
         **{k: v for k, v in (Config.llm_profile(Config.LLM_PROVIDER) or {}).items() if k in ('model', 'temperature')},
         'backend': Config.AGENT_BACKEND, 'workers': args.workers,
         'commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
-        'case_hashes': {str(p.relative_to(DEFAULT_CASES_DIR)): sha256(p.read_bytes()).hexdigest()
-                        for p in sorted(DEFAULT_CASES_DIR.rglob('*.yaml'))},
+        'selection': {'case': args.case, 'group': args.group, 'suite': args.suite,
+                      'mode': args.mode, 'exclude_case': args.exclude_case},
+        'selected_cases': [case['name'] for case in cases],
+        'case_hashes': {str(p.relative_to(args.cases_dir)): sha256(p.read_bytes()).hexdigest()
+                        for p in sorted(args.cases_dir.rglob('*.yaml'))},
         'prompt_hashes': {p.name: sha256(load_prompt(p.stem).encode('utf-8')).hexdigest()
                           for p in Path('src/agent/prompts').glob('*.md')},    }
     (args.output_dir / 'metadata.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
@@ -79,7 +99,8 @@ def main():
                         json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8')
                     log.flush()
     print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0 if summary['passed'] == summary['total'] else 1
 
 
 if __name__ == '__main__':
-    main()
+    raise SystemExit(main())