107 lines
5.4 KiB
Python
107 lines
5.4 KiB
Python
"""Compare unchanged live evaluation cases against the local LangGraph application.
|
|
|
|
Run with: conda run -n py3.11 python -m evals.prompt_compare --output-dir ...
|
|
Uses the configured real LLM; never substitutes a fake provider.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from contextlib import redirect_stdout
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
|
|
import yaml
|
|
|
|
from evals.catalog import add_selection_arguments, listing_requested, print_case_listing, select_cases
|
|
from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(__doc__)
|
|
parser.add_argument('--output-dir', type=Path)
|
|
parser.add_argument('--workers', type=int, default=4)
|
|
parser.add_argument('--cases-dir', type=Path, default=DEFAULT_CASES_DIR)
|
|
add_selection_arguments(parser)
|
|
parser.add_argument('--prompt-dir', type=Path, help='Optional prompt snapshot; other prompts use the project version')
|
|
args = parser.parse_args()
|
|
try:
|
|
catalog = load_cases(args.cases_dir, [])
|
|
cases = select_cases(catalog, names=args.case, groups=args.group, suites=args.suite,
|
|
mode=args.mode, exclude_names=args.exclude_case)
|
|
except (OSError, ValueError, yaml.YAMLError, re.error) as exc:
|
|
parser.error(str(exc))
|
|
if listing_requested(args):
|
|
print_case_listing(args, catalog, cases)
|
|
return 0
|
|
if args.output_dir is None:
|
|
parser.error('--output-dir is required when running evaluations')
|
|
if args.workers < 1:
|
|
parser.error('--workers must be at least 1')
|
|
|
|
# Previewing the catalog never imports or starts the application/model client.
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
from src.core.config import Config
|
|
from src.api.endpoints import router
|
|
from src.core.fastgpt_client import lifespan
|
|
|
|
app = FastAPI(lifespan=lifespan)
|
|
app.include_router(router)
|
|
if args.prompt_dir:
|
|
import src.agent.prompts as registry
|
|
original_loader = registry.load_prompt
|
|
def snapshot_loader(name):
|
|
path = args.prompt_dir / (name+'.md')
|
|
return path.read_text(encoding='utf-8') if path.exists() else original_loader(name)
|
|
registry.load_prompt = snapshot_loader
|
|
if Config.AGENT_BACKEND != 'langgraph' or not Config.LLM_PROVIDER or Config.LLM_PROVIDER == 'fake':
|
|
raise SystemExit('Requires langgraph backend and real openai provider')
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
from src.agent.prompts import load_prompt
|
|
metadata = {
|
|
**{k: v for k, v in (Config.llm_profile(Config.LLM_PROVIDER) or {}).items() if k in ('model', 'temperature')},
|
|
'backend': Config.AGENT_BACKEND, 'workers': args.workers,
|
|
'commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
|
|
'selection': {'case': args.case, 'group': args.group, 'suite': args.suite,
|
|
'mode': args.mode, 'exclude_case': args.exclude_case},
|
|
'selected_cases': [case['name'] for case in cases],
|
|
'case_hashes': {str(p.relative_to(args.cases_dir)): sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted(args.cases_dir.rglob('*.yaml'))},
|
|
'prompt_hashes': {p.name: sha256(load_prompt(p.stem).encode('utf-8')).hexdigest()
|
|
for p in Path('src/agent/prompts').glob('*.md')}, }
|
|
(args.output_dir / 'metadata.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
|
|
started = time.monotonic()
|
|
results = []
|
|
# TestClient runs the real HTTP endpoint and application lifespan in process.
|
|
# The original runner, inputs, expectations and cleanup stay unchanged.
|
|
with (args.output_dir / 'dialogues.log').open('w', encoding='utf-8') as log:
|
|
with redirect_stdout(log), TestClient(app) as client:
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
pending = {pool.submit(run_case, client, case): case for case in cases}
|
|
for future in as_completed(pending):
|
|
case = pending[future]
|
|
try:
|
|
result = future.result()
|
|
save_transcript(args.output_dir, result)
|
|
results.append({'name': result.name, 'passed': result.passed,
|
|
'failures': [t.error for t in result.turns if not t.ok]})
|
|
except Exception as exc:
|
|
results.append({'name': case['name'], 'passed': False,
|
|
'failures': [type(exc).__name__ + ': ' + str(exc)]})
|
|
summary = {'completed': len(results), 'total': len(cases),
|
|
'passed': sum(r['passed'] for r in results),
|
|
'duration_seconds': round(time.monotonic() - started, 2),
|
|
'results': sorted(results, key=lambda r: r['name'])}
|
|
(args.output_dir / 'summary.json').write_text(
|
|
json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
log.flush()
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
return 0 if summary['passed'] == summary['total'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|