ZNJJ-api-server/evals/prompt_compare.py

"""Compare unchanged live evaluation cases against the local LangGraph application.

Run with: conda run -n py3.11 python -m evals.prompt_compare --output-dir ...
Uses the configured real LLM; never substitutes a fake provider.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import redirect_stdout
from hashlib import sha256
import json
from pathlib import Path
import re
import subprocess
import time

import yaml

from evals.catalog import add_selection_arguments, listing_requested, print_case_listing, select_cases
from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript


def main():
    parser = argparse.ArgumentParser(__doc__)
    parser.add_argument('--output-dir', type=Path)
    parser.add_argument('--workers', type=int, default=4)
    parser.add_argument('--cases-dir', type=Path, default=DEFAULT_CASES_DIR)
    add_selection_arguments(parser)
    parser.add_argument('--prompt-dir', type=Path, help='Optional prompt snapshot; other prompts use the project version')
    args = parser.parse_args()
    try:
        catalog = load_cases(args.cases_dir, [])
        cases = select_cases(catalog, names=args.case, groups=args.group, suites=args.suite,
                             mode=args.mode, exclude_names=args.exclude_case)
    except (OSError, ValueError, yaml.YAMLError, re.error) as exc:
        parser.error(str(exc))
    if listing_requested(args):
        print_case_listing(args, catalog, cases)
        return 0
    if args.output_dir is None:
        parser.error('--output-dir is required when running evaluations')
    if args.workers < 1:
        parser.error('--workers must be at least 1')

    # Previewing the catalog never imports or starts the application/model client.
    from fastapi import FastAPI
    from fastapi.testclient import TestClient
    from src.core.config import Config
    from src.api.endpoints import router
    from src.core.fastgpt_client import lifespan

    app = FastAPI(lifespan=lifespan)
    app.include_router(router)
    if args.prompt_dir:
        import src.agent.prompts as registry
        original_loader = registry.load_prompt
        def snapshot_loader(name):
            path = args.prompt_dir / (name+'.md')
            return path.read_text(encoding='utf-8') if path.exists() else original_loader(name)
        registry.load_prompt = snapshot_loader
    if Config.AGENT_BACKEND != 'langgraph' or not Config.LLM_PROVIDER or Config.LLM_PROVIDER == 'fake':
        raise SystemExit('Requires langgraph backend and real openai provider')
    args.output_dir.mkdir(parents=True, exist_ok=True)
    from src.agent.prompts import load_prompt
    metadata = {
        **{k: v for k, v in (Config.llm_profile(Config.LLM_PROVIDER) or {}).items() if k in ('model', 'temperature')},
        'backend': Config.AGENT_BACKEND, 'workers': args.workers,
        'commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
        'selection': {'case': args.case, 'group': args.group, 'suite': args.suite,
                      'mode': args.mode, 'exclude_case': args.exclude_case},
        'selected_cases': [case['name'] for case in cases],
        'case_hashes': {str(p.relative_to(args.cases_dir)): sha256(p.read_bytes()).hexdigest()
                        for p in sorted(args.cases_dir.rglob('*.yaml'))},
        'prompt_hashes': {p.name: sha256(load_prompt(p.stem).encode('utf-8')).hexdigest()
                          for p in Path('src/agent/prompts').glob('*.md')},    }
    (args.output_dir / 'metadata.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
    started = time.monotonic()
    results = []
    # TestClient runs the real HTTP endpoint and application lifespan in process.
    # The original runner, inputs, expectations and cleanup stay unchanged.
    with (args.output_dir / 'dialogues.log').open('w', encoding='utf-8') as log:
        with redirect_stdout(log), TestClient(app) as client:
            with ThreadPoolExecutor(max_workers=args.workers) as pool:
                pending = {pool.submit(run_case, client, case): case for case in cases}
                for future in as_completed(pending):
                    case = pending[future]
                    try:
                        result = future.result()
                        save_transcript(args.output_dir, result)
                        results.append({'name': result.name, 'passed': result.passed,
                                        'failures': [t.error for t in result.turns if not t.ok]})
                    except Exception as exc:
                        results.append({'name': case['name'], 'passed': False,
                                        'failures': [type(exc).__name__ + ': ' + str(exc)]})
                    summary = {'completed': len(results), 'total': len(cases),
                               'passed': sum(r['passed'] for r in results),
                               'duration_seconds': round(time.monotonic() - started, 2),
                               'results': sorted(results, key=lambda r: r['name'])}
                    (args.output_dir / 'summary.json').write_text(
                        json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8')
                    log.flush()
    print(json.dumps(summary, ensure_ascii=False, indent=2))
    return 0 if summary['passed'] == summary['total'] else 1


if __name__ == '__main__':
    raise SystemExit(main())