feat(agent): trusted time evidence and collection prompt optimization

- time_context.py: server-side observation clocks for relative-time
  evidence (anti client-forgery); resolves X分钟前/小时前 against the
  observation timestamp of the turn that produced them, never re-basing
  history on the current clock; flags historical references without a
  reliable baseline
- state: dialogue_timestamps keyed by dialogue message index, persisted
  with checkpoint; welcome seeds index 0
- collect: single clock per turn shared by reply/extraction/repair;
  time_evidence injected into runtime context; reply_validator hook
  rejects time claims lacking evidence (one shared repair budget);
  model still interprets facts and chooses actions
- prompts: collection prompt restructured (shorter, state-bound-to-action
  table, gatekeeper principle, time source rules); extraction prompt
  aligned; workflow@20261021 json updated to match
- evals: prompt_probes / prompt_compare / time_boundary probe scripts
  (supplementary checks, original cases untouched)
- docs: collection-prompt-optimization.md (method + baselines),
  time-boundary-fix.md (problem, design, verification)
- gitignore: editor artifact dirs (.agents/.claude/.vscode, skills-lock)

Verified: 171 unit tests pass (16 new time-context tests)
This commit is contained in:
Eric Wang
2026-10-22 09:41:32 +08:00
parent a890f41f6f
commit 9e28f69b3c
14 changed files with 827 additions and 588 deletions
+85
View File
@@ -0,0 +1,85 @@
"""Compare unchanged live evaluation cases against the local LangGraph application.
Run with: conda run -n py3.11 python -m evals.prompt_compare --output-dir ...
Uses the configured real LLM; never substitutes a fake provider.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import redirect_stdout
from hashlib import sha256
import json
from pathlib import Path
import subprocess
import time
from fastapi import FastAPI
from fastapi.testclient import TestClient
from src.core.config import Config
from src.api.endpoints import router
from src.core.fastgpt_client import lifespan
from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript
# Exercise the production text router and backend lifespan, without loading
# unrelated voice models or downloading their assets during text evaluations.
app = FastAPI(lifespan=lifespan)
app.include_router(router)
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument('--workers', type=int, default=4)
parser.add_argument('--case', action='append', default=[])
parser.add_argument('--prompt-dir', type=Path, help='Optional prompt snapshot; other prompts use the project version')
args = parser.parse_args()
if args.prompt_dir:
import src.agent.prompts as registry
original_loader = registry.load_prompt
def snapshot_loader(name):
path = args.prompt_dir / (name+'.md')
return path.read_text(encoding='utf-8') if path.exists() else original_loader(name)
registry.load_prompt = snapshot_loader
if Config.AGENT_BACKEND != 'langgraph' or Config.AGENT_LLM_PROVIDER != 'openai':
raise SystemExit('Requires langgraph backend and real openai provider')
args.output_dir.mkdir(parents=True, exist_ok=True)
cases = load_cases(DEFAULT_CASES_DIR, args.case)
from src.agent.prompts import load_prompt
metadata = {
'model': Config.LLM_MODEL, 'temperature': Config.LLM_TEMPERATURE,
'backend': Config.AGENT_BACKEND, 'workers': args.workers,
'commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
'case_hashes': {str(p.relative_to(DEFAULT_CASES_DIR)): sha256(p.read_bytes()).hexdigest()
for p in sorted(DEFAULT_CASES_DIR.rglob('*.yaml'))},
'prompt_hashes': {p.name: sha256(load_prompt(p.stem).encode('utf-8')).hexdigest()
for p in Path('src/agent/prompts').glob('*.md')}, }
(args.output_dir / 'metadata.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
started = time.monotonic()
results = []
# TestClient runs the real HTTP endpoint and application lifespan in process.
# The original runner, inputs, expectations and cleanup stay unchanged.
with (args.output_dir / 'dialogues.log').open('w', encoding='utf-8') as log:
with redirect_stdout(log), TestClient(app) as client:
with ThreadPoolExecutor(max_workers=args.workers) as pool:
pending = {pool.submit(run_case, client, case): case for case in cases}
for future in as_completed(pending):
case = pending[future]
try:
result = future.result()
save_transcript(args.output_dir, result)
results.append({'name': result.name, 'passed': result.passed,
'failures': [t.error for t in result.turns if not t.ok]})
except Exception as exc:
results.append({'name': case['name'], 'passed': False,
'failures': [type(exc).__name__ + ': ' + str(exc)]})
summary = {'completed': len(results), 'total': len(cases),
'passed': sum(r['passed'] for r in results),
'duration_seconds': round(time.monotonic() - started, 2),
'results': sorted(results, key=lambda r: r['name'])}
(args.output_dir / 'summary.json').write_text(
json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8')
log.flush()
print(json.dumps(summary, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()