wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add dynamic variables and startup result handling in batch run schemas

Browse Source 
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types.
- Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages.
- Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs.
- Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases.
- Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
This commit is contained in:
Xin Wang
2026-08-12 16:55:25 +08:00
parent 79a239489a
commit 7560f9a525
12 changed files with 1464 additions and 1004 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/test_runs/orchestrator.py
+119 -5
View File
@@ -13,9 +13,14 @@ from db.models import ModelResource, TestCase, TestRun, TestRunCase
from db.session import SessionLocal
from models import RuntimeModelResource
from services.config_resolver import resolve_runtime_config
from services.runtime_variables import prepare_dynamic_config
from services.test_runs.errors import TestExecutionError
from services.test_runs.evaluator import EvaluationEngine
from services.test_runs.text_runner import TextPipelineRunner
from services.test_runs.text_runner import (
RawTurnResult,
TextPipelineRunner,
TextStartupResult,
)
from test_schemas import TestCaseDefinition
@@ -74,12 +79,20 @@ class TestRunOrchestrator:
run.status = "running"
run.started_at = run.started_at or datetime.now(UTC)
await session.commit()
config = dict(run.config or {})
try:
cfg = await resolve_runtime_config(session, str(run.assistant_id or ""))
# Validate the run-level values once. Each case/attempt gets
# a fresh prepared copy below so conversation-scoped system
# variables never leak between concurrent cases or retries.
prepare_dynamic_config(
cfg,
config.get("dynamicVariables") or {},
assistant_id=str(run.assistant_id or ""),
)
except Exception as exc:
await self._fail_run_setup(run_id, str(exc))
return
config = dict(run.config or {})
evaluator_resource_id = str(
config.get("evaluatorModelResourceId") or ""
)
@@ -121,6 +134,7 @@ class TestRunOrchestrator:
run_id,
row.id,
cfg,
str(run.assistant_id or ""),
evaluator_resource,
config,
semaphore,
@@ -146,6 +160,7 @@ class TestRunOrchestrator:
run_id: str,
run_case_id: str,
cfg,
assistant_id: str,
evaluator_resource: RuntimeModelResource,
config: dict,
semaphore: asyncio.Semaphore,
@@ -176,10 +191,29 @@ class TestRunOrchestrator:
or 60
)
async with asyncio.timeout(timeout_seconds):
execution = await TextPipelineRunner().run(
attempt_cfg = prepare_dynamic_config(
cfg.model_copy(deep=True),
config.get("dynamicVariables") or {},
assistant_id=assistant_id,
)
async def save_turn_progress(turn: RawTurnResult) -> None:
await self._mark_turn_progress(run_case_id, turn)
async def save_startup_progress(
startup: TextStartupResult,
) -> None:
await self._mark_startup_progress(
run_case_id,
startup,
)
execution = await TextPipelineRunner().run(
attempt_cfg,
definition,
timeout_seconds=timeout_seconds,
on_turn_complete=save_turn_progress,
on_startup_complete=save_startup_progress,
)
evaluated = await EvaluationEngine(evaluator_resource).evaluate(
definition,
@@ -224,7 +258,12 @@ class TestRunOrchestrator:
return
status = "pass" if evaluated.passed else "fail"
await self._mark_evaluated(run_case_id, status, evaluated)
await self._mark_evaluated(
run_case_id,
status,
evaluated,
execution.startup,
)
if status == "fail" and str(
config.get("failureStrategy") or config.get("failure_strategy")
) == "stop_on_fail":
@@ -256,9 +295,76 @@ class TestRunOrchestrator:
row.started_at = row.started_at or datetime.now(UTC)
row.finished_at = None
row.execution_error = None
row.result = {
"turns": [],
"startup": {
"completed": False,
"greeting": "",
"assistantOutputs": [],
"autoConfirmedMessages": [],
},
"overallCriteria": [],
}
await session.commit()
async def _mark_evaluated(self, run_case_id: str, status: str, evaluated) -> None:
async def _mark_startup_progress(
self,
run_case_id: str,
startup: TextStartupResult,
) -> None:
async with SessionLocal() as session:
row = await session.get(TestRunCase, run_case_id)
if row is None or row.status != "running":
return
result = dict(row.result or {})
result["startup"] = {
"completed": True,
"greeting": startup.greeting,
"assistantOutputs": list(startup.assistant_outputs),
"autoConfirmedMessages": list(
startup.auto_confirmed_messages
),
}
row.result = result
await session.commit()
async def _mark_turn_progress(
self,
run_case_id: str,
turn: RawTurnResult,
) -> None:
"""Persist completed turns so polling clients can show live progress."""
async with SessionLocal() as session:
row = await session.get(TestRunCase, run_case_id)
if row is None or row.status != "running":
return
result = dict(row.result or {})
turns = list(result.get("turns") or [])
payload = {
"id": turn.id,
"index": turn.index,
"userInput": turn.user_input,
"assistantReply": turn.assistant_reply,
"toolCalls": [item.as_result() for item in turn.tool_calls],
"evaluations": [],
}
turns = [item for item in turns if item.get("id") != turn.id]
turns.append(payload)
turns.sort(key=lambda item: int(item.get("index") or 0))
row.result = {
"turns": turns,
"startup": dict(result.get("startup") or {}),
"overallCriteria": list(result.get("overallCriteria") or []),
}
await session.commit()
async def _mark_evaluated(
self,
run_case_id: str,
status: str,
evaluated,
startup: TextStartupResult,
) -> None:
async with SessionLocal() as session:
row = await session.get(TestRunCase, run_case_id)
if row is None:
@@ -269,6 +375,14 @@ class TestRunOrchestrator:
item.model_dump(mode="json", by_alias=True)
for item in evaluated.turns
],
"startup": {
"completed": True,
"greeting": startup.greeting,
"assistantOutputs": list(startup.assistant_outputs),
"autoConfirmedMessages": list(
startup.auto_confirmed_messages
),
},
"overallCriteria": [
item.model_dump(mode="json", by_alias=True)
for item in evaluated.overall_criteria