wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add dynamic variables and startup result handling in batch run schemas

Browse Source 
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types.
- Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages.
- Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs.
- Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases.
- Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
This commit is contained in:
Xin Wang
2026-08-12 16:55:25 +08:00
parent 79a239489a
commit 7560f9a525
12 changed files with 1464 additions and 1004 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/routes/test_runs.py
+12 -1
View File
@@ -17,6 +17,7 @@ from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException
from services.auth import require_admin
from services.config_resolver import resolve_runtime_config
from services.runtime_variables import DynamicVariableError, prepare_dynamic_config
from services.test_runs.orchestrator import test_run_orchestrator
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -25,6 +26,7 @@ from test_schemas import (
BatchRunCaseOut,
BatchRunCreate,
BatchRunSnapshotOut,
BatchStartupResult,
BatchTurnResult,
BatchEvaluationResult,
TestCaseDefinition,
@@ -65,6 +67,9 @@ async def _run_out(
BatchTurnResult.model_validate(item)
for item in result.get("turns") or []
],
startup=BatchStartupResult.model_validate(
result.get("startup") or {}
),
overall_criteria=[
BatchEvaluationResult.model_validate(item)
for item in result.get("overallCriteria")
@@ -118,7 +123,12 @@ async def create_test_run(
raise HTTPException(422, "评估模型不存在、已停用或不是 LLM 资源")
try:
runtime_config = await resolve_runtime_config(session, assistant.id)
except ValueError as exc:
prepare_dynamic_config(
runtime_config,
body.dynamic_variables,
assistant_id=assistant.id,
)
except (ValueError, DynamicVariableError) as exc:
raise HTTPException(422, str(exc)) from exc
rows = (
@@ -175,6 +185,7 @@ async def create_test_run(
"evaluatorModel": str(
(evaluator_resource.values or {}).get("modelId") or ""
),
"dynamicVariables": body.dynamic_variables,
**body.config.model_dump(mode="json", by_alias=True),
},
started_at=now,
backend/services/brains/prompt_brain.py
+4 -11
View File
@@ -681,17 +681,10 @@ class PromptBrain(BaseBrain):
else:
raise ValueError(f"未知系统工具: {kind}")
if runtime.tool_executor_factory is None:
return schema, handler
async def mock_system_tool(params: FunctionCallParams) -> None:
result = await self._tools.execute(
tool,
dict(params.arguments or {}),
)
await params.result_callback(result)
return schema, mock_system_tool
# System tools are platform-owned deterministic behavior. Keep their
# lifecycle and state semantics identical across production and tests;
# only external HTTP, MCP, and client tools use an injected executor.
return schema, handler
def _make_update_state_tool(self, tool, runtime: BrainRuntime):
"""更新已声明的动态变量(会话状态),并让模型继续当前回答。"""
backend/services/brains/workflow_brain.py
+3 -8
View File
@@ -951,14 +951,9 @@ class WorkflowBrain(BaseBrain):
else:
raise ValueError(f"系统工具 {tool.id} 缺少有效 kind")
runtime = self._require_runtime()
if runtime.tool_executor_factory is None:
return schema
async def mock_system_tool(args, _flow_manager):
return await self._tools.execute(tool, dict(args or {}))
return replace(schema, handler=mock_system_tool)
# Platform-owned system behavior is deterministic. Keep it real in all
# runtimes while an injected executor isolates only external tools.
return schema
def _workflow_update_state_tool(
self,
backend/services/test_runs/mock_tools.py
+4
View File
@@ -69,6 +69,9 @@ class MockToolExecutor(ToolExecutor):
class TextClientToolPort:
"""Automatically acknowledge built-in message stages in text tests."""
def __init__(self) -> None:
self.auto_confirmed_messages: list[dict[str, Any]] = []
async def call(
self,
function_name: str,
@@ -76,6 +79,7 @@ class TextClientToolPort:
**_kwargs: Any,
) -> dict[str, Any]:
if function_name == "show_message":
self.auto_confirmed_messages.append(dict(arguments))
return {
"status": "ok",
"data": {"action": "confirmed", "arguments": arguments},
backend/services/test_runs/orchestrator.py
+119 -5
View File
@@ -13,9 +13,14 @@ from db.models import ModelResource, TestCase, TestRun, TestRunCase
from db.session import SessionLocal
from models import RuntimeModelResource
from services.config_resolver import resolve_runtime_config
from services.runtime_variables import prepare_dynamic_config
from services.test_runs.errors import TestExecutionError
from services.test_runs.evaluator import EvaluationEngine
from services.test_runs.text_runner import TextPipelineRunner
from services.test_runs.text_runner import (
RawTurnResult,
TextPipelineRunner,
TextStartupResult,
)
from test_schemas import TestCaseDefinition
@@ -74,12 +79,20 @@ class TestRunOrchestrator:
run.status = "running"
run.started_at = run.started_at or datetime.now(UTC)
await session.commit()
config = dict(run.config or {})
try:
cfg = await resolve_runtime_config(session, str(run.assistant_id or ""))
# Validate the run-level values once. Each case/attempt gets
# a fresh prepared copy below so conversation-scoped system
# variables never leak between concurrent cases or retries.
prepare_dynamic_config(
cfg,
config.get("dynamicVariables") or {},
assistant_id=str(run.assistant_id or ""),
)
except Exception as exc:
await self._fail_run_setup(run_id, str(exc))
return
config = dict(run.config or {})
evaluator_resource_id = str(
config.get("evaluatorModelResourceId") or ""
)
@@ -121,6 +134,7 @@ class TestRunOrchestrator:
run_id,
row.id,
cfg,
str(run.assistant_id or ""),
evaluator_resource,
config,
semaphore,
@@ -146,6 +160,7 @@ class TestRunOrchestrator:
run_id: str,
run_case_id: str,
cfg,
assistant_id: str,
evaluator_resource: RuntimeModelResource,
config: dict,
semaphore: asyncio.Semaphore,
@@ -176,10 +191,29 @@ class TestRunOrchestrator:
or 60
)
async with asyncio.timeout(timeout_seconds):
execution = await TextPipelineRunner().run(
attempt_cfg = prepare_dynamic_config(
cfg.model_copy(deep=True),
config.get("dynamicVariables") or {},
assistant_id=assistant_id,
)
async def save_turn_progress(turn: RawTurnResult) -> None:
await self._mark_turn_progress(run_case_id, turn)
async def save_startup_progress(
startup: TextStartupResult,
) -> None:
await self._mark_startup_progress(
run_case_id,
startup,
)
execution = await TextPipelineRunner().run(
attempt_cfg,
definition,
timeout_seconds=timeout_seconds,
on_turn_complete=save_turn_progress,
on_startup_complete=save_startup_progress,
)
evaluated = await EvaluationEngine(evaluator_resource).evaluate(
definition,
@@ -224,7 +258,12 @@ class TestRunOrchestrator:
return
status = "pass" if evaluated.passed else "fail"
await self._mark_evaluated(run_case_id, status, evaluated)
await self._mark_evaluated(
run_case_id,
status,
evaluated,
execution.startup,
)
if status == "fail" and str(
config.get("failureStrategy") or config.get("failure_strategy")
) == "stop_on_fail":
@@ -256,9 +295,76 @@ class TestRunOrchestrator:
row.started_at = row.started_at or datetime.now(UTC)
row.finished_at = None
row.execution_error = None
row.result = {
"turns": [],
"startup": {
"completed": False,
"greeting": "",
"assistantOutputs": [],
"autoConfirmedMessages": [],
},
"overallCriteria": [],
}
await session.commit()
async def _mark_evaluated(self, run_case_id: str, status: str, evaluated) -> None:
async def _mark_startup_progress(
self,
run_case_id: str,
startup: TextStartupResult,
) -> None:
async with SessionLocal() as session:
row = await session.get(TestRunCase, run_case_id)
if row is None or row.status != "running":
return
result = dict(row.result or {})
result["startup"] = {
"completed": True,
"greeting": startup.greeting,
"assistantOutputs": list(startup.assistant_outputs),
"autoConfirmedMessages": list(
startup.auto_confirmed_messages
),
}
row.result = result
await session.commit()
async def _mark_turn_progress(
self,
run_case_id: str,
turn: RawTurnResult,
) -> None:
"""Persist completed turns so polling clients can show live progress."""
async with SessionLocal() as session:
row = await session.get(TestRunCase, run_case_id)
if row is None or row.status != "running":
return
result = dict(row.result or {})
turns = list(result.get("turns") or [])
payload = {
"id": turn.id,
"index": turn.index,
"userInput": turn.user_input,
"assistantReply": turn.assistant_reply,
"toolCalls": [item.as_result() for item in turn.tool_calls],
"evaluations": [],
}
turns = [item for item in turns if item.get("id") != turn.id]
turns.append(payload)
turns.sort(key=lambda item: int(item.get("index") or 0))
row.result = {
"turns": turns,
"startup": dict(result.get("startup") or {}),
"overallCriteria": list(result.get("overallCriteria") or []),
}
await session.commit()
async def _mark_evaluated(
self,
run_case_id: str,
status: str,
evaluated,
startup: TextStartupResult,
) -> None:
async with SessionLocal() as session:
row = await session.get(TestRunCase, run_case_id)
if row is None:
@@ -269,6 +375,14 @@ class TestRunOrchestrator:
item.model_dump(mode="json", by_alias=True)
for item in evaluated.turns
],
"startup": {
"completed": True,
"greeting": startup.greeting,
"assistantOutputs": list(startup.assistant_outputs),
"autoConfirmedMessages": list(
startup.auto_confirmed_messages
),
},
"overallCriteria": [
item.model_dump(mode="json", by_alias=True)
for item in evaluated.overall_criteria
backend/services/test_runs/text_runner.py
+122 -11
View File
@@ -16,6 +16,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
CancelFrame,
DataFrame,
EndFrame,
ErrorFrame,
FunctionCallInProgressFrame,
@@ -103,10 +104,31 @@ class RawTurnResult:
tool_calls: list[RawToolCall]
@dataclass
class TextStartupResult:
"""Headless startup choices applied before the scripted user turns."""
greeting: str = ""
assistant_outputs: list[str] = field(default_factory=list)
auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list)
@dataclass
class TextCaseResult:
turns: list[RawTurnResult]
transcript: list[dict[str, str]]
startup: TextStartupResult = field(default_factory=TextStartupResult)
TurnCompleteCallback = Callable[[RawTurnResult], Awaitable[None]]
StartupCompleteCallback = Callable[[TextStartupResult], Awaitable[None]]
@dataclass
class _StartupBoundaryFrame(DataFrame):
"""Ordered marker queued behind every frame emitted by startup."""
reached: asyncio.Event
@dataclass
@@ -139,6 +161,26 @@ class _TextCaptureProcessor(FrameProcessor):
self.error: TestExecutionError | None = None
self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None
def fail(self, frame: ErrorFrame) -> None:
"""Release every wait even when Pipecat reports the error upstream."""
message = frame.error or "Pipeline 执行失败"
unmocked_tool = "UNMOCKED_TOOL_CALL" in message
self.error = TestExecutionError(
code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR",
message=message,
stage=(
"tool"
if unmocked_tool
else "model"
if frame.processor
else "pipeline"
),
retryable=not unmocked_tool,
)
self.window.pending_tools.clear()
self.window.touch()
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -196,12 +238,9 @@ class _TextCaptureProcessor(FrameProcessor):
self.window.pending_tools.discard(call.id)
self.window.touch()
elif isinstance(frame, ErrorFrame):
self.error = TestExecutionError(
code="PIPELINE_ERROR",
message=frame.error or "Pipeline 执行失败",
stage="model" if frame.processor else "pipeline",
retryable=True,
)
self.fail(frame)
elif isinstance(frame, _StartupBoundaryFrame):
frame.reached.set()
self.window.touch()
elif isinstance(frame, (EndFrame, CancelFrame)):
self.window.touch()
@@ -281,6 +320,41 @@ async def _wait_for_quiescence(
await asyncio.sleep(0.05)
async def _wait_for_startup(
worker: PipelineWorker,
capture: _TextCaptureProcessor,
runner_task: asyncio.Task[None],
activity_marker: int,
) -> None:
"""Wait until startup-queued frames and any entry LLM reply are complete."""
boundary = asyncio.Event()
await worker.queue_frame(_StartupBoundaryFrame(reached=boundary))
boundary_waiter = asyncio.create_task(boundary.wait())
done, _pending = await asyncio.wait(
{boundary_waiter, runner_task},
timeout=WORKER_START_TIMEOUT_SECONDS,
return_when=asyncio.FIRST_COMPLETED,
)
if not done:
boundary_waiter.cancel()
raise TestExecutionError(
code="STARTUP_DRAIN_TIMEOUT",
message="开场阶段未能进入稳定状态",
stage="pipeline",
retryable=True,
)
if boundary_waiter not in done:
boundary_waiter.cancel()
await asyncio.gather(boundary_waiter, return_exceptions=True)
await _wait_for_quiescence(
capture,
runner_task,
activity_marker,
require_activity=False,
)
class TextPipelineRunner:
"""Execute all fixed input turns in one isolated, transportless pipeline."""
@@ -290,10 +364,17 @@ class TextPipelineRunner:
definition: TestCaseDefinition,
*,
timeout_seconds: int,
on_turn_complete: TurnCompleteCallback | None = None,
on_startup_complete: StartupCompleteCallback | None = None,
) -> TextCaseResult:
try:
async with asyncio.timeout(timeout_seconds):
return await self._run(cfg, definition)
return await self._run(
cfg,
definition,
on_turn_complete=on_turn_complete,
on_startup_complete=on_startup_complete,
)
except TimeoutError as exc:
raise TestExecutionError(
code="PIPELINE_TIMEOUT",
@@ -306,6 +387,9 @@ class TextPipelineRunner:
self,
cfg: AssistantConfig,
definition: TestCaseDefinition,
*,
on_turn_complete: TurnCompleteCallback | None,
on_startup_complete: StartupCompleteCallback | None,
) -> TextCaseResult:
if cfg.runtimeMode != "pipeline":
raise TestExecutionError(
@@ -396,7 +480,12 @@ class TextPipelineRunner:
async def on_pipeline_started(_worker, _frame):
worker_started.set()
@worker.event_handler("on_pipeline_error")
async def on_pipeline_error(_worker, frame):
capture.fail(frame)
executor_holder: dict[str, MockToolExecutor] = {}
text_client_tools = TextClientToolPort()
def create_test_executor(store) -> MockToolExecutor:
executor = MockToolExecutor(store)
@@ -520,7 +609,7 @@ class TextPipelineRunner:
set_tools=set_tools,
call_end=call_end,
session_id=f"test_{uuid4().hex}",
client_tools=TextClientToolPort(),
client_tools=text_client_tools,
worker=worker,
context_aggregator=aggregators,
transport=None,
@@ -536,6 +625,13 @@ class TextPipelineRunner:
)
executor = executor_holder["executor"]
# A transportless text run cannot synthesize audio, but the configured
# greeting has already completed before scripted inputs begin. Mirror
# production's provider-safe context fact so the model does not repeat it.
greeting = str(await brain.greeting(cfg) or "").strip()
if greeting and brain.spec.owns_context:
brain.prepare_greeting_context(greeting, context)
@assistant_aggregator.event_handler("on_assistant_turn_started")
async def on_assistant_turn_started(_aggregator):
capture.window.active_assistant += 1
@@ -591,12 +687,21 @@ class TextPipelineRunner:
marker = capture.window.activity_count
await brain.on_connected(greeting_pending=False)
await brain.on_client_ready()
await _wait_for_quiescence(
await _wait_for_startup(
worker,
capture,
runner_task,
marker,
require_activity=False,
)
startup = TextStartupResult(
greeting=greeting,
assistant_outputs=list(capture.outputs),
auto_confirmed_messages=list(
text_client_tools.auto_confirmed_messages
),
)
if on_startup_complete is not None:
await on_startup_complete(startup)
# Startup messages initialize the session but are not a scripted
# user turn result in the fixed-input editor contract.
capture.outputs.clear()
@@ -620,6 +725,8 @@ class TextPipelineRunner:
index,
)
results.append(result)
if on_turn_complete is not None:
await on_turn_complete(result)
transcript = [
{
@@ -630,7 +737,11 @@ class TextPipelineRunner:
if message.get("role") in {"user", "assistant"}
and str(message.get("content") or "").strip()
]
return TextCaseResult(turns=results, transcript=transcript)
return TextCaseResult(
turns=results,
transcript=transcript,
startup=startup,
)
except TestExecutionError:
raise
except asyncio.CancelledError:
backend/test_schemas.py
+12
View File
@@ -316,6 +316,10 @@ class BatchRunCreate(CamelModel):
assistant_id: str = Field(min_length=1, max_length=40)
evaluator_model_resource_id: str = Field(min_length=1, max_length=40)
case_ids: list[str] = Field(min_length=1, max_length=1_000)
dynamic_variables: dict[str, str | int | float | bool] = Field(
default_factory=dict,
max_length=50,
)
config: BatchRunConfig = Field(default_factory=BatchRunConfig)
title: str | None = Field(default=None, max_length=256)
@@ -363,11 +367,19 @@ class BatchTurnResult(CamelModel):
evaluations: list[BatchEvaluationResult] = Field(default_factory=list)
class BatchStartupResult(CamelModel):
completed: bool = False
greeting: str = ""
assistant_outputs: list[str] = Field(default_factory=list)
auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list)
class BatchRunCaseOut(CamelModel):
id: str
name: str
status: CaseStatus
turns: list[BatchTurnResult] = Field(default_factory=list)
startup: BatchStartupResult = Field(default_factory=BatchStartupResult)
overall_criteria: list[BatchEvaluationResult] = Field(default_factory=list)
attempt_count: int
max_attempts: int
backend/tests/test_debug_auto_test.py
+264
View File
@@ -0,0 +1,264 @@
import asyncio
import unittest
from models import AssistantConfig, RuntimeTool
from pipecat.frames.frames import ErrorFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from services.brains import BrainRuntime, build_brain
from services.brains.base import GREETING_CONTEXT_MARKER
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from services.test_runs.text_runner import (
_TextCaptureProcessor,
_wait_for_startup,
_wait_for_quiescence,
)
from test_schemas import BatchRunCreate
class FakeLLM:
def __init__(self):
self.functions = {}
def register_function(self, name, handler, **_options):
self.functions[name] = handler
class FakeCallEnd:
def __init__(self):
self.ending = False
self.reason = ""
@property
def accepts_input(self):
return not self.ending
def begin(self, reason, *, interruptible=False):
self.ending = True
self.reason = reason
def cancel_pending_end(self):
return False
def begin_response(self):
return None
def arm_after_speech(self):
return None
def track_speech(self):
return None
async def arm_after_tracked_speech(self):
return None
async def finish_after_current_speech(self, *, has_text):
return None
async def finish(self):
return None
class FakeFunctionParams:
def __init__(self, arguments=None):
self.arguments = arguments or {}
self.result = None
self.properties = None
async def result_callback(self, result, properties=None):
self.result = result
self.properties = properties
def system_runtime_tool(kind: str) -> RuntimeTool:
return RuntimeTool(
id=f"tool_{kind}",
name=kind,
function_name=kind,
type="system",
definition={"config": {"kind": kind}},
)
class DebugAutoTestContractTests(unittest.TestCase):
def test_batch_run_accepts_camel_case_dynamic_variables(self):
request = BatchRunCreate.model_validate(
{
"assistantId": "asst_1",
"evaluatorModelResourceId": "model_1",
"caseIds": ["case_1"],
"dynamicVariables": {"customer_name": "测试用户", "count": 2},
}
)
self.assertEqual(
request.dynamic_variables,
{"customer_name": "测试用户", "count": 2},
)
def test_each_test_case_gets_fresh_conversation_variables(self):
config = AssistantConfig(
type="prompt",
prompt="你好，{{ customer_name }}",
dynamic_variable_definitions={
"customer_name": {
"type": "string",
"required": True,
"default": None,
}
},
)
first = prepare_dynamic_config(
config,
{"customer_name": "测试用户"},
assistant_id="asst_1",
)
second = prepare_dynamic_config(
config,
{"customer_name": "测试用户"},
assistant_id="asst_1",
)
self.assertNotEqual(first.conversation_id, second.conversation_id)
self.assertEqual(
DynamicVariableStore.from_config(first).render(first.prompt),
"你好，测试用户",
)
def test_text_client_port_records_automatic_message_confirmation(self):
port = TextClientToolPort()
result = asyncio.run(
port.call(
"show_message",
{"title": "重要提示", "message": "请确认"},
)
)
self.assertEqual(result["data"]["action"], "confirmed")
self.assertEqual(
port.auto_confirmed_messages,
[{"title": "重要提示", "message": "请确认"}],
)
class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
cfg = AssistantConfig(
type="prompt",
tools=[system_runtime_tool("end_conversation")],
)
brain = build_brain(cfg)
llm = FakeLLM()
call_end = FakeCallEnd()
executor_holder = {}
def create_executor(store):
executor = MockToolExecutor(store)
executor_holder["executor"] = executor
return executor
await brain.setup(
cfg,
BrainRuntime(
context=LLMContext(messages=[]),
llm=llm,
queue_frame=lambda _frame: asyncio.sleep(0),
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=call_end,
tool_executor_factory=create_executor,
),
)
params = FakeFunctionParams({"reason": "测试完成"})
await llm.functions["end_conversation"](params)
self.assertTrue(call_end.ending)
self.assertEqual(call_end.reason, "测试完成")
self.assertEqual(params.result["action"], "ending_call")
self.assertEqual(executor_holder["executor"].unmocked_calls, [])
async def test_startup_waits_for_queued_entry_reply(self):
capture = _TextCaptureProcessor(LLMContext(messages=[]))
class FakeWorker:
async def queue_frame(self, frame):
async def process_startup_queue():
capture.window.active_llm += 1
capture.window.touch()
await asyncio.sleep(0.01)
capture.outputs.append("开场后的自动首句")
capture.window.active_llm -= 1
capture.window.touch()
frame.reached.set()
asyncio.create_task(process_startup_queue())
runner_task = asyncio.create_task(asyncio.sleep(60))
try:
await asyncio.wait_for(
_wait_for_startup(
FakeWorker(),
capture,
runner_task,
capture.window.activity_count,
),
timeout=1,
)
self.assertEqual(capture.outputs, ["开场后的自动首句"])
finally:
runner_task.cancel()
await asyncio.gather(runner_task, return_exceptions=True)
async def test_text_startup_can_record_greeting_as_already_played(self):
cfg = AssistantConfig(
type="prompt",
greeting="您好，这是一段开场白。",
prompt="不要重复开场白。",
)
brain = build_brain(cfg)
context = LLMContext(
messages=[{"role": "system", "content": cfg.prompt}]
)
greeting = await brain.greeting(cfg)
brain.prepare_greeting_context(greeting, context)
self.assertIn(
GREETING_CONTEXT_MARKER,
context.get_messages()[1]["content"],
)
self.assertIn(cfg.greeting, context.get_messages()[1]["content"])
async def test_upstream_pipeline_error_releases_pending_tool_wait(self):
capture = _TextCaptureProcessor(LLMContext(messages=[]))
capture.window.pending_tools.add("call_1")
runner_task = asyncio.create_task(asyncio.sleep(60))
try:
capture.fail(
ErrorFrame(
error=(
"UNMOCKED_TOOL_CALL: 工具 external_lookup "
"未配置 Mock 返回值"
)
)
)
with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"):
await asyncio.wait_for(
_wait_for_quiescence(capture, runner_task, 0),
timeout=0.1,
)
self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL")
self.assertEqual(capture.error.stage, "tool")
self.assertFalse(capture.error.retryable)
self.assertEqual(capture.window.pending_tools, set())
finally:
runner_task.cancel()
await asyncio.gather(runner_task, return_exceptions=True)
if __name__ == "__main__":
unittest.main()
frontend/src/components/assistant-editor/debug-auto-test.tsx
+896 -966
View File
File diff suppressed because it is too large Load Diff
frontend/src/components/assistant-editor/debug-preview.tsx
+15 -2
View File
@@ -183,8 +183,6 @@ export function DebugDrawer({
}) {
const preview = useVoicePreview(assistantId, onNodeActive);
const camera = useCameraPreview();
const autoTest = useAutoTestRunner();
const stopAutoTest = autoTest.stop;
const [debugMode, setDebugMode] = useState<DebugMode>("manual");
const [showTranscript, setShowTranscript] = useState(false);
const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
@@ -220,6 +218,14 @@ export function DebugDrawer({
}
resolvedDynamicVariables[name] = value;
}
const autoTest = useAutoTestRunner({
assistantId,
enabled: debugMode === "auto",
hasUnsavedChanges,
dynamicVariables: resolvedDynamicVariables,
dynamicVariablesError,
});
const stopAutoTest = autoTest.stop;
const selectCamera = useCallback(
async (deviceId: string) => {
@@ -369,6 +375,9 @@ function AutoTestPanel({
start,
stop,
rerun,
suites,
cases,
catalogLoading,
} = autoTest;
return (
@@ -379,6 +388,10 @@ function AutoTestPanel({
<div className="shrink-0 border-t border-hairline bg-card p-3">
<AutoTestRunBar
state={state}
testCase={selectedCase}
suites={suites}
cases={cases}
loading={catalogLoading}
onSelectCase={selectCase}
onClearCase={clearCase}
onStart={start}
frontend/src/data/batch-run.ts
+12
View File
@@ -59,11 +59,23 @@ export type BatchTurnResult = {
evaluations: BatchEvaluationResult[];
};
export type BatchStartupResult = {
completed: boolean;
greeting: string;
assistantOutputs: string[];
autoConfirmedMessages: Array<{
title?: string;
message?: string;
actions?: Array<{ id?: string; label?: string }>;
}>;
};
export type BatchRunCase = {
id: string;
name: string;
status: BatchCaseStatus;
turns: BatchTurnResult[];
startup?: BatchStartupResult;
overallCriteria: BatchEvaluationResult[];
attemptCount: number;
maxAttempts: number;
frontend/src/lib/api.ts
+1
View File
@@ -851,6 +851,7 @@ export const batchRunsApi = {
assistantId: string;
evaluatorModelResourceId: string;
caseIds: string[];
dynamicVariables?: Record<string, string | number | boolean>;
title?: string;
config: {
concurrency: number;