diff --git a/backend/routes/test_runs.py b/backend/routes/test_runs.py index fec5c7b..80a3eea 100644 --- a/backend/routes/test_runs.py +++ b/backend/routes/test_runs.py @@ -17,6 +17,7 @@ from db.session import get_session from fastapi import APIRouter, Depends, HTTPException from services.auth import require_admin from services.config_resolver import resolve_runtime_config +from services.runtime_variables import DynamicVariableError, prepare_dynamic_config from services.test_runs.orchestrator import test_run_orchestrator from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -25,6 +26,7 @@ from test_schemas import ( BatchRunCaseOut, BatchRunCreate, BatchRunSnapshotOut, + BatchStartupResult, BatchTurnResult, BatchEvaluationResult, TestCaseDefinition, @@ -65,6 +67,9 @@ async def _run_out( BatchTurnResult.model_validate(item) for item in result.get("turns") or [] ], + startup=BatchStartupResult.model_validate( + result.get("startup") or {} + ), overall_criteria=[ BatchEvaluationResult.model_validate(item) for item in result.get("overallCriteria") @@ -118,7 +123,12 @@ async def create_test_run( raise HTTPException(422, "评估模型不存在、已停用或不是 LLM 资源") try: runtime_config = await resolve_runtime_config(session, assistant.id) - except ValueError as exc: + prepare_dynamic_config( + runtime_config, + body.dynamic_variables, + assistant_id=assistant.id, + ) + except (ValueError, DynamicVariableError) as exc: raise HTTPException(422, str(exc)) from exc rows = ( @@ -175,6 +185,7 @@ async def create_test_run( "evaluatorModel": str( (evaluator_resource.values or {}).get("modelId") or "" ), + "dynamicVariables": body.dynamic_variables, **body.config.model_dump(mode="json", by_alias=True), }, started_at=now, diff --git a/backend/services/brains/prompt_brain.py b/backend/services/brains/prompt_brain.py index b21c0f4..659fbe0 100644 --- a/backend/services/brains/prompt_brain.py +++ b/backend/services/brains/prompt_brain.py @@ -681,17 +681,10 @@ class PromptBrain(BaseBrain): else: raise ValueError(f"未知系统工具: {kind}") - if runtime.tool_executor_factory is None: - return schema, handler - - async def mock_system_tool(params: FunctionCallParams) -> None: - result = await self._tools.execute( - tool, - dict(params.arguments or {}), - ) - await params.result_callback(result) - - return schema, mock_system_tool + # System tools are platform-owned deterministic behavior. Keep their + # lifecycle and state semantics identical across production and tests; + # only external HTTP, MCP, and client tools use an injected executor. + return schema, handler def _make_update_state_tool(self, tool, runtime: BrainRuntime): """更新已声明的动态变量(会话状态),并让模型继续当前回答。""" diff --git a/backend/services/brains/workflow_brain.py b/backend/services/brains/workflow_brain.py index 517e939..1de265c 100644 --- a/backend/services/brains/workflow_brain.py +++ b/backend/services/brains/workflow_brain.py @@ -951,14 +951,9 @@ class WorkflowBrain(BaseBrain): else: raise ValueError(f"系统工具 {tool.id} 缺少有效 kind") - runtime = self._require_runtime() - if runtime.tool_executor_factory is None: - return schema - - async def mock_system_tool(args, _flow_manager): - return await self._tools.execute(tool, dict(args or {})) - - return replace(schema, handler=mock_system_tool) + # Platform-owned system behavior is deterministic. Keep it real in all + # runtimes while an injected executor isolates only external tools. + return schema def _workflow_update_state_tool( self, diff --git a/backend/services/test_runs/mock_tools.py b/backend/services/test_runs/mock_tools.py index 559f630..7b1d330 100644 --- a/backend/services/test_runs/mock_tools.py +++ b/backend/services/test_runs/mock_tools.py @@ -69,6 +69,9 @@ class MockToolExecutor(ToolExecutor): class TextClientToolPort: """Automatically acknowledge built-in message stages in text tests.""" + def __init__(self) -> None: + self.auto_confirmed_messages: list[dict[str, Any]] = [] + async def call( self, function_name: str, @@ -76,6 +79,7 @@ class TextClientToolPort: **_kwargs: Any, ) -> dict[str, Any]: if function_name == "show_message": + self.auto_confirmed_messages.append(dict(arguments)) return { "status": "ok", "data": {"action": "confirmed", "arguments": arguments}, diff --git a/backend/services/test_runs/orchestrator.py b/backend/services/test_runs/orchestrator.py index 7da4a0f..292b466 100644 --- a/backend/services/test_runs/orchestrator.py +++ b/backend/services/test_runs/orchestrator.py @@ -13,9 +13,14 @@ from db.models import ModelResource, TestCase, TestRun, TestRunCase from db.session import SessionLocal from models import RuntimeModelResource from services.config_resolver import resolve_runtime_config +from services.runtime_variables import prepare_dynamic_config from services.test_runs.errors import TestExecutionError from services.test_runs.evaluator import EvaluationEngine -from services.test_runs.text_runner import TextPipelineRunner +from services.test_runs.text_runner import ( + RawTurnResult, + TextPipelineRunner, + TextStartupResult, +) from test_schemas import TestCaseDefinition @@ -74,12 +79,20 @@ class TestRunOrchestrator: run.status = "running" run.started_at = run.started_at or datetime.now(UTC) await session.commit() + config = dict(run.config or {}) try: cfg = await resolve_runtime_config(session, str(run.assistant_id or "")) + # Validate the run-level values once. Each case/attempt gets + # a fresh prepared copy below so conversation-scoped system + # variables never leak between concurrent cases or retries. + prepare_dynamic_config( + cfg, + config.get("dynamicVariables") or {}, + assistant_id=str(run.assistant_id or ""), + ) except Exception as exc: await self._fail_run_setup(run_id, str(exc)) return - config = dict(run.config or {}) evaluator_resource_id = str( config.get("evaluatorModelResourceId") or "" ) @@ -121,6 +134,7 @@ class TestRunOrchestrator: run_id, row.id, cfg, + str(run.assistant_id or ""), evaluator_resource, config, semaphore, @@ -146,6 +160,7 @@ class TestRunOrchestrator: run_id: str, run_case_id: str, cfg, + assistant_id: str, evaluator_resource: RuntimeModelResource, config: dict, semaphore: asyncio.Semaphore, @@ -176,10 +191,29 @@ class TestRunOrchestrator: or 60 ) async with asyncio.timeout(timeout_seconds): - execution = await TextPipelineRunner().run( + attempt_cfg = prepare_dynamic_config( cfg.model_copy(deep=True), + config.get("dynamicVariables") or {}, + assistant_id=assistant_id, + ) + + async def save_turn_progress(turn: RawTurnResult) -> None: + await self._mark_turn_progress(run_case_id, turn) + + async def save_startup_progress( + startup: TextStartupResult, + ) -> None: + await self._mark_startup_progress( + run_case_id, + startup, + ) + + execution = await TextPipelineRunner().run( + attempt_cfg, definition, timeout_seconds=timeout_seconds, + on_turn_complete=save_turn_progress, + on_startup_complete=save_startup_progress, ) evaluated = await EvaluationEngine(evaluator_resource).evaluate( definition, @@ -224,7 +258,12 @@ class TestRunOrchestrator: return status = "pass" if evaluated.passed else "fail" - await self._mark_evaluated(run_case_id, status, evaluated) + await self._mark_evaluated( + run_case_id, + status, + evaluated, + execution.startup, + ) if status == "fail" and str( config.get("failureStrategy") or config.get("failure_strategy") ) == "stop_on_fail": @@ -256,9 +295,76 @@ class TestRunOrchestrator: row.started_at = row.started_at or datetime.now(UTC) row.finished_at = None row.execution_error = None + row.result = { + "turns": [], + "startup": { + "completed": False, + "greeting": "", + "assistantOutputs": [], + "autoConfirmedMessages": [], + }, + "overallCriteria": [], + } await session.commit() - async def _mark_evaluated(self, run_case_id: str, status: str, evaluated) -> None: + async def _mark_startup_progress( + self, + run_case_id: str, + startup: TextStartupResult, + ) -> None: + async with SessionLocal() as session: + row = await session.get(TestRunCase, run_case_id) + if row is None or row.status != "running": + return + result = dict(row.result or {}) + result["startup"] = { + "completed": True, + "greeting": startup.greeting, + "assistantOutputs": list(startup.assistant_outputs), + "autoConfirmedMessages": list( + startup.auto_confirmed_messages + ), + } + row.result = result + await session.commit() + + async def _mark_turn_progress( + self, + run_case_id: str, + turn: RawTurnResult, + ) -> None: + """Persist completed turns so polling clients can show live progress.""" + async with SessionLocal() as session: + row = await session.get(TestRunCase, run_case_id) + if row is None or row.status != "running": + return + result = dict(row.result or {}) + turns = list(result.get("turns") or []) + payload = { + "id": turn.id, + "index": turn.index, + "userInput": turn.user_input, + "assistantReply": turn.assistant_reply, + "toolCalls": [item.as_result() for item in turn.tool_calls], + "evaluations": [], + } + turns = [item for item in turns if item.get("id") != turn.id] + turns.append(payload) + turns.sort(key=lambda item: int(item.get("index") or 0)) + row.result = { + "turns": turns, + "startup": dict(result.get("startup") or {}), + "overallCriteria": list(result.get("overallCriteria") or []), + } + await session.commit() + + async def _mark_evaluated( + self, + run_case_id: str, + status: str, + evaluated, + startup: TextStartupResult, + ) -> None: async with SessionLocal() as session: row = await session.get(TestRunCase, run_case_id) if row is None: @@ -269,6 +375,14 @@ class TestRunOrchestrator: item.model_dump(mode="json", by_alias=True) for item in evaluated.turns ], + "startup": { + "completed": True, + "greeting": startup.greeting, + "assistantOutputs": list(startup.assistant_outputs), + "autoConfirmedMessages": list( + startup.auto_confirmed_messages + ), + }, "overallCriteria": [ item.model_dump(mode="json", by_alias=True) for item in evaluated.overall_criteria diff --git a/backend/services/test_runs/text_runner.py b/backend/services/test_runs/text_runner.py index b70b320..d47ed0f 100644 --- a/backend/services/test_runs/text_runner.py +++ b/backend/services/test_runs/text_runner.py @@ -16,6 +16,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, + DataFrame, EndFrame, ErrorFrame, FunctionCallInProgressFrame, @@ -103,10 +104,31 @@ class RawTurnResult: tool_calls: list[RawToolCall] +@dataclass +class TextStartupResult: + """Headless startup choices applied before the scripted user turns.""" + + greeting: str = "" + assistant_outputs: list[str] = field(default_factory=list) + auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list) + + @dataclass class TextCaseResult: turns: list[RawTurnResult] transcript: list[dict[str, str]] + startup: TextStartupResult = field(default_factory=TextStartupResult) + + +TurnCompleteCallback = Callable[[RawTurnResult], Awaitable[None]] +StartupCompleteCallback = Callable[[TextStartupResult], Awaitable[None]] + + +@dataclass +class _StartupBoundaryFrame(DataFrame): + """Ordered marker queued behind every frame emitted by startup.""" + + reached: asyncio.Event @dataclass @@ -139,6 +161,26 @@ class _TextCaptureProcessor(FrameProcessor): self.error: TestExecutionError | None = None self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None + def fail(self, frame: ErrorFrame) -> None: + """Release every wait even when Pipecat reports the error upstream.""" + + message = frame.error or "Pipeline 执行失败" + unmocked_tool = "UNMOCKED_TOOL_CALL" in message + self.error = TestExecutionError( + code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR", + message=message, + stage=( + "tool" + if unmocked_tool + else "model" + if frame.processor + else "pipeline" + ), + retryable=not unmocked_tool, + ) + self.window.pending_tools.clear() + self.window.touch() + async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -196,12 +238,9 @@ class _TextCaptureProcessor(FrameProcessor): self.window.pending_tools.discard(call.id) self.window.touch() elif isinstance(frame, ErrorFrame): - self.error = TestExecutionError( - code="PIPELINE_ERROR", - message=frame.error or "Pipeline 执行失败", - stage="model" if frame.processor else "pipeline", - retryable=True, - ) + self.fail(frame) + elif isinstance(frame, _StartupBoundaryFrame): + frame.reached.set() self.window.touch() elif isinstance(frame, (EndFrame, CancelFrame)): self.window.touch() @@ -281,6 +320,41 @@ async def _wait_for_quiescence( await asyncio.sleep(0.05) +async def _wait_for_startup( + worker: PipelineWorker, + capture: _TextCaptureProcessor, + runner_task: asyncio.Task[None], + activity_marker: int, +) -> None: + """Wait until startup-queued frames and any entry LLM reply are complete.""" + + boundary = asyncio.Event() + await worker.queue_frame(_StartupBoundaryFrame(reached=boundary)) + boundary_waiter = asyncio.create_task(boundary.wait()) + done, _pending = await asyncio.wait( + {boundary_waiter, runner_task}, + timeout=WORKER_START_TIMEOUT_SECONDS, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + boundary_waiter.cancel() + raise TestExecutionError( + code="STARTUP_DRAIN_TIMEOUT", + message="开场阶段未能进入稳定状态", + stage="pipeline", + retryable=True, + ) + if boundary_waiter not in done: + boundary_waiter.cancel() + await asyncio.gather(boundary_waiter, return_exceptions=True) + await _wait_for_quiescence( + capture, + runner_task, + activity_marker, + require_activity=False, + ) + + class TextPipelineRunner: """Execute all fixed input turns in one isolated, transportless pipeline.""" @@ -290,10 +364,17 @@ class TextPipelineRunner: definition: TestCaseDefinition, *, timeout_seconds: int, + on_turn_complete: TurnCompleteCallback | None = None, + on_startup_complete: StartupCompleteCallback | None = None, ) -> TextCaseResult: try: async with asyncio.timeout(timeout_seconds): - return await self._run(cfg, definition) + return await self._run( + cfg, + definition, + on_turn_complete=on_turn_complete, + on_startup_complete=on_startup_complete, + ) except TimeoutError as exc: raise TestExecutionError( code="PIPELINE_TIMEOUT", @@ -306,6 +387,9 @@ class TextPipelineRunner: self, cfg: AssistantConfig, definition: TestCaseDefinition, + *, + on_turn_complete: TurnCompleteCallback | None, + on_startup_complete: StartupCompleteCallback | None, ) -> TextCaseResult: if cfg.runtimeMode != "pipeline": raise TestExecutionError( @@ -396,7 +480,12 @@ class TextPipelineRunner: async def on_pipeline_started(_worker, _frame): worker_started.set() + @worker.event_handler("on_pipeline_error") + async def on_pipeline_error(_worker, frame): + capture.fail(frame) + executor_holder: dict[str, MockToolExecutor] = {} + text_client_tools = TextClientToolPort() def create_test_executor(store) -> MockToolExecutor: executor = MockToolExecutor(store) @@ -520,7 +609,7 @@ class TextPipelineRunner: set_tools=set_tools, call_end=call_end, session_id=f"test_{uuid4().hex}", - client_tools=TextClientToolPort(), + client_tools=text_client_tools, worker=worker, context_aggregator=aggregators, transport=None, @@ -536,6 +625,13 @@ class TextPipelineRunner: ) executor = executor_holder["executor"] + # A transportless text run cannot synthesize audio, but the configured + # greeting has already completed before scripted inputs begin. Mirror + # production's provider-safe context fact so the model does not repeat it. + greeting = str(await brain.greeting(cfg) or "").strip() + if greeting and brain.spec.owns_context: + brain.prepare_greeting_context(greeting, context) + @assistant_aggregator.event_handler("on_assistant_turn_started") async def on_assistant_turn_started(_aggregator): capture.window.active_assistant += 1 @@ -591,12 +687,21 @@ class TextPipelineRunner: marker = capture.window.activity_count await brain.on_connected(greeting_pending=False) await brain.on_client_ready() - await _wait_for_quiescence( + await _wait_for_startup( + worker, capture, runner_task, marker, - require_activity=False, ) + startup = TextStartupResult( + greeting=greeting, + assistant_outputs=list(capture.outputs), + auto_confirmed_messages=list( + text_client_tools.auto_confirmed_messages + ), + ) + if on_startup_complete is not None: + await on_startup_complete(startup) # Startup messages initialize the session but are not a scripted # user turn result in the fixed-input editor contract. capture.outputs.clear() @@ -620,6 +725,8 @@ class TextPipelineRunner: index, ) results.append(result) + if on_turn_complete is not None: + await on_turn_complete(result) transcript = [ { @@ -630,7 +737,11 @@ class TextPipelineRunner: if message.get("role") in {"user", "assistant"} and str(message.get("content") or "").strip() ] - return TextCaseResult(turns=results, transcript=transcript) + return TextCaseResult( + turns=results, + transcript=transcript, + startup=startup, + ) except TestExecutionError: raise except asyncio.CancelledError: diff --git a/backend/test_schemas.py b/backend/test_schemas.py index 297836e..b97c958 100644 --- a/backend/test_schemas.py +++ b/backend/test_schemas.py @@ -316,6 +316,10 @@ class BatchRunCreate(CamelModel): assistant_id: str = Field(min_length=1, max_length=40) evaluator_model_resource_id: str = Field(min_length=1, max_length=40) case_ids: list[str] = Field(min_length=1, max_length=1_000) + dynamic_variables: dict[str, str | int | float | bool] = Field( + default_factory=dict, + max_length=50, + ) config: BatchRunConfig = Field(default_factory=BatchRunConfig) title: str | None = Field(default=None, max_length=256) @@ -363,11 +367,19 @@ class BatchTurnResult(CamelModel): evaluations: list[BatchEvaluationResult] = Field(default_factory=list) +class BatchStartupResult(CamelModel): + completed: bool = False + greeting: str = "" + assistant_outputs: list[str] = Field(default_factory=list) + auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list) + + class BatchRunCaseOut(CamelModel): id: str name: str status: CaseStatus turns: list[BatchTurnResult] = Field(default_factory=list) + startup: BatchStartupResult = Field(default_factory=BatchStartupResult) overall_criteria: list[BatchEvaluationResult] = Field(default_factory=list) attempt_count: int max_attempts: int diff --git a/backend/tests/test_debug_auto_test.py b/backend/tests/test_debug_auto_test.py new file mode 100644 index 0000000..6e03129 --- /dev/null +++ b/backend/tests/test_debug_auto_test.py @@ -0,0 +1,264 @@ +import asyncio +import unittest + +from models import AssistantConfig, RuntimeTool +from pipecat.frames.frames import ErrorFrame +from pipecat.processors.aggregators.llm_context import LLMContext +from services.brains import BrainRuntime, build_brain +from services.brains.base import GREETING_CONTEXT_MARKER +from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config +from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort +from services.test_runs.text_runner import ( + _TextCaptureProcessor, + _wait_for_startup, + _wait_for_quiescence, +) +from test_schemas import BatchRunCreate + + +class FakeLLM: + def __init__(self): + self.functions = {} + + def register_function(self, name, handler, **_options): + self.functions[name] = handler + + +class FakeCallEnd: + def __init__(self): + self.ending = False + self.reason = "" + + @property + def accepts_input(self): + return not self.ending + + def begin(self, reason, *, interruptible=False): + self.ending = True + self.reason = reason + + def cancel_pending_end(self): + return False + + def begin_response(self): + return None + + def arm_after_speech(self): + return None + + def track_speech(self): + return None + + async def arm_after_tracked_speech(self): + return None + + async def finish_after_current_speech(self, *, has_text): + return None + + async def finish(self): + return None + + +class FakeFunctionParams: + def __init__(self, arguments=None): + self.arguments = arguments or {} + self.result = None + self.properties = None + + async def result_callback(self, result, properties=None): + self.result = result + self.properties = properties + + +def system_runtime_tool(kind: str) -> RuntimeTool: + return RuntimeTool( + id=f"tool_{kind}", + name=kind, + function_name=kind, + type="system", + definition={"config": {"kind": kind}}, + ) + + +class DebugAutoTestContractTests(unittest.TestCase): + def test_batch_run_accepts_camel_case_dynamic_variables(self): + request = BatchRunCreate.model_validate( + { + "assistantId": "asst_1", + "evaluatorModelResourceId": "model_1", + "caseIds": ["case_1"], + "dynamicVariables": {"customer_name": "测试用户", "count": 2}, + } + ) + + self.assertEqual( + request.dynamic_variables, + {"customer_name": "测试用户", "count": 2}, + ) + + def test_each_test_case_gets_fresh_conversation_variables(self): + config = AssistantConfig( + type="prompt", + prompt="你好，{{ customer_name }}", + dynamic_variable_definitions={ + "customer_name": { + "type": "string", + "required": True, + "default": None, + } + }, + ) + + first = prepare_dynamic_config( + config, + {"customer_name": "测试用户"}, + assistant_id="asst_1", + ) + second = prepare_dynamic_config( + config, + {"customer_name": "测试用户"}, + assistant_id="asst_1", + ) + + self.assertNotEqual(first.conversation_id, second.conversation_id) + self.assertEqual( + DynamicVariableStore.from_config(first).render(first.prompt), + "你好，测试用户", + ) + + def test_text_client_port_records_automatic_message_confirmation(self): + port = TextClientToolPort() + + result = asyncio.run( + port.call( + "show_message", + {"title": "重要提示", "message": "请确认"}, + ) + ) + + self.assertEqual(result["data"]["action"], "confirmed") + self.assertEqual( + port.auto_confirmed_messages, + [{"title": "重要提示", "message": "请确认"}], + ) + + +class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase): + async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self): + cfg = AssistantConfig( + type="prompt", + tools=[system_runtime_tool("end_conversation")], + ) + brain = build_brain(cfg) + llm = FakeLLM() + call_end = FakeCallEnd() + executor_holder = {} + + def create_executor(store): + executor = MockToolExecutor(store) + executor_holder["executor"] = executor + return executor + + await brain.setup( + cfg, + BrainRuntime( + context=LLMContext(messages=[]), + llm=llm, + queue_frame=lambda _frame: asyncio.sleep(0), + set_system_prompt=lambda _prompt: None, + set_tools=lambda _tools: None, + call_end=call_end, + tool_executor_factory=create_executor, + ), + ) + + params = FakeFunctionParams({"reason": "测试完成"}) + await llm.functions["end_conversation"](params) + + self.assertTrue(call_end.ending) + self.assertEqual(call_end.reason, "测试完成") + self.assertEqual(params.result["action"], "ending_call") + self.assertEqual(executor_holder["executor"].unmocked_calls, []) + + async def test_startup_waits_for_queued_entry_reply(self): + capture = _TextCaptureProcessor(LLMContext(messages=[])) + + class FakeWorker: + async def queue_frame(self, frame): + async def process_startup_queue(): + capture.window.active_llm += 1 + capture.window.touch() + await asyncio.sleep(0.01) + capture.outputs.append("开场后的自动首句") + capture.window.active_llm -= 1 + capture.window.touch() + frame.reached.set() + + asyncio.create_task(process_startup_queue()) + + runner_task = asyncio.create_task(asyncio.sleep(60)) + try: + await asyncio.wait_for( + _wait_for_startup( + FakeWorker(), + capture, + runner_task, + capture.window.activity_count, + ), + timeout=1, + ) + self.assertEqual(capture.outputs, ["开场后的自动首句"]) + finally: + runner_task.cancel() + await asyncio.gather(runner_task, return_exceptions=True) + + async def test_text_startup_can_record_greeting_as_already_played(self): + cfg = AssistantConfig( + type="prompt", + greeting="您好，这是一段开场白。", + prompt="不要重复开场白。", + ) + brain = build_brain(cfg) + context = LLMContext( + messages=[{"role": "system", "content": cfg.prompt}] + ) + + greeting = await brain.greeting(cfg) + brain.prepare_greeting_context(greeting, context) + + self.assertIn( + GREETING_CONTEXT_MARKER, + context.get_messages()[1]["content"], + ) + self.assertIn(cfg.greeting, context.get_messages()[1]["content"]) + + async def test_upstream_pipeline_error_releases_pending_tool_wait(self): + capture = _TextCaptureProcessor(LLMContext(messages=[])) + capture.window.pending_tools.add("call_1") + runner_task = asyncio.create_task(asyncio.sleep(60)) + try: + capture.fail( + ErrorFrame( + error=( + "UNMOCKED_TOOL_CALL: 工具 external_lookup " + "未配置 Mock 返回值" + ) + ) + ) + + with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"): + await asyncio.wait_for( + _wait_for_quiescence(capture, runner_task, 0), + timeout=0.1, + ) + self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL") + self.assertEqual(capture.error.stage, "tool") + self.assertFalse(capture.error.retryable) + self.assertEqual(capture.window.pending_tools, set()) + finally: + runner_task.cancel() + await asyncio.gather(runner_task, return_exceptions=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/frontend/src/components/assistant-editor/debug-auto-test.tsx b/frontend/src/components/assistant-editor/debug-auto-test.tsx index c39cc56..8aece17 100644 --- a/frontend/src/components/assistant-editor/debug-auto-test.tsx +++ b/frontend/src/components/assistant-editor/debug-auto-test.tsx @@ -1,8 +1,11 @@ "use client"; /** - * 调试抽屉「自动测试」模式：选择用例 → 一键运行 → 原地看进度与结果。 - * MVP 用本地 mock 数据模拟跑测，不接后端；后续换成真实 API 时主要改这里。 + * Debug Drawer 的真实自动测试客户端。 + * + * 它复用批量测试的持久化 Test Run API，但每次只运行一个用例。这样 Drawer + * 与批量测试共享同一套 Pipeline、工具 Mock、评估器和运行记录，不维护第二套 + * 测试语义。 */ import type React from "react"; @@ -10,11 +13,12 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Check, ChevronDown, - ChevronLeft, - ChevronRight, FlaskConical, + Loader2, Play, Square, + TriangleAlert, + Wrench, X, } from "lucide-react"; @@ -27,352 +31,76 @@ import { } from "@/components/ui/popover"; import { SearchInput } from "@/components/ui/search-input"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import type { + BatchEvaluationResult, + BatchRunCase, + BatchRunSnapshot, + BatchStartupResult, +} from "@/data/batch-run"; +import { + getTestCaseValidationMessage, + TEST_CASE_INPUT_MODE_SHORT_LABEL, + type TestCase, + type TestSuite, +} from "@/data/test-suites"; +import { + batchRunsApi, + modelResourcesApi, + testCasesApi, + testSuitesApi, + type ModelResource, +} from "@/lib/api"; import { cn } from "@/lib/utils"; -// ─── Types ─────────────────────────────────────────────────────────────────── +const RUN_POLL_MS = 1000; -export type TestCaseKind = - | "user_simulation" - | "tool_call" - | "fixed_dialogue" - | "single_turn" - | "voice_run"; - -export type ChatScriptTurn = { - role: "user" | "assistant"; - content: string; -}; - -export type ToolScriptResult = { - input: string; - expected: string; - actual: string; - actualArgs?: Record; - toolOk: boolean; - argsOk: boolean; -}; - -export type DebugTestCase = { - id: string; - name: string; - kind: TestCaseKind; - goal: string; - maxTurns: number; - /** 对话类用例的脚本；工具调用类可为空 */ - script: ChatScriptTurn[]; - toolResult?: ToolScriptResult; - /** mock：最终判定；failAtTurn 表示第几轮失败（1-based） */ - outcome: "pass" | "fail"; - failAtTurn?: number; -}; - -/** 测试集：Debug Drawer 只浏览和选择，不做管理。 */ -export type DebugTestSuite = { - id: string; - name: string; - caseIds: string[]; -}; - -export type AutoTestPhase = "idle" | "ready" | "running" | "passed" | "failed"; - -export type AutoTestLiveMessage = { - id: string; - role: "user" | "assistant"; - content: string; - /** 固定输入 / AI Simulator 进度角标 */ - progressLabel?: string; -}; +export type AutoTestPhase = + | "idle" + | "ready" + | "starting" + | "running" + | "passed" + | "failed" + | "error" + | "cancelled"; export type AutoTestRunState = { phase: AutoTestPhase; selectedId: string | null; - runCount: "1" | "3" | "5"; currentTurn: number; maxTurns: number; - liveMessages: AutoTestLiveMessage[]; - toolResult: ToolScriptResult | null; - failAtTurn: number | null; + run: BatchRunSnapshot | null; + runCase: BatchRunCase | null; + error: string; + stopping: boolean; + blockedMessage: string; + evaluatorName: string; }; -const KIND_LABEL: Record = { - user_simulation: "用户模拟", - tool_call: "工具调用", - fixed_dialogue: "固定对话", - single_turn: "单轮响应", - voice_run: "语音运行", +type AutoTestRunnerOptions = { + assistantId: string | null; + enabled: boolean; + hasUnsavedChanges: boolean; + dynamicVariables: Record; + dynamicVariablesError: string; }; -const RECENT_IDS = ["tc-dual-car", "tc-hello"]; - -/** 轻量占位用例：能选中浏览，点运行会快速 PASS（完整剧本见下方 runnable cases） */ -function stubCase( - id: string, - name: string, - kind: TestCaseKind, - goal: string, -): DebugTestCase { - return { - id, - name, - kind, - goal, - maxTurns: kind === "tool_call" || kind === "single_turn" ? 1 : 4, - outcome: "pass", - script: - kind === "tool_call" - ? [] - : [ - { role: "user", content: `（${name}）开始` }, - { role: "assistant", content: "好的，已收到。" }, - ], - toolResult: - kind === "tool_call" - ? { - input: name, - expected: "ok", - actual: "ok", - toolOk: true, - argsOk: true, - } - : undefined, - }; -} - -// ─── Mock catalog ──────────────────────────────────────────────────────────── - -export const MOCK_TEST_CASES: DebugTestCase[] = [ - { - id: "tc-dual-car", - name: "正常双车事故", - kind: "user_simulation", - goal: "完成事故处理且不得转人工", - maxTurns: 10, - outcome: "pass", - script: [ - { role: "user", content: "我这里刚刚撞了一下。" }, - { role: "assistant", content: "请您简单描述一下事发经过。" }, - { - role: "user", - content: "我正常开着，对方突然变道，两车侧面蹭到一起。", - }, - { - role: "assistant", - content: "好的，请问有没有人员受伤？车辆能否正常移动？", - }, - { role: "user", content: "人都没事，车还能开，停在应急车道了。" }, - { - role: "assistant", - content: "请保持安全距离，我帮您记录事故信息并引导后续处理。", - }, - { role: "user", content: "好的，对方也同意了。" }, - { - role: "assistant", - content: "已记录双方信息。请按提示完成拍照取证，稍后会生成处理指引。", - }, - ], - }, - stubCase( - "tc-no-injury", - "无人伤流程", - "fixed_dialogue", - "无人伤场景下完成快处引导", - ), - { - id: "tc-injury-transfer", - name: "有人伤转人工", - kind: "tool_call", - goal: "检测到人伤后调用 transfer_to_human", - maxTurns: 1, - outcome: "pass", - script: [], - toolResult: { - input: "有人受伤", - expected: "transfer_to_human", - actual: "transfer_to_human", - actualArgs: { reason: "injury" }, - toolOk: true, - argsOk: true, - }, - }, - stubCase( - "tc-refuse-photo", - "拒绝拍照", - "user_simulation", - "用户拒绝拍照时仍能完成必要取证引导", - ), - { - id: "tc-hello", - name: "用户说“喂”", - kind: "single_turn", - goal: "礼貌回应并引导用户说明来意", - maxTurns: 1, - outcome: "pass", - script: [ - { role: "user", content: "喂" }, - { - role: "assistant", - content: "您好，我是事故处理助手。请问需要什么帮助？", - }, - ], - }, - { - id: "tc-fixed-claim", - name: "固定话术理赔引导", - kind: "fixed_dialogue", - goal: "按固定剧本完成理赔引导", - maxTurns: 5, - outcome: "fail", - failAtTurn: 5, - script: [ - { role: "user", content: "我想报案理赔。" }, - { role: "assistant", content: "好的，请问事故发生时间是？" }, - { role: "user", content: "今天下午三点左右。" }, - { role: "assistant", content: "地点呢？" }, - { role: "user", content: "环城高速出口附近。" }, - { - role: "assistant", - content: "抱歉，我没听清地点，请您再说一次。", - }, - ], - }, - { - id: "tc-noise", - name: "噪声打断恢复", - kind: "user_simulation", - goal: "噪声后仍能继续对话", - maxTurns: 6, - outcome: "pass", - script: [ - { role: "user", content: "喂？听得到吗？" }, - { role: "assistant", content: "听得到，请继续说。" }, - { role: "user", content: "（环境嘈杂）对方车窗没摇上…" }, - { - role: "assistant", - content: "抱歉刚才有些听不清，您是说对方车窗没有摇上吗？", - }, - ], - }, - stubCase( - "tc-mumble", - "含糊地点澄清", - "user_simulation", - "地点表述不清时主动澄清", - ), - stubCase( - "tc-empty-utterance", - "空输入等待", - "single_turn", - "空输入时不误触发工具", - ), - stubCase( - "tc-transfer-tool", - "主动转人工工具", - "tool_call", - "用户明确要求转人工时调用工具", - ), - stubCase( - "tc-end-call-tool", - "结束通话工具", - "tool_call", - "流程结束后调用结束通话", - ), - stubCase( - "tc-barge-in", - "用户打断播报", - "user_simulation", - "播报中打断后正确切换聆听", - ), - stubCase( - "tc-long-silence", - "长静音追问", - "user_simulation", - "长静音后温和追问一次", - ), -]; - -/** 测试集（MVP mock）；测试集本身不绑定助手。 */ -export const MOCK_TEST_SUITES: DebugTestSuite[] = [ - { - id: "suite-accident-basic", - name: "事故快处基础流程", - caseIds: [ - "tc-dual-car", - "tc-no-injury", - "tc-injury-transfer", - "tc-refuse-photo", - ], - }, - { - id: "suite-clarify", - name: "异常输入与澄清", - caseIds: [ - "tc-hello", - "tc-noise", - "tc-mumble", - "tc-empty-utterance", - "tc-fixed-claim", - ], - }, - { - id: "suite-tools", - name: "工具与系统联动", - caseIds: ["tc-injury-transfer", "tc-transfer-tool", "tc-end-call-tool"], - }, - { - id: "suite-realtime-voice", - name: "实时语音交互", - caseIds: ["tc-hello", "tc-noise", "tc-barge-in", "tc-long-silence"], - }, -]; - -function findCase(id: string | null): DebugTestCase | null { - if (!id) return null; - return MOCK_TEST_CASES.find((item) => item.id === id) ?? null; -} - -function findSuite(id: string | null): DebugTestSuite | null { - if (!id) return null; - return MOCK_TEST_SUITES.find((item) => item.id === id) ?? null; -} - -function casesInSuites(suites: DebugTestSuite[]): DebugTestCase[] { - const seen = new Set(); - const cases: DebugTestCase[] = []; - for (const suite of suites) { - for (const caseId of suite.caseIds) { - if (seen.has(caseId)) continue; - seen.add(caseId); - const item = findCase(caseId); - if (item) cases.push(item); - } - } - return cases; -} - -function matchCaseQuery(item: DebugTestCase, query: string): boolean { - const q = query.trim().toLowerCase(); - if (!q) return true; +function isRunFinished(run: BatchRunSnapshot): boolean { return ( - item.name.toLowerCase().includes(q) || - KIND_LABEL[item.kind].toLowerCase().includes(q) + Boolean(run.finishedAt) || + run.status === "completed" || + run.status === "cancelled" ); } -function progressLabelFor( - kind: TestCaseKind, - turnIndex: number, - maxTurns: number, -): string { - if (kind === "fixed_dialogue") { - return `固定输入 ${turnIndex} / ${maxTurns}`; +function runnableMessage(item: TestCase): string { + if (item.inputMode !== "fixed_script_text") { + return "当前自动测试仅支持固定脚本 · 文字"; } - if (kind === "user_simulation") { - return `AI Simulator ${turnIndex} / ${maxTurns}`; - } - return `${turnIndex} / ${maxTurns}`; + return getTestCaseValidationMessage(item) ?? ""; } -// ─── Mode tabs (手动 / 自动) ────────────────────────────────────────────────── +// ─── Mode tabs ────────────────────────────────────────────────────────────── export function DebugModeTabs({ mode, @@ -395,436 +123,552 @@ export function DebugModeTabs({ ); } -// ─── Case summary (选中未开跑时居中展示) ───────────────────────────────────── +// ─── Case picker ──────────────────────────────────────────────────────────── -export function AutoTestCaseSummary({ - testCase, -}: { - testCase: DebugTestCase; -}) { - return ( -
-
-
- {testCase.name} -
- - {KIND_LABEL[testCase.kind]} - -
-
- ); -} - -// ─── Live transcript / tool result ─────────────────────────────────────────── - -export function AutoTestLivePanel({ - testCase, - state, -}: { - testCase: DebugTestCase | null; - state: AutoTestRunState; -}) { - const scrollRef = useRef(null); - const started = - state.phase === "running" || - state.phase === "passed" || - state.phase === "failed"; - - useEffect(() => { - const el = scrollRef.current; - if (el) el.scrollTop = el.scrollHeight; - }, [state.liveMessages, state.toolResult, state.phase]); - - if (!testCase) { - return ( -
- -
- 选择一个测试用例 -
-

- 在底部运行条中选择用例，即可在这里查看目标与自动对话回放。 -

-
- ); - } - - // 已选中、尚未开始：中间居中展示用例详情 - if (!started) { - return ; - } - - if (testCase.kind === "tool_call") { - const result = state.toolResult; - - if (state.phase === "running" && !result) { - return ( -
- -

正在执行工具断言…

-
- ); - } - - if (!result) { - return ( -
-

暂无工具调用结果

-
- ); - } - - return ( -
-
- - - -
- - {result.toolOk ? : } - Tool - - - {result.argsOk ? : } - Arguments - -
-
-
- ); - } - - if (state.liveMessages.length === 0) { - return ( -
- -

正在开始自动对话…

-
- ); - } - - return ( -
-
- {state.liveMessages.map((message) => - message.role === "assistant" ? ( -
- 助手 -
- {message.content} -
-
- ) : ( -
- - AI 用户 - {message.progressLabel && ( - - {message.progressLabel} - - )} - -
- {message.content} -
-
- ), - )} -
-
- ); -} - -function ToolResultBlock({ - label, - body, - mono = false, - json, -}: { - label: string; - body: string; - mono?: boolean; - json?: Record; -}) { - return ( -
-
{label}
-
- {body} - {json && ( - 
-            {JSON.stringify(json, null, 2)}
-
- )} -
-
- ); -} - -// ─── Case picker popover ───────────────────────────────────────────────────── - -function CaseRowButton({ - item, - selected, - onSelect, -}: { - item: DebugTestCase; - selected: boolean; - onSelect: () => void; -}) { - return ( - - ); -} - -function TestCasePickerPopover({ - open, - onOpenChange, +function CasePicker({ + suites, + cases, selectedId, + loading, onSelect, trigger, }: { - open: boolean; - onOpenChange: (open: boolean) => void; + suites: TestSuite[]; + cases: TestCase[]; selectedId: string | null; + loading: boolean; onSelect: (id: string) => void; trigger: React.ReactNode; }) { + const [open, setOpen] = useState(false); const [query, setQuery] = useState(""); - const [activeSuiteId, setActiveSuiteId] = useState(null); + const normalizedQuery = query.trim().toLocaleLowerCase(); - const suites = MOCK_TEST_SUITES; - const scopedCases = useMemo(() => casesInSuites(suites), [suites]); - const activeSuite = findSuite(activeSuiteId); - - const recent = useMemo(() => { - const scopedIds = new Set(scopedCases.map((item) => item.id)); - return RECENT_IDS.map((id) => findCase(id)).filter( - (item): item is DebugTestCase => - item !== null && scopedIds.has(item.id), - ); - }, [scopedCases]); - - const suiteCases = useMemo(() => { - if (!activeSuite) return []; - return activeSuite.caseIds - .map((id) => findCase(id)) - .filter((item): item is DebugTestCase => Boolean(item)) - .filter((item) => matchCaseQuery(item, query)); - }, [activeSuite, query]); - - const searchHits = useMemo(() => { - if (!query.trim()) return []; - return scopedCases.filter((item) => matchCaseQuery(item, query)); - }, [scopedCases, query]); - - function pickCase(id: string) { - onSelect(id); - onOpenChange(false); - } - - const inSuite = Boolean(activeSuite); + const visibleSuites = useMemo( + () => + suites + .map((suite) => ({ + suite, + cases: cases.filter((item) => item.suiteId === suite.id), + })) + .map((group) => ({ + ...group, + cases: group.cases.filter((item) => { + if (!normalizedQuery) return true; + return [ + item.name, + item.description, + TEST_CASE_INPUT_MODE_SHORT_LABEL[item.inputMode], + ].some((value) => value.toLocaleLowerCase().includes(normalizedQuery)); + }), + })) + .filter((group) => group.cases.length > 0), + [cases, normalizedQuery, suites], + ); return ( { - onOpenChange(next); - if (!next) { - setQuery(""); - setActiveSuiteId(null); - } + setOpen(next); + if (!next) setQuery(""); }} > {trigger} - {inSuite && activeSuite ? ( - <> - - -
- {suiteCases.length === 0 ? ( -
- 没有匹配的测试用例 +
+ 选择测试用例 +
+ +
+ {loading ? ( +
+ + 加载真实测试用例… +
+ ) : visibleSuites.length === 0 ? ( +
+ {cases.length === 0 ? "还没有测试用例" : "没有匹配的测试用例"} +
+ ) : ( + visibleSuites.map(({ suite, cases: suiteCases }) => ( +
+
+ {suite.name} + {suiteCases.length}
- ) : ( - suiteCases.map((item) => ( - pickCase(item.id)} - /> - )) - )} -
- - ) : ( - <> -
- 选择测试用例 -
- -
- {query.trim() ? ( -
- {searchHits.length === 0 ? ( -
- 没有匹配的测试用例 -
- ) : ( - searchHits.map((item) => ( - pickCase(item.id)} - /> - )) - )} -
- ) : ( - <> - {recent.length > 0 && ( -
-
- 最近使用 -
- {recent.map((item) => ( - pickCase(item.id)} - /> - ))} -
- )} -
-
- 测试集 -
- {suites.length === 0 ? ( -
- 暂无测试集 -
- ) : ( - suites.map((suite) => ( - - )) - )} -
- - )} -
- - )} + {suiteCases.map((item) => { + const unsupported = Boolean(runnableMessage(item)); + return ( + + ); + })} + + )) + )} +
); } -// ─── Bottom run bar ────────────────────────────────────────────────────────── +// ─── Result panel ─────────────────────────────────────────────────────────── + +export function AutoTestLivePanel({ + testCase, + state, +}: { + testCase: TestCase | null; + state: AutoTestRunState; +}) { + if (!testCase) { + return ( + } + title="选择一个测试用例" + description="运行真实 Pipeline，并在这里查看逐轮回复、工具调用和判断结果。" + /> + ); + } + + if (state.phase === "idle" || state.phase === "ready") { + const validationMessage = runnableMessage(testCase); + return ( +
+
+
+ {testCase.name} +
+
+ + {TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]} + + + {testCase.turns.length} 轮 + +
+ {testCase.description && ( +

+ {testCase.description} +

+ )} + {validationMessage ? ( +
+ {validationMessage} +
+ ) : state.evaluatorName ? ( +

+ 评估模型 · {state.evaluatorName} +

+ ) : null} +
+
+ ); + } + + if ( + state.phase === "error" && + state.runCase?.executionError && + state.runCase.turns.length === 0 + ) { + return ( + + ); + } + + if (state.phase === "error" && state.error && !state.runCase) { + return ; + } + + const turns = state.runCase?.turns ?? []; + const startup = state.runCase?.startup; + if (turns.length === 0) { + if (startup?.completed) { + return ( +
+ + +
+ + 正在执行第 1 轮固定输入… +
+
+ ); + } + return ( + } + title={state.stopping ? "正在停止测试…" : "正在启动真实 Pipeline…"} + description="测试在后端运行，结果会自动同步到这里。" + /> + ); + } + + const evaluating = + state.phase === "running" && state.currentTurn >= state.maxTurns; + + return ( +
+
+ {startup?.completed && } + {startup?.completed && } + {turns.map((turn) => ( +
+
+ + 固定输入 + + {turn.index + 1} / {state.maxTurns} + + +
+ {turn.userInput} +
+
+ +
+ 助手 +
+ {turn.assistantReply || "（没有生成文字回复）"} +
+
+ + {turn.toolCalls.length > 0 && ( +
+
+ + 工具调用 +
+ {turn.toolCalls.map((toolCall) => ( +
+ {toolCall.functionName} + + {toolCall.outcome === "success" ? "成功" : "错误"} ·{" "} + {toolCall.durationMs} ms + + 
+                      {toolCall.argumentsJson}
+
+
+ ))} +
+ )} + + {turn.evaluations.length > 0 && ( +
+ {turn.evaluations.map((evaluation) => ( + + ))} +
+ )} +
+ ))} + + {state.phase === "running" && ( +
+ + {state.stopping + ? "正在停止测试…" + : evaluating + ? "对话已完成，正在评估结果…" + : `正在执行第 ${state.currentTurn + 1} 轮…`} +
+ )} + + {state.phase === "error" && state.runCase?.executionError && ( +
+ + {state.runCase.executionError.code} + + · {state.runCase.executionError.message} +
+ )} + + {state.runCase && state.runCase.overallCriteria.length > 0 && ( +
+
+ 整体评估标准 +
+ {state.runCase.overallCriteria.map((evaluation) => ( + + ))} +
+ )} +
+
+ ); +} + +function StartupDetails({ startup }: { startup: BatchStartupResult }) { + const greeting = startup.greeting.trim(); + + return ( +
+
+
+
开场记录
+

+ 已按真实开场流程模拟，不计入脚本轮次断言 +

+
+ + 已完成 + +
+ + {greeting && ( + + )} + + {startup.autoConfirmedMessages.map((message, index) => { + const confirmedAction = + message.actions?.find((action) => action.id === "confirmed") ?? + message.actions?.[0]; + return ( +
+
+ 消息弹窗{startup.autoConfirmedMessages.length > 1 ? ` ${index + 1}` : ""} +
+
+ {message.title?.trim() || "未命名消息"} +
+

+ {message.message?.trim() || "（没有消息正文）"} +

+
+ + 自动点击“{confirmedAction?.label?.trim() || "确认"}” +
+
+ ); + })} +
+ ); +} + +function StartupAssistantMessages({ + startup, +}: { + startup: BatchStartupResult; +}) { + const openingTexts = new Set( + [ + startup.greeting, + ...startup.autoConfirmedMessages.map((message) => message.message ?? ""), + ] + .map(normalizeDisplayText) + .filter(Boolean), + ); + const seen = new Set(); + const assistantMessages = startup.assistantOutputs.filter((output) => { + const normalizedOutput = normalizeDisplayText(output); + if ( + !normalizedOutput || + openingTexts.has(normalizedOutput) || + seen.has(normalizedOutput) + ) { + return false; + } + seen.add(normalizedOutput); + return true; + }); + + if (assistantMessages.length === 0) return null; + + const label = + startup.autoConfirmedMessages.length > 0 + ? "助手 · 确认后自动输出" + : "助手 · 开场自动输出"; + + return ( +
+ {assistantMessages.map((message, index) => ( +
+ {label} +
+ {message.trim()} +
+
+ ))} +
+ ); +} + +function normalizeDisplayText(value: string): string { + return value.trim().replace(/\s+/g, " "); +} + +function StartupTextBlock({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+

+ {value} +

+
+ ); +} + +function EvaluationDetails({ + evaluation, +}: { + evaluation: BatchEvaluationResult; +}) { + const passed = evaluation.status === "pass"; + return ( +
+
+ + {passed ? : } + +
+ {evaluation.label} +
+ + {passed ? "通过" : "未通过"} + +
+
+
评估标准
+
+ {evaluation.expected || "（未提供评估标准）"} +
+
实际结果
+
+ {evaluation.actual || "（没有可评估内容）"} +
+ {evaluation.reason && ( + <> +
+ 判断说明 +
+
+ {evaluation.reason} +
+ + )} +
+
+ ); +} + +function EmptyPanel({ + icon, + title, + description, +}: { + icon: React.ReactNode; + title: string; + description: string; +}) { + return ( +
+ {icon} +
{title}
+

+ {description} +

+
+ ); +} + +function ErrorPanel({ title, description }: { title: string; description: string }) { + return ( +
+ + + +
{title}
+

+ {description} +

+
+ ); +} + +// ─── Bottom run bar ───────────────────────────────────────────────────────── export function AutoTestRunBar({ state, + testCase, + suites, + cases, + loading, onSelectCase, onClearCase, onStart, @@ -832,114 +676,118 @@ export function AutoTestRunBar({ onRerun, }: { state: AutoTestRunState; + testCase: TestCase | null; + suites: TestSuite[]; + cases: TestCase[]; + loading: boolean; onSelectCase: (id: string) => void; onClearCase: () => void; onStart: () => void; onStop: () => void; onRerun: () => void; }) { - const [pickerOpen, setPickerOpen] = useState(false); - const testCase = findCase(state.selectedId); - - if (state.phase === "idle" || !testCase) { - return ( -
- - - 选择测试用例… - - - } - /> - -
- ); - } - - if (state.phase === "running") { - return ( -
+ const pickerTrigger = ( + + ); + + if (state.phase === "starting" || state.phase === "running") { + return ( +
+ - {testCase.name} + {testCase?.name} - - - 运行中 - {testCase.kind !== "tool_call" && ( - - {state.currentTurn} / {state.maxTurns} - - )} + + {state.stopping + ? "停止中" + : state.phase === "starting" + ? "启动中" + : `${state.currentTurn} / ${state.maxTurns}`}
); } - if (state.phase === "passed" || state.phase === "failed") { + if ( + state.phase === "passed" || + state.phase === "failed" || + state.phase === "error" || + state.phase === "cancelled" + ) { const passed = state.phase === "passed"; + const label = passed + ? `PASS · ${state.currentTurn} 轮` + : state.phase === "failed" + ? "FAIL" + : state.phase === "cancelled" + ? "已停止" + : "执行错误"; return ( -
+
{passed ? : } - {testCase.name} + {testCase?.name} - {passed - ? `PASS · ${state.currentTurn}轮` - : `FAIL · 第${state.failAtTurn ?? state.currentTurn}轮`} + {label} @@ -947,219 +795,301 @@ export function AutoTestRunBar({ ); } - // ready：左侧选用例 + 清除；右侧灰色开始按钮 return ( -
-
- +
+
+ + {testCase && ( - } - /> - + + 开始测试 +
- + {state.blockedMessage && testCase && ( +

+ {state.blockedMessage} +

+ )}
); } -// ─── Hook: run simulation ──────────────────────────────────────────────────── +// ─── Hook: real single-case run ───────────────────────────────────────────── -const INITIAL_RUN_STATE: AutoTestRunState = { - phase: "idle", - selectedId: null, - runCount: "1", - currentTurn: 0, - maxTurns: 0, - liveMessages: [], - toolResult: null, - failAtTurn: null, -}; +export function useAutoTestRunner({ + assistantId, + enabled, + hasUnsavedChanges, + dynamicVariables, + dynamicVariablesError, +}: AutoTestRunnerOptions) { + const [suites, setSuites] = useState([]); + const [cases, setCases] = useState([]); + const [evaluatorModels, setEvaluatorModels] = useState([]); + const [evaluatorModelId, setEvaluatorModelId] = useState(""); + const [catalogLoading, setCatalogLoading] = useState(true); + const [catalogError, setCatalogError] = useState(""); + const [selectedId, setSelectedId] = useState(null); + const [run, setRun] = useState(null); + const activeRunRef = useRef(null); + const [starting, setStarting] = useState(false); + const [stopping, setStopping] = useState(false); + const [runError, setRunError] = useState(""); -export function useAutoTestRunner() { - const [state, setState] = useState(INITIAL_RUN_STATE); - const timersRef = useRef([]); - const stoppedRef = useRef(false); + useEffect(() => { + activeRunRef.current = run; + }, [run]); - const clearTimers = useCallback(() => { - for (const id of timersRef.current) window.clearTimeout(id); - timersRef.current = []; - }, []); + useEffect( + () => () => { + const activeRun = activeRunRef.current; + if (activeRun && !isRunFinished(activeRun)) { + void batchRunsApi.cancel(activeRun.id); + } + }, + [], + ); - useEffect(() => () => clearTimers(), [clearTimers]); + useEffect(() => { + if (!enabled) return; + let disposed = false; + + void Promise.all([ + testSuitesApi.list(), + testCasesApi.list(), + modelResourcesApi.list(), + ]) + .then(([loadedSuites, loadedCases, resources]) => { + if (disposed) return; + const llms = resources.filter( + (item) => item.capability === "LLM" && item.enabled, + ); + setSuites(loadedSuites); + setCases(loadedCases); + setEvaluatorModels(llms); + setCatalogError(""); + const evaluator = llms.find((item) => item.isDefault) ?? llms[0]; + setEvaluatorModelId(evaluator?.id ?? ""); + }) + .catch((error) => { + if (disposed) return; + setCatalogError( + error instanceof Error ? error.message : "加载测试用例失败", + ); + }) + .finally(() => { + if (!disposed) setCatalogLoading(false); + }); + + return () => { + disposed = true; + }; + }, [enabled]); + + useEffect(() => { + if (!enabled || !run || isRunFinished(run)) return; + let disposed = false; + let timer = 0; + + async function poll() { + try { + const snapshot = await batchRunsApi.get(run!.id); + if (disposed) return; + setRun(snapshot); + setRunError(""); + if (isRunFinished(snapshot)) { + setStopping(false); + return; + } + } catch (error) { + if (!disposed) { + setRunError( + error instanceof Error ? error.message : "刷新测试结果失败", + ); + } + } + if (!disposed) timer = window.setTimeout(() => void poll(), RUN_POLL_MS); + } + + timer = window.setTimeout(() => void poll(), RUN_POLL_MS); + return () => { + disposed = true; + window.clearTimeout(timer); + }; + }, [enabled, run]); + + const selectedCase = useMemo( + () => cases.find((item) => item.id === selectedId) ?? null, + [cases, selectedId], + ); + const evaluator = useMemo( + () => evaluatorModels.find((item) => item.id === evaluatorModelId) ?? null, + [evaluatorModelId, evaluatorModels], + ); + const runCase = run?.cases[0] ?? null; + + let blockedMessage = ""; + if (catalogLoading) blockedMessage = "正在加载测试用例…"; + else if (catalogError) blockedMessage = catalogError; + else if (!assistantId) blockedMessage = "请先保存助手，再开始自动测试"; + else if (hasUnsavedChanges) blockedMessage = "请先保存当前改动，再开始自动测试"; + else if (dynamicVariablesError) blockedMessage = dynamicVariablesError; + else if (!selectedCase) blockedMessage = "请先选择测试用例"; + else if (runnableMessage(selectedCase)) { + blockedMessage = runnableMessage(selectedCase); + } else if (!evaluatorModelId) blockedMessage = "请先配置可用的 LLM 评估模型"; + + let phase: AutoTestPhase = selectedCase ? "ready" : "idle"; + if (starting) phase = "starting"; + else if (run) { + if (run.status === "cancelled" || runCase?.status === "skipped") { + phase = "cancelled"; + } else if (runCase?.status === "pass") phase = "passed"; + else if (runCase?.status === "fail") phase = "failed"; + else if (runCase?.status === "error") phase = "error"; + else phase = "running"; + } else if (runError) phase = "error"; + + const state: AutoTestRunState = { + phase, + selectedId, + currentTurn: runCase?.turns.length ?? 0, + maxTurns: selectedCase?.turns.length ?? 0, + run, + runCase, + error: runError, + stopping, + blockedMessage, + evaluatorName: evaluator?.name ?? "", + }; + + const cancelActiveRun = useCallback(async () => { + if (!run || isRunFinished(run)) return; + setStopping(true); + try { + const snapshot = await batchRunsApi.cancel(run.id); + setRun(snapshot); + setRunError(""); + if (isRunFinished(snapshot)) setStopping(false); + } catch (error) { + setRunError(error instanceof Error ? error.message : "停止测试失败"); + setStopping(false); + } + }, [run]); const selectCase = useCallback( (id: string) => { - clearTimers(); - stoppedRef.current = true; - const testCase = findCase(id); - setState({ - ...INITIAL_RUN_STATE, - phase: "ready", - selectedId: id, - runCount: "1", - maxTurns: testCase?.maxTurns ?? 0, - }); + if (run && !isRunFinished(run)) void batchRunsApi.cancel(run.id); + setSelectedId(id); + setRun(null); + setRunError(""); + setStarting(false); + setStopping(false); }, - [clearTimers], + [run], ); - const setRunCount = useCallback((runCount: "1" | "3" | "5") => { - setState((current) => ({ ...current, runCount })); - }, []); - - const stop = useCallback(() => { - stoppedRef.current = true; - clearTimers(); - setState((current) => ({ - ...current, - phase: current.selectedId ? "ready" : "idle", - currentTurn: 0, - liveMessages: [], - toolResult: null, - failAtTurn: null, - })); - }, [clearTimers]); - const clearCase = useCallback(() => { - stoppedRef.current = true; - clearTimers(); - setState(INITIAL_RUN_STATE); - }, [clearTimers]); + if (run && !isRunFinished(run)) void batchRunsApi.cancel(run.id); + setSelectedId(null); + setRun(null); + setRunError(""); + setStarting(false); + setStopping(false); + }, [run]); - const start = useCallback(() => { - const testCase = findCase(state.selectedId); - if (!testCase) return; - - clearTimers(); - stoppedRef.current = false; - - setState((current) => ({ - ...current, - phase: "running", - currentTurn: 0, - maxTurns: testCase.maxTurns, - liveMessages: [], - toolResult: null, - failAtTurn: null, - })); - - // 工具调用：短暂“执行中”，再展示结果 - if (testCase.kind === "tool_call") { - const timer = window.setTimeout(() => { - if (stoppedRef.current) return; - const passed = testCase.outcome === "pass"; - setState((current) => ({ - ...current, - phase: passed ? "passed" : "failed", - currentTurn: 1, - toolResult: testCase.toolResult ?? null, - failAtTurn: passed ? null : 1, - })); - }, 900); - timersRef.current.push(timer); + const start = useCallback(async () => { + if ( + !assistantId || + !selectedCase || + !evaluatorModelId || + blockedMessage || + starting + ) { return; } + setStarting(true); + setRun(null); + setRunError(""); + try { + const snapshot = await batchRunsApi.create({ + assistantId, + evaluatorModelResourceId: evaluatorModelId, + caseIds: [selectedCase.id], + dynamicVariables, + title: `调试 · ${selectedCase.name}`, + config: { + concurrency: 1, + timeoutSecs: 60, + failureStrategy: "continue", + errorRetryCount: 0, + errorStrategy: "continue", + }, + }); + setRun(snapshot); + } catch (error) { + setRunError(error instanceof Error ? error.message : "启动自动测试失败"); + } finally { + setStarting(false); + } + }, [ + assistantId, + blockedMessage, + dynamicVariables, + evaluatorModelId, + selectedCase, + starting, + ]); - const script = testCase.script; - const userTurns = script.filter((turn) => turn.role === "user").length; - const displayMax = Math.max(testCase.maxTurns, userTurns); - let userTurnCount = 0; - - script.forEach((turn, index) => { - const delay = 700 + index * 1100; - const timer = window.setTimeout(() => { - if (stoppedRef.current) return; - - if (turn.role === "user") userTurnCount += 1; - const nextTurn = - turn.role === "user" ? userTurnCount : userTurnCount; - const label = - turn.role === "user" - ? progressLabelFor(testCase.kind, nextTurn, displayMax) - : undefined; - - setState((current) => ({ - ...current, - currentTurn: nextTurn, - maxTurns: displayMax, - liveMessages: [ - ...current.liveMessages, - { - id: `auto-${index}`, - role: turn.role, - content: turn.content, - progressLabel: label, - }, - ], - })); - - const isLast = index === script.length - 1; - if (!isLast) return; - - const finishTimer = window.setTimeout(() => { - if (stoppedRef.current) return; - const passed = testCase.outcome === "pass"; - setState((current) => ({ - ...current, - phase: passed ? "passed" : "failed", - failAtTurn: passed - ? null - : (testCase.failAtTurn ?? nextTurn), - currentTurn: passed - ? nextTurn - : (testCase.failAtTurn ?? nextTurn), - })); - }, 500); - timersRef.current.push(finishTimer); - }, delay); - timersRef.current.push(timer); - }); - }, [clearTimers, state.selectedId]); + const stop = useCallback(() => { + if (run && !isRunFinished(run)) { + void cancelActiveRun(); + return; + } + setRun(null); + setRunError(""); + }, [cancelActiveRun, run]); const rerun = useCallback(() => { - start(); + setRun(null); + setRunError(""); + window.setTimeout(() => void start(), 0); }, [start]); - const selectedCase = findCase(state.selectedId); - return { state, + suites, + cases, + catalogLoading, selectedCase, selectCase, clearCase, - setRunCount, - start, + start: () => void start(), stop, rerun, }; diff --git a/frontend/src/components/assistant-editor/debug-preview.tsx b/frontend/src/components/assistant-editor/debug-preview.tsx index f2ab65d..0e68790 100644 --- a/frontend/src/components/assistant-editor/debug-preview.tsx +++ b/frontend/src/components/assistant-editor/debug-preview.tsx @@ -183,8 +183,6 @@ export function DebugDrawer({ }) { const preview = useVoicePreview(assistantId, onNodeActive); const camera = useCameraPreview(); - const autoTest = useAutoTestRunner(); - const stopAutoTest = autoTest.stop; const [debugMode, setDebugMode] = useState("manual"); const [showTranscript, setShowTranscript] = useState(false); const [vizStyle, setVizStyle] = useState("aura"); @@ -220,6 +218,14 @@ export function DebugDrawer({ } resolvedDynamicVariables[name] = value; } + const autoTest = useAutoTestRunner({ + assistantId, + enabled: debugMode === "auto", + hasUnsavedChanges, + dynamicVariables: resolvedDynamicVariables, + dynamicVariablesError, + }); + const stopAutoTest = autoTest.stop; const selectCamera = useCallback( async (deviceId: string) => { @@ -369,6 +375,9 @@ function AutoTestPanel({ start, stop, rerun, + suites, + cases, + catalogLoading, } = autoTest; return ( @@ -379,6 +388,10 @@ function AutoTestPanel({
; + }>; +}; + export type BatchRunCase = { id: string; name: string; status: BatchCaseStatus; turns: BatchTurnResult[]; + startup?: BatchStartupResult; overallCriteria: BatchEvaluationResult[]; attemptCount: number; maxAttempts: number; diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 65a31dc..3585e8e 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -851,6 +851,7 @@ export const batchRunsApi = { assistantId: string; evaluatorModelResourceId: string; caseIds: string[]; + dynamicVariables?: Record; title?: string; config: { concurrency: number;