feat: add dynamic variables and startup result handling in batch run schemas

- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types.
- Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages.
- Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs.
- Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases.
- Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.

backend/routes/test_runs.py

@@ -17,6 +17,7 @@ from db.session import get_session
 from fastapi import APIRouter, Depends, HTTPException
 from services.auth import require_admin
 from services.config_resolver import resolve_runtime_config
+from services.runtime_variables import DynamicVariableError, prepare_dynamic_config
 from services.test_runs.orchestrator import test_run_orchestrator
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -25,6 +26,7 @@ from test_schemas import (
     BatchRunCaseOut,
     BatchRunCreate,
     BatchRunSnapshotOut,
+    BatchStartupResult,
     BatchTurnResult,
     BatchEvaluationResult,
     TestCaseDefinition,
@@ -65,6 +67,9 @@ async def _run_out(
                     BatchTurnResult.model_validate(item)
                     for item in result.get("turns") or []
                 ],
+                startup=BatchStartupResult.model_validate(
+                    result.get("startup") or {}
+                ),
                 overall_criteria=[
                     BatchEvaluationResult.model_validate(item)
                     for item in result.get("overallCriteria")
@@ -118,7 +123,12 @@ async def create_test_run(
         raise HTTPException(422, "评估模型不存在、已停用或不是 LLM 资源")
     try:
         runtime_config = await resolve_runtime_config(session, assistant.id)
-    except ValueError as exc:
+        prepare_dynamic_config(
+            runtime_config,
+            body.dynamic_variables,
+            assistant_id=assistant.id,
+        )
+    except (ValueError, DynamicVariableError) as exc:
         raise HTTPException(422, str(exc)) from exc
 
     rows = (
@@ -175,6 +185,7 @@ async def create_test_run(
             "evaluatorModel": str(
                 (evaluator_resource.values or {}).get("modelId") or ""
             ),
+            "dynamicVariables": body.dynamic_variables,
             **body.config.model_dump(mode="json", by_alias=True),
         },
         started_at=now,

backend/services/brains/prompt_brain.py

@@ -681,17 +681,10 @@ class PromptBrain(BaseBrain):
         else:
             raise ValueError(f"未知系统工具: {kind}")
 
-        if runtime.tool_executor_factory is None:
+        # System tools are platform-owned deterministic behavior. Keep their
-            return schema, handler
+        # lifecycle and state semantics identical across production and tests;
-
+        # only external HTTP, MCP, and client tools use an injected executor.
-        async def mock_system_tool(params: FunctionCallParams) -> None:
+        return schema, handler
-            result = await self._tools.execute(
-                tool,
-                dict(params.arguments or {}),
-            )
-            await params.result_callback(result)
-
-        return schema, mock_system_tool
 
     def _make_update_state_tool(self, tool, runtime: BrainRuntime):
         """更新已声明的动态变量(会话状态),并让模型继续当前回答。"""

backend/services/brains/workflow_brain.py

@@ -951,14 +951,9 @@ class WorkflowBrain(BaseBrain):
         else:
             raise ValueError(f"系统工具 {tool.id} 缺少有效 kind")
 
-        runtime = self._require_runtime()
+        # Platform-owned system behavior is deterministic. Keep it real in all
-        if runtime.tool_executor_factory is None:
+        # runtimes while an injected executor isolates only external tools.
-            return schema
+        return schema
-
-        async def mock_system_tool(args, _flow_manager):
-            return await self._tools.execute(tool, dict(args or {}))
-
-        return replace(schema, handler=mock_system_tool)
 
     def _workflow_update_state_tool(
         self,

backend/services/test_runs/mock_tools.py

@@ -69,6 +69,9 @@ class MockToolExecutor(ToolExecutor):
 class TextClientToolPort:
     """Automatically acknowledge built-in message stages in text tests."""
 
+    def __init__(self) -> None:
+        self.auto_confirmed_messages: list[dict[str, Any]] = []
+
     async def call(
         self,
         function_name: str,
@@ -76,6 +79,7 @@ class TextClientToolPort:
         **_kwargs: Any,
     ) -> dict[str, Any]:
         if function_name == "show_message":
+            self.auto_confirmed_messages.append(dict(arguments))
             return {
                 "status": "ok",
                 "data": {"action": "confirmed", "arguments": arguments},

backend/services/test_runs/orchestrator.py

@@ -13,9 +13,14 @@ from db.models import ModelResource, TestCase, TestRun, TestRunCase
 from db.session import SessionLocal
 from models import RuntimeModelResource
 from services.config_resolver import resolve_runtime_config
+from services.runtime_variables import prepare_dynamic_config
 from services.test_runs.errors import TestExecutionError
 from services.test_runs.evaluator import EvaluationEngine
-from services.test_runs.text_runner import TextPipelineRunner
+from services.test_runs.text_runner import (
+    RawTurnResult,
+    TextPipelineRunner,
+    TextStartupResult,
+)
 from test_schemas import TestCaseDefinition
 
 
@@ -74,12 +79,20 @@ class TestRunOrchestrator:
                 run.status = "running"
                 run.started_at = run.started_at or datetime.now(UTC)
                 await session.commit()
+                config = dict(run.config or {})
                 try:
                     cfg = await resolve_runtime_config(session, str(run.assistant_id or ""))
+                    # Validate the run-level values once. Each case/attempt gets
+                    # a fresh prepared copy below so conversation-scoped system
+                    # variables never leak between concurrent cases or retries.
+                    prepare_dynamic_config(
+                        cfg,
+                        config.get("dynamicVariables") or {},
+                        assistant_id=str(run.assistant_id or ""),
+                    )
                 except Exception as exc:
                     await self._fail_run_setup(run_id, str(exc))
                     return
-                config = dict(run.config or {})
                 evaluator_resource_id = str(
                     config.get("evaluatorModelResourceId") or ""
                 )
@@ -121,6 +134,7 @@ class TestRunOrchestrator:
                         run_id,
                         row.id,
                         cfg,
+                        str(run.assistant_id or ""),
                         evaluator_resource,
                         config,
                         semaphore,
@@ -146,6 +160,7 @@ class TestRunOrchestrator:
         run_id: str,
         run_case_id: str,
         cfg,
+        assistant_id: str,
         evaluator_resource: RuntimeModelResource,
         config: dict,
         semaphore: asyncio.Semaphore,
@@ -176,10 +191,29 @@ class TestRunOrchestrator:
                         or 60
                     )
                     async with asyncio.timeout(timeout_seconds):
-                        execution = await TextPipelineRunner().run(
+                        attempt_cfg = prepare_dynamic_config(
                             cfg.model_copy(deep=True),
+                            config.get("dynamicVariables") or {},
+                            assistant_id=assistant_id,
+                        )
+
+                        async def save_turn_progress(turn: RawTurnResult) -> None:
+                            await self._mark_turn_progress(run_case_id, turn)
+
+                        async def save_startup_progress(
+                            startup: TextStartupResult,
+                        ) -> None:
+                            await self._mark_startup_progress(
+                                run_case_id,
+                                startup,
+                            )
+
+                        execution = await TextPipelineRunner().run(
+                            attempt_cfg,
                             definition,
                             timeout_seconds=timeout_seconds,
+                            on_turn_complete=save_turn_progress,
+                            on_startup_complete=save_startup_progress,
                         )
                         evaluated = await EvaluationEngine(evaluator_resource).evaluate(
                             definition,
@@ -224,7 +258,12 @@ class TestRunOrchestrator:
                     return
 
                 status = "pass" if evaluated.passed else "fail"
-                await self._mark_evaluated(run_case_id, status, evaluated)
+                await self._mark_evaluated(
+                    run_case_id,
+                    status,
+                    evaluated,
+                    execution.startup,
+                )
                 if status == "fail" and str(
                     config.get("failureStrategy") or config.get("failure_strategy")
                 ) == "stop_on_fail":
@@ -256,9 +295,76 @@ class TestRunOrchestrator:
             row.started_at = row.started_at or datetime.now(UTC)
             row.finished_at = None
             row.execution_error = None
+            row.result = {
+                "turns": [],
+                "startup": {
+                    "completed": False,
+                    "greeting": "",
+                    "assistantOutputs": [],
+                    "autoConfirmedMessages": [],
+                },
+                "overallCriteria": [],
+            }
             await session.commit()
 
-    async def _mark_evaluated(self, run_case_id: str, status: str, evaluated) -> None:
+    async def _mark_startup_progress(
+        self,
+        run_case_id: str,
+        startup: TextStartupResult,
+    ) -> None:
+        async with SessionLocal() as session:
+            row = await session.get(TestRunCase, run_case_id)
+            if row is None or row.status != "running":
+                return
+            result = dict(row.result or {})
+            result["startup"] = {
+                "completed": True,
+                "greeting": startup.greeting,
+                "assistantOutputs": list(startup.assistant_outputs),
+                "autoConfirmedMessages": list(
+                    startup.auto_confirmed_messages
+                ),
+            }
+            row.result = result
+            await session.commit()
+
+    async def _mark_turn_progress(
+        self,
+        run_case_id: str,
+        turn: RawTurnResult,
+    ) -> None:
+        """Persist completed turns so polling clients can show live progress."""
+        async with SessionLocal() as session:
+            row = await session.get(TestRunCase, run_case_id)
+            if row is None or row.status != "running":
+                return
+            result = dict(row.result or {})
+            turns = list(result.get("turns") or [])
+            payload = {
+                "id": turn.id,
+                "index": turn.index,
+                "userInput": turn.user_input,
+                "assistantReply": turn.assistant_reply,
+                "toolCalls": [item.as_result() for item in turn.tool_calls],
+                "evaluations": [],
+            }
+            turns = [item for item in turns if item.get("id") != turn.id]
+            turns.append(payload)
+            turns.sort(key=lambda item: int(item.get("index") or 0))
+            row.result = {
+                "turns": turns,
+                "startup": dict(result.get("startup") or {}),
+                "overallCriteria": list(result.get("overallCriteria") or []),
+            }
+            await session.commit()
+
+    async def _mark_evaluated(
+        self,
+        run_case_id: str,
+        status: str,
+        evaluated,
+        startup: TextStartupResult,
+    ) -> None:
         async with SessionLocal() as session:
             row = await session.get(TestRunCase, run_case_id)
             if row is None:
@@ -269,6 +375,14 @@ class TestRunOrchestrator:
                     item.model_dump(mode="json", by_alias=True)
                     for item in evaluated.turns
                 ],
+                "startup": {
+                    "completed": True,
+                    "greeting": startup.greeting,
+                    "assistantOutputs": list(startup.assistant_outputs),
+                    "autoConfirmedMessages": list(
+                        startup.auto_confirmed_messages
+                    ),
+                },
                 "overallCriteria": [
                     item.model_dump(mode="json", by_alias=True)
                     for item in evaluated.overall_criteria

backend/services/test_runs/text_runner.py

@@ -16,6 +16,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.frames.frames import (
     BotStoppedSpeakingFrame,
     CancelFrame,
+    DataFrame,
     EndFrame,
     ErrorFrame,
     FunctionCallInProgressFrame,
@@ -103,10 +104,31 @@ class RawTurnResult:
     tool_calls: list[RawToolCall]
 
 
+@dataclass
+class TextStartupResult:
+    """Headless startup choices applied before the scripted user turns."""
+
+    greeting: str = ""
+    assistant_outputs: list[str] = field(default_factory=list)
+    auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list)
+
+
 @dataclass
 class TextCaseResult:
     turns: list[RawTurnResult]
     transcript: list[dict[str, str]]
+    startup: TextStartupResult = field(default_factory=TextStartupResult)
+
+
+TurnCompleteCallback = Callable[[RawTurnResult], Awaitable[None]]
+StartupCompleteCallback = Callable[[TextStartupResult], Awaitable[None]]
+
+
+@dataclass
+class _StartupBoundaryFrame(DataFrame):
+    """Ordered marker queued behind every frame emitted by startup."""
+
+    reached: asyncio.Event
 
 
 @dataclass
@@ -139,6 +161,26 @@ class _TextCaptureProcessor(FrameProcessor):
         self.error: TestExecutionError | None = None
         self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None
 
+    def fail(self, frame: ErrorFrame) -> None:
+        """Release every wait even when Pipecat reports the error upstream."""
+
+        message = frame.error or "Pipeline 执行失败"
+        unmocked_tool = "UNMOCKED_TOOL_CALL" in message
+        self.error = TestExecutionError(
+            code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR",
+            message=message,
+            stage=(
+                "tool"
+                if unmocked_tool
+                else "model"
+                if frame.processor
+                else "pipeline"
+            ),
+            retryable=not unmocked_tool,
+        )
+        self.window.pending_tools.clear()
+        self.window.touch()
+
     async def process_frame(self, frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
@@ -196,12 +238,9 @@ class _TextCaptureProcessor(FrameProcessor):
             self.window.pending_tools.discard(call.id)
             self.window.touch()
         elif isinstance(frame, ErrorFrame):
-            self.error = TestExecutionError(
+            self.fail(frame)
-                code="PIPELINE_ERROR",
+        elif isinstance(frame, _StartupBoundaryFrame):
-                message=frame.error or "Pipeline 执行失败",
+            frame.reached.set()
-                stage="model" if frame.processor else "pipeline",
-                retryable=True,
-            )
             self.window.touch()
         elif isinstance(frame, (EndFrame, CancelFrame)):
             self.window.touch()
@@ -281,6 +320,41 @@ async def _wait_for_quiescence(
         await asyncio.sleep(0.05)
 
 
+async def _wait_for_startup(
+    worker: PipelineWorker,
+    capture: _TextCaptureProcessor,
+    runner_task: asyncio.Task[None],
+    activity_marker: int,
+) -> None:
+    """Wait until startup-queued frames and any entry LLM reply are complete."""
+
+    boundary = asyncio.Event()
+    await worker.queue_frame(_StartupBoundaryFrame(reached=boundary))
+    boundary_waiter = asyncio.create_task(boundary.wait())
+    done, _pending = await asyncio.wait(
+        {boundary_waiter, runner_task},
+        timeout=WORKER_START_TIMEOUT_SECONDS,
+        return_when=asyncio.FIRST_COMPLETED,
+    )
+    if not done:
+        boundary_waiter.cancel()
+        raise TestExecutionError(
+            code="STARTUP_DRAIN_TIMEOUT",
+            message="开场阶段未能进入稳定状态",
+            stage="pipeline",
+            retryable=True,
+        )
+    if boundary_waiter not in done:
+        boundary_waiter.cancel()
+    await asyncio.gather(boundary_waiter, return_exceptions=True)
+    await _wait_for_quiescence(
+        capture,
+        runner_task,
+        activity_marker,
+        require_activity=False,
+    )
+
+
 class TextPipelineRunner:
     """Execute all fixed input turns in one isolated, transportless pipeline."""
 
@@ -290,10 +364,17 @@ class TextPipelineRunner:
         definition: TestCaseDefinition,
         *,
         timeout_seconds: int,
+        on_turn_complete: TurnCompleteCallback | None = None,
+        on_startup_complete: StartupCompleteCallback | None = None,
     ) -> TextCaseResult:
         try:
             async with asyncio.timeout(timeout_seconds):
-                return await self._run(cfg, definition)
+                return await self._run(
+                    cfg,
+                    definition,
+                    on_turn_complete=on_turn_complete,
+                    on_startup_complete=on_startup_complete,
+                )
         except TimeoutError as exc:
             raise TestExecutionError(
                 code="PIPELINE_TIMEOUT",
@@ -306,6 +387,9 @@ class TextPipelineRunner:
         self,
         cfg: AssistantConfig,
         definition: TestCaseDefinition,
+        *,
+        on_turn_complete: TurnCompleteCallback | None,
+        on_startup_complete: StartupCompleteCallback | None,
     ) -> TextCaseResult:
         if cfg.runtimeMode != "pipeline":
             raise TestExecutionError(
@@ -396,7 +480,12 @@ class TextPipelineRunner:
         async def on_pipeline_started(_worker, _frame):
             worker_started.set()
 
+        @worker.event_handler("on_pipeline_error")
+        async def on_pipeline_error(_worker, frame):
+            capture.fail(frame)
+
         executor_holder: dict[str, MockToolExecutor] = {}
+        text_client_tools = TextClientToolPort()
 
         def create_test_executor(store) -> MockToolExecutor:
             executor = MockToolExecutor(store)
@@ -520,7 +609,7 @@ class TextPipelineRunner:
                 set_tools=set_tools,
                 call_end=call_end,
                 session_id=f"test_{uuid4().hex}",
-                client_tools=TextClientToolPort(),
+                client_tools=text_client_tools,
                 worker=worker,
                 context_aggregator=aggregators,
                 transport=None,
@@ -536,6 +625,13 @@ class TextPipelineRunner:
         )
         executor = executor_holder["executor"]
 
+        # A transportless text run cannot synthesize audio, but the configured
+        # greeting has already completed before scripted inputs begin. Mirror
+        # production's provider-safe context fact so the model does not repeat it.
+        greeting = str(await brain.greeting(cfg) or "").strip()
+        if greeting and brain.spec.owns_context:
+            brain.prepare_greeting_context(greeting, context)
+
         @assistant_aggregator.event_handler("on_assistant_turn_started")
         async def on_assistant_turn_started(_aggregator):
             capture.window.active_assistant += 1
@@ -591,12 +687,21 @@ class TextPipelineRunner:
             marker = capture.window.activity_count
             await brain.on_connected(greeting_pending=False)
             await brain.on_client_ready()
-            await _wait_for_quiescence(
+            await _wait_for_startup(
+                worker,
                 capture,
                 runner_task,
                 marker,
-                require_activity=False,
             )
+            startup = TextStartupResult(
+                greeting=greeting,
+                assistant_outputs=list(capture.outputs),
+                auto_confirmed_messages=list(
+                    text_client_tools.auto_confirmed_messages
+                ),
+            )
+            if on_startup_complete is not None:
+                await on_startup_complete(startup)
             # Startup messages initialize the session but are not a scripted
             # user turn result in the fixed-input editor contract.
             capture.outputs.clear()
@@ -620,6 +725,8 @@ class TextPipelineRunner:
                     index,
                 )
                 results.append(result)
+                if on_turn_complete is not None:
+                    await on_turn_complete(result)
 
             transcript = [
                 {
@@ -630,7 +737,11 @@ class TextPipelineRunner:
                 if message.get("role") in {"user", "assistant"}
                 and str(message.get("content") or "").strip()
             ]
-            return TextCaseResult(turns=results, transcript=transcript)
+            return TextCaseResult(
+                turns=results,
+                transcript=transcript,
+                startup=startup,
+            )
         except TestExecutionError:
             raise
         except asyncio.CancelledError:

backend/test_schemas.py

@@ -316,6 +316,10 @@ class BatchRunCreate(CamelModel):
     assistant_id: str = Field(min_length=1, max_length=40)
     evaluator_model_resource_id: str = Field(min_length=1, max_length=40)
     case_ids: list[str] = Field(min_length=1, max_length=1_000)
+    dynamic_variables: dict[str, str | int | float | bool] = Field(
+        default_factory=dict,
+        max_length=50,
+    )
     config: BatchRunConfig = Field(default_factory=BatchRunConfig)
     title: str | None = Field(default=None, max_length=256)
 
@@ -363,11 +367,19 @@ class BatchTurnResult(CamelModel):
     evaluations: list[BatchEvaluationResult] = Field(default_factory=list)
 
 
+class BatchStartupResult(CamelModel):
+    completed: bool = False
+    greeting: str = ""
+    assistant_outputs: list[str] = Field(default_factory=list)
+    auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list)
+
+
 class BatchRunCaseOut(CamelModel):
     id: str
     name: str
     status: CaseStatus
     turns: list[BatchTurnResult] = Field(default_factory=list)
+    startup: BatchStartupResult = Field(default_factory=BatchStartupResult)
     overall_criteria: list[BatchEvaluationResult] = Field(default_factory=list)
     attempt_count: int
     max_attempts: int

backend/tests/test_debug_auto_test.py

@@ -0,0 +1,264 @@
+import asyncio
+import unittest
+
+from models import AssistantConfig, RuntimeTool
+from pipecat.frames.frames import ErrorFrame
+from pipecat.processors.aggregators.llm_context import LLMContext
+from services.brains import BrainRuntime, build_brain
+from services.brains.base import GREETING_CONTEXT_MARKER
+from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
+from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
+from services.test_runs.text_runner import (
+    _TextCaptureProcessor,
+    _wait_for_startup,
+    _wait_for_quiescence,
+)
+from test_schemas import BatchRunCreate
+
+
+class FakeLLM:
+    def __init__(self):
+        self.functions = {}
+
+    def register_function(self, name, handler, **_options):
+        self.functions[name] = handler
+
+
+class FakeCallEnd:
+    def __init__(self):
+        self.ending = False
+        self.reason = ""
+
+    @property
+    def accepts_input(self):
+        return not self.ending
+
+    def begin(self, reason, *, interruptible=False):
+        self.ending = True
+        self.reason = reason
+
+    def cancel_pending_end(self):
+        return False
+
+    def begin_response(self):
+        return None
+
+    def arm_after_speech(self):
+        return None
+
+    def track_speech(self):
+        return None
+
+    async def arm_after_tracked_speech(self):
+        return None
+
+    async def finish_after_current_speech(self, *, has_text):
+        return None
+
+    async def finish(self):
+        return None
+
+
+class FakeFunctionParams:
+    def __init__(self, arguments=None):
+        self.arguments = arguments or {}
+        self.result = None
+        self.properties = None
+
+    async def result_callback(self, result, properties=None):
+        self.result = result
+        self.properties = properties
+
+
+def system_runtime_tool(kind: str) -> RuntimeTool:
+    return RuntimeTool(
+        id=f"tool_{kind}",
+        name=kind,
+        function_name=kind,
+        type="system",
+        definition={"config": {"kind": kind}},
+    )
+
+
+class DebugAutoTestContractTests(unittest.TestCase):
+    def test_batch_run_accepts_camel_case_dynamic_variables(self):
+        request = BatchRunCreate.model_validate(
+            {
+                "assistantId": "asst_1",
+                "evaluatorModelResourceId": "model_1",
+                "caseIds": ["case_1"],
+                "dynamicVariables": {"customer_name": "测试用户", "count": 2},
+            }
+        )
+
+        self.assertEqual(
+            request.dynamic_variables,
+            {"customer_name": "测试用户", "count": 2},
+        )
+
+    def test_each_test_case_gets_fresh_conversation_variables(self):
+        config = AssistantConfig(
+            type="prompt",
+            prompt="你好，{{ customer_name }}",
+            dynamic_variable_definitions={
+                "customer_name": {
+                    "type": "string",
+                    "required": True,
+                    "default": None,
+                }
+            },
+        )
+
+        first = prepare_dynamic_config(
+            config,
+            {"customer_name": "测试用户"},
+            assistant_id="asst_1",
+        )
+        second = prepare_dynamic_config(
+            config,
+            {"customer_name": "测试用户"},
+            assistant_id="asst_1",
+        )
+
+        self.assertNotEqual(first.conversation_id, second.conversation_id)
+        self.assertEqual(
+            DynamicVariableStore.from_config(first).render(first.prompt),
+            "你好，测试用户",
+        )
+
+    def test_text_client_port_records_automatic_message_confirmation(self):
+        port = TextClientToolPort()
+
+        result = asyncio.run(
+            port.call(
+                "show_message",
+                {"title": "重要提示", "message": "请确认"},
+            )
+        )
+
+        self.assertEqual(result["data"]["action"], "confirmed")
+        self.assertEqual(
+            port.auto_confirmed_messages,
+            [{"title": "重要提示", "message": "请确认"}],
+        )
+
+
+class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
+    async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
+        cfg = AssistantConfig(
+            type="prompt",
+            tools=[system_runtime_tool("end_conversation")],
+        )
+        brain = build_brain(cfg)
+        llm = FakeLLM()
+        call_end = FakeCallEnd()
+        executor_holder = {}
+
+        def create_executor(store):
+            executor = MockToolExecutor(store)
+            executor_holder["executor"] = executor
+            return executor
+
+        await brain.setup(
+            cfg,
+            BrainRuntime(
+                context=LLMContext(messages=[]),
+                llm=llm,
+                queue_frame=lambda _frame: asyncio.sleep(0),
+                set_system_prompt=lambda _prompt: None,
+                set_tools=lambda _tools: None,
+                call_end=call_end,
+                tool_executor_factory=create_executor,
+            ),
+        )
+
+        params = FakeFunctionParams({"reason": "测试完成"})
+        await llm.functions["end_conversation"](params)
+
+        self.assertTrue(call_end.ending)
+        self.assertEqual(call_end.reason, "测试完成")
+        self.assertEqual(params.result["action"], "ending_call")
+        self.assertEqual(executor_holder["executor"].unmocked_calls, [])
+
+    async def test_startup_waits_for_queued_entry_reply(self):
+        capture = _TextCaptureProcessor(LLMContext(messages=[]))
+
+        class FakeWorker:
+            async def queue_frame(self, frame):
+                async def process_startup_queue():
+                    capture.window.active_llm += 1
+                    capture.window.touch()
+                    await asyncio.sleep(0.01)
+                    capture.outputs.append("开场后的自动首句")
+                    capture.window.active_llm -= 1
+                    capture.window.touch()
+                    frame.reached.set()
+
+                asyncio.create_task(process_startup_queue())
+
+        runner_task = asyncio.create_task(asyncio.sleep(60))
+        try:
+            await asyncio.wait_for(
+                _wait_for_startup(
+                    FakeWorker(),
+                    capture,
+                    runner_task,
+                    capture.window.activity_count,
+                ),
+                timeout=1,
+            )
+            self.assertEqual(capture.outputs, ["开场后的自动首句"])
+        finally:
+            runner_task.cancel()
+            await asyncio.gather(runner_task, return_exceptions=True)
+
+    async def test_text_startup_can_record_greeting_as_already_played(self):
+        cfg = AssistantConfig(
+            type="prompt",
+            greeting="您好，这是一段开场白。",
+            prompt="不要重复开场白。",
+        )
+        brain = build_brain(cfg)
+        context = LLMContext(
+            messages=[{"role": "system", "content": cfg.prompt}]
+        )
+
+        greeting = await brain.greeting(cfg)
+        brain.prepare_greeting_context(greeting, context)
+
+        self.assertIn(
+            GREETING_CONTEXT_MARKER,
+            context.get_messages()[1]["content"],
+        )
+        self.assertIn(cfg.greeting, context.get_messages()[1]["content"])
+
+    async def test_upstream_pipeline_error_releases_pending_tool_wait(self):
+        capture = _TextCaptureProcessor(LLMContext(messages=[]))
+        capture.window.pending_tools.add("call_1")
+        runner_task = asyncio.create_task(asyncio.sleep(60))
+        try:
+            capture.fail(
+                ErrorFrame(
+                    error=(
+                        "UNMOCKED_TOOL_CALL: 工具 external_lookup "
+                        "未配置 Mock 返回值"
+                    )
+                )
+            )
+
+            with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"):
+                await asyncio.wait_for(
+                    _wait_for_quiescence(capture, runner_task, 0),
+                    timeout=0.1,
+                )
+            self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL")
+            self.assertEqual(capture.error.stage, "tool")
+            self.assertFalse(capture.error.retryable)
+            self.assertEqual(capture.window.pending_tools, set())
+        finally:
+            runner_task.cancel()
+            await asyncio.gather(runner_task, return_exceptions=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

frontend/src/components/assistant-editor/debug-preview.tsx

@@ -183,8 +183,6 @@ export function DebugDrawer({
 }) {
   const preview = useVoicePreview(assistantId, onNodeActive);
   const camera = useCameraPreview();
-  const autoTest = useAutoTestRunner();
-  const stopAutoTest = autoTest.stop;
   const [debugMode, setDebugMode] = useState<DebugMode>("manual");
   const [showTranscript, setShowTranscript] = useState(false);
   const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
@@ -220,6 +218,14 @@ export function DebugDrawer({
     }
     resolvedDynamicVariables[name] = value;
   }
+  const autoTest = useAutoTestRunner({
+    assistantId,
+    enabled: debugMode === "auto",
+    hasUnsavedChanges,
+    dynamicVariables: resolvedDynamicVariables,
+    dynamicVariablesError,
+  });
+  const stopAutoTest = autoTest.stop;
 
   const selectCamera = useCallback(
     async (deviceId: string) => {
@@ -369,6 +375,9 @@ function AutoTestPanel({
     start,
     stop,
     rerun,
+    suites,
+    cases,
+    catalogLoading,
   } = autoTest;
 
   return (
@@ -379,6 +388,10 @@ function AutoTestPanel({
       <div className="shrink-0 border-t border-hairline bg-card p-3">
         <AutoTestRunBar
           state={state}
+          testCase={selectedCase}
+          suites={suites}
+          cases={cases}
+          loading={catalogLoading}
           onSelectCase={selectCase}
           onClearCase={clearCase}
           onStart={start}

frontend/src/data/batch-run.ts

@@ -59,11 +59,23 @@ export type BatchTurnResult = {
   evaluations: BatchEvaluationResult[];
 };
 
+export type BatchStartupResult = {
+  completed: boolean;
+  greeting: string;
+  assistantOutputs: string[];
+  autoConfirmedMessages: Array<{
+    title?: string;
+    message?: string;
+    actions?: Array<{ id?: string; label?: string }>;
+  }>;
+};
+
 export type BatchRunCase = {
   id: string;
   name: string;
   status: BatchCaseStatus;
   turns: BatchTurnResult[];
+  startup?: BatchStartupResult;
   overallCriteria: BatchEvaluationResult[];
   attemptCount: number;
   maxAttempts: number;

frontend/src/lib/api.ts

@@ -851,6 +851,7 @@ export const batchRunsApi = {
     assistantId: string;
     evaluatorModelResourceId: string;
     caseIds: string[];
+    dynamicVariables?: Record<string, string | number | boolean>;
     title?: string;
     config: {
       concurrency: number;