ai-video-fullstack/backend/tests/test_debug_auto_test.py

import asyncio
import unittest

from models import AssistantConfig, RuntimeTool
from pipecat.frames.frames import ErrorFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from services.brains import BrainRuntime, build_brain
from services.brains.base import GREETING_CONTEXT_MARKER
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
from services.test_runs.mock_tools import (
    MockToolExecutor,
    TextClientToolPort,
    ToolSimulationResolver,
)
from services.test_runs.text_runner import (
    _TextCaptureProcessor,
    _opening_speech_from_message,
    _resolve_startup_greeting,
    _runtime_record_from_message,
    _wait_for_startup,
    _wait_for_quiescence,
)
from test_schemas import BatchRunCreate, BatchStartupResult, FixedToolSimulation


class FakeLLM:
    def __init__(self):
        self.functions = {}

    def register_function(self, name, handler, **_options):
        self.functions[name] = handler


class FakeCallEnd:
    def __init__(self):
        self.ending = False
        self.reason = ""

    @property
    def accepts_input(self):
        return not self.ending

    def begin(self, reason, *, interruptible=False):
        self.ending = True
        self.reason = reason

    def cancel_pending_end(self):
        return False

    def begin_response(self):
        return None

    def arm_after_speech(self):
        return None

    def track_speech(self):
        return None

    async def arm_after_tracked_speech(self):
        return None

    async def finish_after_current_speech(self, *, has_text):
        return None

    async def finish(self):
        return None


class FakeFunctionParams:
    def __init__(self, arguments=None):
        self.arguments = arguments or {}
        self.result = None
        self.properties = None

    async def result_callback(self, result, properties=None):
        self.result = result
        self.properties = properties


def system_runtime_tool(kind: str) -> RuntimeTool:
    return RuntimeTool(
        id=f"tool_{kind}",
        name=kind,
        function_name=kind,
        type="system",
        definition={
            "config": {
                "kind": kind,
                "allow_interruptions": True,
                "execution_mode": "immediate",
            }
        },
    )


class DebugAutoTestContractTests(unittest.TestCase):
    def test_batch_run_accepts_camel_case_dynamic_variables(self):
        request = BatchRunCreate.model_validate(
            {
                "assistantId": "asst_1",
                "evaluatorModelResourceId": "model_1",
                "caseIds": ["case_1"],
                "dynamicVariables": {"customer_name": "测试用户", "count": 2},
            }
        )

        self.assertEqual(
            request.dynamic_variables,
            {"customer_name": "测试用户", "count": 2},
        )

    def test_each_test_case_gets_fresh_conversation_variables(self):
        config = AssistantConfig(
            type="prompt",
            prompt="你好，{{ customer_name }}",
            dynamic_variable_definitions={
                "customer_name": {
                    "type": "string",
                    "required": True,
                    "default": None,
                }
            },
        )

        first = prepare_dynamic_config(
            config,
            {"customer_name": "测试用户"},
            assistant_id="asst_1",
        )
        second = prepare_dynamic_config(
            config,
            {"customer_name": "测试用户"},
            assistant_id="asst_1",
        )

        self.assertNotEqual(first.conversation_id, second.conversation_id)
        self.assertEqual(
            DynamicVariableStore.from_config(first).render(first.prompt),
            "你好，测试用户",
        )

    def test_startup_result_exposes_real_runtime_records(self):
        result = BatchStartupResult.model_validate(
            {
                "completed": True,
                "runtimeRecords": [
                    {
                        "type": "workflow-event",
                        "eventId": "wfe_opening",
                        "event": "message_started",
                    }
                ],
            }
        )

        self.assertEqual(
            result.runtime_records[0]["event"],
            "message_started",
        )

    def test_text_client_port_records_configured_message_confirmation(self):
        resolver = ToolSimulationResolver(
            [
                FixedToolSimulation.model_validate(
                    {
                        "toolId": "tool_message",
                        "functionName": "show_message",
                        "mode": "fixed",
                        "response": {
                            "outcome": "success",
                            "body": '{"action":"confirmed"}',
                            "delayMs": 0,
                        },
                    }
                )
            ]
        )
        port = TextClientToolPort(resolver)

        result = asyncio.run(
            port.call(
                "show_message",
                {"title": "重要提示", "message": "请确认"},
            )
        )

        self.assertEqual(result["data"]["action"], "confirmed")
        self.assertEqual(
            port.auto_confirmed_messages,
            [{"title": "重要提示", "message": "请确认"}],
        )

    def test_confirmation_owned_speech_is_kept_as_startup_greeting(self):
        self.assertEqual(
            _resolve_startup_greeting(
                "",
                ["您好，请先阅读弹窗。"],
            ),
            "您好，请先阅读弹窗。",
        )
        self.assertEqual(
            _resolve_startup_greeting(
                "普通开场白",
                ["其他固定播报"],
            ),
            "普通开场白",
        )
        self.assertEqual(
            _resolve_startup_greeting(
                "",
                [],
            ),
            "",
        )

    def test_only_prompt_opening_transcript_is_identified_as_greeting(self):
        self.assertEqual(
            _opening_speech_from_message(
                {
                    "type": "transcript",
                    "content": "您好，请先阅读弹窗。",
                    "source": "prompt-opening-speech",
                }
            ),
            "您好，请先阅读弹窗。",
        )
        self.assertEqual(
            _opening_speech_from_message(
                {
                    "type": "transcript",
                    "content": "普通 Workflow 播报",
                    "source": "workflow-message-speech",
                }
            ),
            "",
        )


class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
    async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
        cfg = AssistantConfig(
            type="prompt",
            built_in_tools={
                "end_conversation": {
                    "description": "结束对话",
                    "capture_reason": True,
                }
            },
        )
        brain = build_brain(cfg)
        llm = FakeLLM()
        call_end = FakeCallEnd()

        def create_executor(store):
            return MockToolExecutor(store, ToolSimulationResolver([]))

        await brain.setup(
            cfg,
            BrainRuntime(
                context=LLMContext(messages=[]),
                llm=llm,
                queue_frame=lambda _frame: asyncio.sleep(0),
                set_system_prompt=lambda _prompt: None,
                set_tools=lambda _tools: None,
                call_end=call_end,
                tool_executor_factory=create_executor,
            ),
        )

        params = FakeFunctionParams({"reason": "测试完成"})
        await llm.functions["end_conversation"](params)

        self.assertTrue(call_end.ending)
        self.assertEqual(call_end.reason, "测试完成")
        self.assertEqual(params.result["action"], "ending_call")

    async def test_text_capture_keeps_runtime_records_for_auto_test_ui(self):
        event = {
            "type": "workflow-event",
            "eventId": "wfe_end",
            "event": "conversation_end_requested",
            "reason": "测试完成",
        }

        self.assertEqual(_runtime_record_from_message(event), event)
        self.assertIsNone(
            _runtime_record_from_message({"type": "call-end-state"})
        )
        self.assertIsNone(
            _runtime_record_from_message(
                {
                    "type": "workflow-event",
                    "eventId": "wfe_internal",
                    "event": "client_tool_completed",
                    "internalStage": True,
                }
            )
        )

    async def test_startup_waits_for_queued_entry_reply(self):
        capture = _TextCaptureProcessor(LLMContext(messages=[]))

        class FakeWorker:
            async def queue_frame(self, frame):
                async def process_startup_queue():
                    capture.window.active_llm += 1
                    capture.window.touch()
                    await asyncio.sleep(0.01)
                    capture.outputs.append("开场后的自动首句")
                    capture.window.active_llm -= 1
                    capture.window.touch()
                    frame.reached.set()

                asyncio.create_task(process_startup_queue())

        runner_task = asyncio.create_task(asyncio.sleep(60))
        try:
            await asyncio.wait_for(
                _wait_for_startup(
                    FakeWorker(),
                    capture,
                    runner_task,
                    capture.window.activity_count,
                ),
                timeout=1,
            )
            self.assertEqual(capture.outputs, ["开场后的自动首句"])
        finally:
            runner_task.cancel()
            await asyncio.gather(runner_task, return_exceptions=True)

    async def test_text_startup_can_record_greeting_as_already_played(self):
        cfg = AssistantConfig(
            type="prompt",
            greeting="您好，这是一段开场白。",
            prompt="不要重复开场白。",
        )
        brain = build_brain(cfg)
        context = LLMContext(
            messages=[{"role": "system", "content": cfg.prompt}]
        )

        greeting = await brain.greeting(cfg)
        brain.prepare_greeting_context(greeting, context)

        self.assertIn(
            GREETING_CONTEXT_MARKER,
            context.get_messages()[1]["content"],
        )
        self.assertIn(cfg.greeting, context.get_messages()[1]["content"])

    async def test_upstream_pipeline_error_releases_pending_tool_wait(self):
        capture = _TextCaptureProcessor(LLMContext(messages=[]))
        capture.window.pending_tools.add("call_1")
        runner_task = asyncio.create_task(asyncio.sleep(60))
        try:
            capture.fail(
                ErrorFrame(
                    error=(
                        "UNMOCKED_TOOL_CALL: 工具 external_lookup "
                        "未配置 Mock 返回值"
                    )
                )
            )

            with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"):
                await asyncio.wait_for(
                    _wait_for_quiescence(capture, runner_task, 0),
                    timeout=0.1,
                )
            self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL")
            self.assertEqual(capture.error.stage, "tool")
            self.assertFalse(capture.error.retryable)
            self.assertEqual(capture.window.pending_tools, set())
        finally:
            runner_task.cancel()
            await asyncio.gather(runner_task, return_exceptions=True)


if __name__ == "__main__":
    unittest.main()