import asyncio import unittest from models import AssistantConfig, RuntimeTool from pipecat.frames.frames import ErrorFrame from pipecat.processors.aggregators.llm_context import LLMContext from services.brains import BrainRuntime, build_brain from services.brains.base import GREETING_CONTEXT_MARKER from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort from services.test_runs.text_runner import ( _TextCaptureProcessor, _opening_speech_from_message, _resolve_startup_greeting, _runtime_record_from_message, _wait_for_startup, _wait_for_quiescence, ) from test_schemas import BatchRunCreate, BatchStartupResult class FakeLLM: def __init__(self): self.functions = {} def register_function(self, name, handler, **_options): self.functions[name] = handler class FakeCallEnd: def __init__(self): self.ending = False self.reason = "" @property def accepts_input(self): return not self.ending def begin(self, reason, *, interruptible=False): self.ending = True self.reason = reason def cancel_pending_end(self): return False def begin_response(self): return None def arm_after_speech(self): return None def track_speech(self): return None async def arm_after_tracked_speech(self): return None async def finish_after_current_speech(self, *, has_text): return None async def finish(self): return None class FakeFunctionParams: def __init__(self, arguments=None): self.arguments = arguments or {} self.result = None self.properties = None async def result_callback(self, result, properties=None): self.result = result self.properties = properties def system_runtime_tool(kind: str) -> RuntimeTool: return RuntimeTool( id=f"tool_{kind}", name=kind, function_name=kind, type="system", definition={"config": {"kind": kind}}, ) class DebugAutoTestContractTests(unittest.TestCase): def test_batch_run_accepts_camel_case_dynamic_variables(self): request = BatchRunCreate.model_validate( { "assistantId": "asst_1", "evaluatorModelResourceId": "model_1", "caseIds": ["case_1"], "dynamicVariables": {"customer_name": "测试用户", "count": 2}, } ) self.assertEqual( request.dynamic_variables, {"customer_name": "测试用户", "count": 2}, ) def test_each_test_case_gets_fresh_conversation_variables(self): config = AssistantConfig( type="prompt", prompt="你好，{{ customer_name }}", dynamic_variable_definitions={ "customer_name": { "type": "string", "required": True, "default": None, } }, ) first = prepare_dynamic_config( config, {"customer_name": "测试用户"}, assistant_id="asst_1", ) second = prepare_dynamic_config( config, {"customer_name": "测试用户"}, assistant_id="asst_1", ) self.assertNotEqual(first.conversation_id, second.conversation_id) self.assertEqual( DynamicVariableStore.from_config(first).render(first.prompt), "你好，测试用户", ) def test_startup_result_exposes_real_runtime_records(self): result = BatchStartupResult.model_validate( { "completed": True, "runtimeRecords": [ { "type": "workflow-event", "eventId": "wfe_opening", "event": "message_started", } ], } ) self.assertEqual( result.runtime_records[0]["event"], "message_started", ) def test_text_client_port_records_automatic_message_confirmation(self): port = TextClientToolPort() result = asyncio.run( port.call( "show_message", {"title": "重要提示", "message": "请确认"}, ) ) self.assertEqual(result["data"]["action"], "confirmed") self.assertEqual( port.auto_confirmed_messages, [{"title": "重要提示", "message": "请确认"}], ) def test_confirmation_owned_speech_is_kept_as_startup_greeting(self): self.assertEqual( _resolve_startup_greeting( "", ["您好，请先阅读弹窗。"], ), "您好，请先阅读弹窗。", ) self.assertEqual( _resolve_startup_greeting( "普通开场白", ["其他固定播报"], ), "普通开场白", ) self.assertEqual( _resolve_startup_greeting( "", [], ), "", ) def test_only_prompt_opening_transcript_is_identified_as_greeting(self): self.assertEqual( _opening_speech_from_message( { "type": "transcript", "content": "您好，请先阅读弹窗。", "source": "prompt-opening-speech", } ), "您好，请先阅读弹窗。", ) self.assertEqual( _opening_speech_from_message( { "type": "transcript", "content": "普通 Workflow 播报", "source": "workflow-message-speech", } ), "", ) class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase): async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self): cfg = AssistantConfig( type="prompt", built_in_tools={ "end_conversation": { "description": "结束对话", "capture_reason": True, } }, ) brain = build_brain(cfg) llm = FakeLLM() call_end = FakeCallEnd() executor_holder = {} def create_executor(store): executor = MockToolExecutor(store) executor_holder["executor"] = executor return executor await brain.setup( cfg, BrainRuntime( context=LLMContext(messages=[]), llm=llm, queue_frame=lambda _frame: asyncio.sleep(0), set_system_prompt=lambda _prompt: None, set_tools=lambda _tools: None, call_end=call_end, tool_executor_factory=create_executor, ), ) params = FakeFunctionParams({"reason": "测试完成"}) await llm.functions["end_conversation"](params) self.assertTrue(call_end.ending) self.assertEqual(call_end.reason, "测试完成") self.assertEqual(params.result["action"], "ending_call") self.assertEqual(executor_holder["executor"].unmocked_calls, []) async def test_text_capture_keeps_runtime_records_for_auto_test_ui(self): event = { "type": "workflow-event", "eventId": "wfe_end", "event": "conversation_end_requested", "reason": "测试完成", } self.assertEqual(_runtime_record_from_message(event), event) self.assertIsNone( _runtime_record_from_message({"type": "call-end-state"}) ) self.assertIsNone( _runtime_record_from_message( { "type": "workflow-event", "eventId": "wfe_internal", "event": "client_tool_completed", "internalStage": True, } ) ) async def test_startup_waits_for_queued_entry_reply(self): capture = _TextCaptureProcessor(LLMContext(messages=[])) class FakeWorker: async def queue_frame(self, frame): async def process_startup_queue(): capture.window.active_llm += 1 capture.window.touch() await asyncio.sleep(0.01) capture.outputs.append("开场后的自动首句") capture.window.active_llm -= 1 capture.window.touch() frame.reached.set() asyncio.create_task(process_startup_queue()) runner_task = asyncio.create_task(asyncio.sleep(60)) try: await asyncio.wait_for( _wait_for_startup( FakeWorker(), capture, runner_task, capture.window.activity_count, ), timeout=1, ) self.assertEqual(capture.outputs, ["开场后的自动首句"]) finally: runner_task.cancel() await asyncio.gather(runner_task, return_exceptions=True) async def test_text_startup_can_record_greeting_as_already_played(self): cfg = AssistantConfig( type="prompt", greeting="您好，这是一段开场白。", prompt="不要重复开场白。", ) brain = build_brain(cfg) context = LLMContext( messages=[{"role": "system", "content": cfg.prompt}] ) greeting = await brain.greeting(cfg) brain.prepare_greeting_context(greeting, context) self.assertIn( GREETING_CONTEXT_MARKER, context.get_messages()[1]["content"], ) self.assertIn(cfg.greeting, context.get_messages()[1]["content"]) async def test_upstream_pipeline_error_releases_pending_tool_wait(self): capture = _TextCaptureProcessor(LLMContext(messages=[])) capture.window.pending_tools.add("call_1") runner_task = asyncio.create_task(asyncio.sleep(60)) try: capture.fail( ErrorFrame( error=( "UNMOCKED_TOOL_CALL: 工具 external_lookup " "未配置 Mock 返回值" ) ) ) with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"): await asyncio.wait_for( _wait_for_quiescence(capture, runner_task, 0), timeout=0.1, ) self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL") self.assertEqual(capture.error.stage, "tool") self.assertFalse(capture.error.retryable) self.assertEqual(capture.window.pending_tools, set()) finally: runner_task.cancel() await asyncio.gather(runner_task, return_exceptions=True) if __name__ == "__main__": unittest.main()