366 lines
11 KiB
Python
366 lines
11 KiB
Python
import asyncio
|
|
import unittest
|
|
|
|
from models import AssistantConfig, RuntimeTool
|
|
from pipecat.frames.frames import ErrorFrame
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from services.brains import BrainRuntime, build_brain
|
|
from services.brains.base import GREETING_CONTEXT_MARKER
|
|
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
|
|
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
|
|
from services.test_runs.text_runner import (
|
|
_TextCaptureProcessor,
|
|
_opening_speech_from_message,
|
|
_resolve_startup_greeting,
|
|
_runtime_record_from_message,
|
|
_wait_for_startup,
|
|
_wait_for_quiescence,
|
|
)
|
|
from test_schemas import BatchRunCreate, BatchStartupResult
|
|
|
|
|
|
class FakeLLM:
|
|
def __init__(self):
|
|
self.functions = {}
|
|
|
|
def register_function(self, name, handler, **_options):
|
|
self.functions[name] = handler
|
|
|
|
|
|
class FakeCallEnd:
|
|
def __init__(self):
|
|
self.ending = False
|
|
self.reason = ""
|
|
|
|
@property
|
|
def accepts_input(self):
|
|
return not self.ending
|
|
|
|
def begin(self, reason, *, interruptible=False):
|
|
self.ending = True
|
|
self.reason = reason
|
|
|
|
def cancel_pending_end(self):
|
|
return False
|
|
|
|
def begin_response(self):
|
|
return None
|
|
|
|
def arm_after_speech(self):
|
|
return None
|
|
|
|
def track_speech(self):
|
|
return None
|
|
|
|
async def arm_after_tracked_speech(self):
|
|
return None
|
|
|
|
async def finish_after_current_speech(self, *, has_text):
|
|
return None
|
|
|
|
async def finish(self):
|
|
return None
|
|
|
|
|
|
class FakeFunctionParams:
|
|
def __init__(self, arguments=None):
|
|
self.arguments = arguments or {}
|
|
self.result = None
|
|
self.properties = None
|
|
|
|
async def result_callback(self, result, properties=None):
|
|
self.result = result
|
|
self.properties = properties
|
|
|
|
|
|
def system_runtime_tool(kind: str) -> RuntimeTool:
|
|
return RuntimeTool(
|
|
id=f"tool_{kind}",
|
|
name=kind,
|
|
function_name=kind,
|
|
type="system",
|
|
definition={
|
|
"config": {
|
|
"kind": kind,
|
|
"allow_interruptions": True,
|
|
"execution_mode": "immediate",
|
|
}
|
|
},
|
|
)
|
|
|
|
|
|
class DebugAutoTestContractTests(unittest.TestCase):
|
|
def test_batch_run_accepts_camel_case_dynamic_variables(self):
|
|
request = BatchRunCreate.model_validate(
|
|
{
|
|
"assistantId": "asst_1",
|
|
"evaluatorModelResourceId": "model_1",
|
|
"caseIds": ["case_1"],
|
|
"dynamicVariables": {"customer_name": "测试用户", "count": 2},
|
|
}
|
|
)
|
|
|
|
self.assertEqual(
|
|
request.dynamic_variables,
|
|
{"customer_name": "测试用户", "count": 2},
|
|
)
|
|
|
|
def test_each_test_case_gets_fresh_conversation_variables(self):
|
|
config = AssistantConfig(
|
|
type="prompt",
|
|
prompt="你好，{{ customer_name }}",
|
|
dynamic_variable_definitions={
|
|
"customer_name": {
|
|
"type": "string",
|
|
"required": True,
|
|
"default": None,
|
|
}
|
|
},
|
|
)
|
|
|
|
first = prepare_dynamic_config(
|
|
config,
|
|
{"customer_name": "测试用户"},
|
|
assistant_id="asst_1",
|
|
)
|
|
second = prepare_dynamic_config(
|
|
config,
|
|
{"customer_name": "测试用户"},
|
|
assistant_id="asst_1",
|
|
)
|
|
|
|
self.assertNotEqual(first.conversation_id, second.conversation_id)
|
|
self.assertEqual(
|
|
DynamicVariableStore.from_config(first).render(first.prompt),
|
|
"你好，测试用户",
|
|
)
|
|
|
|
def test_startup_result_exposes_real_runtime_records(self):
|
|
result = BatchStartupResult.model_validate(
|
|
{
|
|
"completed": True,
|
|
"runtimeRecords": [
|
|
{
|
|
"type": "workflow-event",
|
|
"eventId": "wfe_opening",
|
|
"event": "message_started",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(
|
|
result.runtime_records[0]["event"],
|
|
"message_started",
|
|
)
|
|
|
|
def test_text_client_port_records_automatic_message_confirmation(self):
|
|
port = TextClientToolPort()
|
|
|
|
result = asyncio.run(
|
|
port.call(
|
|
"show_message",
|
|
{"title": "重要提示", "message": "请确认"},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(result["data"]["action"], "confirmed")
|
|
self.assertEqual(
|
|
port.auto_confirmed_messages,
|
|
[{"title": "重要提示", "message": "请确认"}],
|
|
)
|
|
|
|
def test_confirmation_owned_speech_is_kept_as_startup_greeting(self):
|
|
self.assertEqual(
|
|
_resolve_startup_greeting(
|
|
"",
|
|
["您好，请先阅读弹窗。"],
|
|
),
|
|
"您好，请先阅读弹窗。",
|
|
)
|
|
self.assertEqual(
|
|
_resolve_startup_greeting(
|
|
"普通开场白",
|
|
["其他固定播报"],
|
|
),
|
|
"普通开场白",
|
|
)
|
|
self.assertEqual(
|
|
_resolve_startup_greeting(
|
|
"",
|
|
[],
|
|
),
|
|
"",
|
|
)
|
|
|
|
def test_only_prompt_opening_transcript_is_identified_as_greeting(self):
|
|
self.assertEqual(
|
|
_opening_speech_from_message(
|
|
{
|
|
"type": "transcript",
|
|
"content": "您好，请先阅读弹窗。",
|
|
"source": "prompt-opening-speech",
|
|
}
|
|
),
|
|
"您好，请先阅读弹窗。",
|
|
)
|
|
self.assertEqual(
|
|
_opening_speech_from_message(
|
|
{
|
|
"type": "transcript",
|
|
"content": "普通 Workflow 播报",
|
|
"source": "workflow-message-speech",
|
|
}
|
|
),
|
|
"",
|
|
)
|
|
|
|
|
|
class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
built_in_tools={
|
|
"end_conversation": {
|
|
"description": "结束对话",
|
|
"capture_reason": True,
|
|
}
|
|
},
|
|
)
|
|
brain = build_brain(cfg)
|
|
llm = FakeLLM()
|
|
call_end = FakeCallEnd()
|
|
executor_holder = {}
|
|
|
|
def create_executor(store):
|
|
executor = MockToolExecutor(store)
|
|
executor_holder["executor"] = executor
|
|
return executor
|
|
|
|
await brain.setup(
|
|
cfg,
|
|
BrainRuntime(
|
|
context=LLMContext(messages=[]),
|
|
llm=llm,
|
|
queue_frame=lambda _frame: asyncio.sleep(0),
|
|
set_system_prompt=lambda _prompt: None,
|
|
set_tools=lambda _tools: None,
|
|
call_end=call_end,
|
|
tool_executor_factory=create_executor,
|
|
),
|
|
)
|
|
|
|
params = FakeFunctionParams({"reason": "测试完成"})
|
|
await llm.functions["end_conversation"](params)
|
|
|
|
self.assertTrue(call_end.ending)
|
|
self.assertEqual(call_end.reason, "测试完成")
|
|
self.assertEqual(params.result["action"], "ending_call")
|
|
self.assertEqual(executor_holder["executor"].unmocked_calls, [])
|
|
|
|
async def test_text_capture_keeps_runtime_records_for_auto_test_ui(self):
|
|
event = {
|
|
"type": "workflow-event",
|
|
"eventId": "wfe_end",
|
|
"event": "conversation_end_requested",
|
|
"reason": "测试完成",
|
|
}
|
|
|
|
self.assertEqual(_runtime_record_from_message(event), event)
|
|
self.assertIsNone(
|
|
_runtime_record_from_message({"type": "call-end-state"})
|
|
)
|
|
self.assertIsNone(
|
|
_runtime_record_from_message(
|
|
{
|
|
"type": "workflow-event",
|
|
"eventId": "wfe_internal",
|
|
"event": "client_tool_completed",
|
|
"internalStage": True,
|
|
}
|
|
)
|
|
)
|
|
|
|
async def test_startup_waits_for_queued_entry_reply(self):
|
|
capture = _TextCaptureProcessor(LLMContext(messages=[]))
|
|
|
|
class FakeWorker:
|
|
async def queue_frame(self, frame):
|
|
async def process_startup_queue():
|
|
capture.window.active_llm += 1
|
|
capture.window.touch()
|
|
await asyncio.sleep(0.01)
|
|
capture.outputs.append("开场后的自动首句")
|
|
capture.window.active_llm -= 1
|
|
capture.window.touch()
|
|
frame.reached.set()
|
|
|
|
asyncio.create_task(process_startup_queue())
|
|
|
|
runner_task = asyncio.create_task(asyncio.sleep(60))
|
|
try:
|
|
await asyncio.wait_for(
|
|
_wait_for_startup(
|
|
FakeWorker(),
|
|
capture,
|
|
runner_task,
|
|
capture.window.activity_count,
|
|
),
|
|
timeout=1,
|
|
)
|
|
self.assertEqual(capture.outputs, ["开场后的自动首句"])
|
|
finally:
|
|
runner_task.cancel()
|
|
await asyncio.gather(runner_task, return_exceptions=True)
|
|
|
|
async def test_text_startup_can_record_greeting_as_already_played(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
greeting="您好，这是一段开场白。",
|
|
prompt="不要重复开场白。",
|
|
)
|
|
brain = build_brain(cfg)
|
|
context = LLMContext(
|
|
messages=[{"role": "system", "content": cfg.prompt}]
|
|
)
|
|
|
|
greeting = await brain.greeting(cfg)
|
|
brain.prepare_greeting_context(greeting, context)
|
|
|
|
self.assertIn(
|
|
GREETING_CONTEXT_MARKER,
|
|
context.get_messages()[1]["content"],
|
|
)
|
|
self.assertIn(cfg.greeting, context.get_messages()[1]["content"])
|
|
|
|
async def test_upstream_pipeline_error_releases_pending_tool_wait(self):
|
|
capture = _TextCaptureProcessor(LLMContext(messages=[]))
|
|
capture.window.pending_tools.add("call_1")
|
|
runner_task = asyncio.create_task(asyncio.sleep(60))
|
|
try:
|
|
capture.fail(
|
|
ErrorFrame(
|
|
error=(
|
|
"UNMOCKED_TOOL_CALL: 工具 external_lookup "
|
|
"未配置 Mock 返回值"
|
|
)
|
|
)
|
|
)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"):
|
|
await asyncio.wait_for(
|
|
_wait_for_quiescence(capture, runner_task, 0),
|
|
timeout=0.1,
|
|
)
|
|
self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL")
|
|
self.assertEqual(capture.error.stage, "tool")
|
|
self.assertFalse(capture.error.retryable)
|
|
self.assertEqual(capture.window.pending_tools, set())
|
|
finally:
|
|
runner_task.cancel()
|
|
await asyncio.gather(runner_task, return_exceptions=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|