Xin Wang
7560f9a525
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types. - Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages. - Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs. - Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases. - Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
265 lines
8.2 KiB
Python
265 lines
8.2 KiB
Python
import asyncio
|
|
import unittest
|
|
|
|
from models import AssistantConfig, RuntimeTool
|
|
from pipecat.frames.frames import ErrorFrame
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from services.brains import BrainRuntime, build_brain
|
|
from services.brains.base import GREETING_CONTEXT_MARKER
|
|
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
|
|
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
|
|
from services.test_runs.text_runner import (
|
|
_TextCaptureProcessor,
|
|
_wait_for_startup,
|
|
_wait_for_quiescence,
|
|
)
|
|
from test_schemas import BatchRunCreate
|
|
|
|
|
|
class FakeLLM:
|
|
def __init__(self):
|
|
self.functions = {}
|
|
|
|
def register_function(self, name, handler, **_options):
|
|
self.functions[name] = handler
|
|
|
|
|
|
class FakeCallEnd:
|
|
def __init__(self):
|
|
self.ending = False
|
|
self.reason = ""
|
|
|
|
@property
|
|
def accepts_input(self):
|
|
return not self.ending
|
|
|
|
def begin(self, reason, *, interruptible=False):
|
|
self.ending = True
|
|
self.reason = reason
|
|
|
|
def cancel_pending_end(self):
|
|
return False
|
|
|
|
def begin_response(self):
|
|
return None
|
|
|
|
def arm_after_speech(self):
|
|
return None
|
|
|
|
def track_speech(self):
|
|
return None
|
|
|
|
async def arm_after_tracked_speech(self):
|
|
return None
|
|
|
|
async def finish_after_current_speech(self, *, has_text):
|
|
return None
|
|
|
|
async def finish(self):
|
|
return None
|
|
|
|
|
|
class FakeFunctionParams:
|
|
def __init__(self, arguments=None):
|
|
self.arguments = arguments or {}
|
|
self.result = None
|
|
self.properties = None
|
|
|
|
async def result_callback(self, result, properties=None):
|
|
self.result = result
|
|
self.properties = properties
|
|
|
|
|
|
def system_runtime_tool(kind: str) -> RuntimeTool:
|
|
return RuntimeTool(
|
|
id=f"tool_{kind}",
|
|
name=kind,
|
|
function_name=kind,
|
|
type="system",
|
|
definition={"config": {"kind": kind}},
|
|
)
|
|
|
|
|
|
class DebugAutoTestContractTests(unittest.TestCase):
|
|
def test_batch_run_accepts_camel_case_dynamic_variables(self):
|
|
request = BatchRunCreate.model_validate(
|
|
{
|
|
"assistantId": "asst_1",
|
|
"evaluatorModelResourceId": "model_1",
|
|
"caseIds": ["case_1"],
|
|
"dynamicVariables": {"customer_name": "测试用户", "count": 2},
|
|
}
|
|
)
|
|
|
|
self.assertEqual(
|
|
request.dynamic_variables,
|
|
{"customer_name": "测试用户", "count": 2},
|
|
)
|
|
|
|
def test_each_test_case_gets_fresh_conversation_variables(self):
|
|
config = AssistantConfig(
|
|
type="prompt",
|
|
prompt="你好，{{ customer_name }}",
|
|
dynamic_variable_definitions={
|
|
"customer_name": {
|
|
"type": "string",
|
|
"required": True,
|
|
"default": None,
|
|
}
|
|
},
|
|
)
|
|
|
|
first = prepare_dynamic_config(
|
|
config,
|
|
{"customer_name": "测试用户"},
|
|
assistant_id="asst_1",
|
|
)
|
|
second = prepare_dynamic_config(
|
|
config,
|
|
{"customer_name": "测试用户"},
|
|
assistant_id="asst_1",
|
|
)
|
|
|
|
self.assertNotEqual(first.conversation_id, second.conversation_id)
|
|
self.assertEqual(
|
|
DynamicVariableStore.from_config(first).render(first.prompt),
|
|
"你好，测试用户",
|
|
)
|
|
|
|
def test_text_client_port_records_automatic_message_confirmation(self):
|
|
port = TextClientToolPort()
|
|
|
|
result = asyncio.run(
|
|
port.call(
|
|
"show_message",
|
|
{"title": "重要提示", "message": "请确认"},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(result["data"]["action"], "confirmed")
|
|
self.assertEqual(
|
|
port.auto_confirmed_messages,
|
|
[{"title": "重要提示", "message": "请确认"}],
|
|
)
|
|
|
|
|
|
class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
tools=[system_runtime_tool("end_conversation")],
|
|
)
|
|
brain = build_brain(cfg)
|
|
llm = FakeLLM()
|
|
call_end = FakeCallEnd()
|
|
executor_holder = {}
|
|
|
|
def create_executor(store):
|
|
executor = MockToolExecutor(store)
|
|
executor_holder["executor"] = executor
|
|
return executor
|
|
|
|
await brain.setup(
|
|
cfg,
|
|
BrainRuntime(
|
|
context=LLMContext(messages=[]),
|
|
llm=llm,
|
|
queue_frame=lambda _frame: asyncio.sleep(0),
|
|
set_system_prompt=lambda _prompt: None,
|
|
set_tools=lambda _tools: None,
|
|
call_end=call_end,
|
|
tool_executor_factory=create_executor,
|
|
),
|
|
)
|
|
|
|
params = FakeFunctionParams({"reason": "测试完成"})
|
|
await llm.functions["end_conversation"](params)
|
|
|
|
self.assertTrue(call_end.ending)
|
|
self.assertEqual(call_end.reason, "测试完成")
|
|
self.assertEqual(params.result["action"], "ending_call")
|
|
self.assertEqual(executor_holder["executor"].unmocked_calls, [])
|
|
|
|
async def test_startup_waits_for_queued_entry_reply(self):
|
|
capture = _TextCaptureProcessor(LLMContext(messages=[]))
|
|
|
|
class FakeWorker:
|
|
async def queue_frame(self, frame):
|
|
async def process_startup_queue():
|
|
capture.window.active_llm += 1
|
|
capture.window.touch()
|
|
await asyncio.sleep(0.01)
|
|
capture.outputs.append("开场后的自动首句")
|
|
capture.window.active_llm -= 1
|
|
capture.window.touch()
|
|
frame.reached.set()
|
|
|
|
asyncio.create_task(process_startup_queue())
|
|
|
|
runner_task = asyncio.create_task(asyncio.sleep(60))
|
|
try:
|
|
await asyncio.wait_for(
|
|
_wait_for_startup(
|
|
FakeWorker(),
|
|
capture,
|
|
runner_task,
|
|
capture.window.activity_count,
|
|
),
|
|
timeout=1,
|
|
)
|
|
self.assertEqual(capture.outputs, ["开场后的自动首句"])
|
|
finally:
|
|
runner_task.cancel()
|
|
await asyncio.gather(runner_task, return_exceptions=True)
|
|
|
|
async def test_text_startup_can_record_greeting_as_already_played(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
greeting="您好，这是一段开场白。",
|
|
prompt="不要重复开场白。",
|
|
)
|
|
brain = build_brain(cfg)
|
|
context = LLMContext(
|
|
messages=[{"role": "system", "content": cfg.prompt}]
|
|
)
|
|
|
|
greeting = await brain.greeting(cfg)
|
|
brain.prepare_greeting_context(greeting, context)
|
|
|
|
self.assertIn(
|
|
GREETING_CONTEXT_MARKER,
|
|
context.get_messages()[1]["content"],
|
|
)
|
|
self.assertIn(cfg.greeting, context.get_messages()[1]["content"])
|
|
|
|
async def test_upstream_pipeline_error_releases_pending_tool_wait(self):
|
|
capture = _TextCaptureProcessor(LLMContext(messages=[]))
|
|
capture.window.pending_tools.add("call_1")
|
|
runner_task = asyncio.create_task(asyncio.sleep(60))
|
|
try:
|
|
capture.fail(
|
|
ErrorFrame(
|
|
error=(
|
|
"UNMOCKED_TOOL_CALL: 工具 external_lookup "
|
|
"未配置 Mock 返回值"
|
|
)
|
|
)
|
|
)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"):
|
|
await asyncio.wait_for(
|
|
_wait_for_quiescence(capture, runner_task, 0),
|
|
timeout=0.1,
|
|
)
|
|
self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL")
|
|
self.assertEqual(capture.error.stage, "tool")
|
|
self.assertFalse(capture.error.retryable)
|
|
self.assertEqual(capture.window.pending_tools, set())
|
|
finally:
|
|
runner_task.cancel()
|
|
await asyncio.gather(runner_task, return_exceptions=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|