feat: add dynamic variables and startup result handling in batch run schemas
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types. - Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages. - Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs. - Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases. - Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
This commit is contained in:
Xin Wang
@@ -0,0 +1,264 @@
|
||||
import asyncio
|
||||
import unittest
|
||||
|
||||
from models import AssistantConfig, RuntimeTool
|
||||
from pipecat.frames.frames import ErrorFrame
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from services.brains import BrainRuntime, build_brain
|
||||
from services.brains.base import GREETING_CONTEXT_MARKER
|
||||
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
|
||||
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
|
||||
from services.test_runs.text_runner import (
|
||||
_TextCaptureProcessor,
|
||||
_wait_for_startup,
|
||||
_wait_for_quiescence,
|
||||
)
|
||||
from test_schemas import BatchRunCreate
|
||||
|
||||
|
||||
class FakeLLM:
|
||||
def __init__(self):
|
||||
self.functions = {}
|
||||
|
||||
def register_function(self, name, handler, **_options):
|
||||
self.functions[name] = handler
|
||||
|
||||
|
||||
class FakeCallEnd:
|
||||
def __init__(self):
|
||||
self.ending = False
|
||||
self.reason = ""
|
||||
|
||||
@property
|
||||
def accepts_input(self):
|
||||
return not self.ending
|
||||
|
||||
def begin(self, reason, *, interruptible=False):
|
||||
self.ending = True
|
||||
self.reason = reason
|
||||
|
||||
def cancel_pending_end(self):
|
||||
return False
|
||||
|
||||
def begin_response(self):
|
||||
return None
|
||||
|
||||
def arm_after_speech(self):
|
||||
return None
|
||||
|
||||
def track_speech(self):
|
||||
return None
|
||||
|
||||
async def arm_after_tracked_speech(self):
|
||||
return None
|
||||
|
||||
async def finish_after_current_speech(self, *, has_text):
|
||||
return None
|
||||
|
||||
async def finish(self):
|
||||
return None
|
||||
|
||||
|
||||
class FakeFunctionParams:
|
||||
def __init__(self, arguments=None):
|
||||
self.arguments = arguments or {}
|
||||
self.result = None
|
||||
self.properties = None
|
||||
|
||||
async def result_callback(self, result, properties=None):
|
||||
self.result = result
|
||||
self.properties = properties
|
||||
|
||||
|
||||
def system_runtime_tool(kind: str) -> RuntimeTool:
|
||||
return RuntimeTool(
|
||||
id=f"tool_{kind}",
|
||||
name=kind,
|
||||
function_name=kind,
|
||||
type="system",
|
||||
definition={"config": {"kind": kind}},
|
||||
)
|
||||
|
||||
|
||||
class DebugAutoTestContractTests(unittest.TestCase):
|
||||
def test_batch_run_accepts_camel_case_dynamic_variables(self):
|
||||
request = BatchRunCreate.model_validate(
|
||||
{
|
||||
"assistantId": "asst_1",
|
||||
"evaluatorModelResourceId": "model_1",
|
||||
"caseIds": ["case_1"],
|
||||
"dynamicVariables": {"customer_name": "测试用户", "count": 2},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
request.dynamic_variables,
|
||||
{"customer_name": "测试用户", "count": 2},
|
||||
)
|
||||
|
||||
def test_each_test_case_gets_fresh_conversation_variables(self):
|
||||
config = AssistantConfig(
|
||||
type="prompt",
|
||||
prompt="你好，{{ customer_name }}",
|
||||
dynamic_variable_definitions={
|
||||
"customer_name": {
|
||||
"type": "string",
|
||||
"required": True,
|
||||
"default": None,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
first = prepare_dynamic_config(
|
||||
config,
|
||||
{"customer_name": "测试用户"},
|
||||
assistant_id="asst_1",
|
||||
)
|
||||
second = prepare_dynamic_config(
|
||||
config,
|
||||
{"customer_name": "测试用户"},
|
||||
assistant_id="asst_1",
|
||||
)
|
||||
|
||||
self.assertNotEqual(first.conversation_id, second.conversation_id)
|
||||
self.assertEqual(
|
||||
DynamicVariableStore.from_config(first).render(first.prompt),
|
||||
"你好，测试用户",
|
||||
)
|
||||
|
||||
def test_text_client_port_records_automatic_message_confirmation(self):
|
||||
port = TextClientToolPort()
|
||||
|
||||
result = asyncio.run(
|
||||
port.call(
|
||||
"show_message",
|
||||
{"title": "重要提示", "message": "请确认"},
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(result["data"]["action"], "confirmed")
|
||||
self.assertEqual(
|
||||
port.auto_confirmed_messages,
|
||||
[{"title": "重要提示", "message": "请确认"}],
|
||||
)
|
||||
|
||||
|
||||
class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
|
||||
cfg = AssistantConfig(
|
||||
type="prompt",
|
||||
tools=[system_runtime_tool("end_conversation")],
|
||||
)
|
||||
brain = build_brain(cfg)
|
||||
llm = FakeLLM()
|
||||
call_end = FakeCallEnd()
|
||||
executor_holder = {}
|
||||
|
||||
def create_executor(store):
|
||||
executor = MockToolExecutor(store)
|
||||
executor_holder["executor"] = executor
|
||||
return executor
|
||||
|
||||
await brain.setup(
|
||||
cfg,
|
||||
BrainRuntime(
|
||||
context=LLMContext(messages=[]),
|
||||
llm=llm,
|
||||
queue_frame=lambda _frame: asyncio.sleep(0),
|
||||
set_system_prompt=lambda _prompt: None,
|
||||
set_tools=lambda _tools: None,
|
||||
call_end=call_end,
|
||||
tool_executor_factory=create_executor,
|
||||
),
|
||||
)
|
||||
|
||||
params = FakeFunctionParams({"reason": "测试完成"})
|
||||
await llm.functions["end_conversation"](params)
|
||||
|
||||
self.assertTrue(call_end.ending)
|
||||
self.assertEqual(call_end.reason, "测试完成")
|
||||
self.assertEqual(params.result["action"], "ending_call")
|
||||
self.assertEqual(executor_holder["executor"].unmocked_calls, [])
|
||||
|
||||
async def test_startup_waits_for_queued_entry_reply(self):
|
||||
capture = _TextCaptureProcessor(LLMContext(messages=[]))
|
||||
|
||||
class FakeWorker:
|
||||
async def queue_frame(self, frame):
|
||||
async def process_startup_queue():
|
||||
capture.window.active_llm += 1
|
||||
capture.window.touch()
|
||||
await asyncio.sleep(0.01)
|
||||
capture.outputs.append("开场后的自动首句")
|
||||
capture.window.active_llm -= 1
|
||||
capture.window.touch()
|
||||
frame.reached.set()
|
||||
|
||||
asyncio.create_task(process_startup_queue())
|
||||
|
||||
runner_task = asyncio.create_task(asyncio.sleep(60))
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
_wait_for_startup(
|
||||
FakeWorker(),
|
||||
capture,
|
||||
runner_task,
|
||||
capture.window.activity_count,
|
||||
),
|
||||
timeout=1,
|
||||
)
|
||||
self.assertEqual(capture.outputs, ["开场后的自动首句"])
|
||||
finally:
|
||||
runner_task.cancel()
|
||||
await asyncio.gather(runner_task, return_exceptions=True)
|
||||
|
||||
async def test_text_startup_can_record_greeting_as_already_played(self):
|
||||
cfg = AssistantConfig(
|
||||
type="prompt",
|
||||
greeting="您好，这是一段开场白。",
|
||||
prompt="不要重复开场白。",
|
||||
)
|
||||
brain = build_brain(cfg)
|
||||
context = LLMContext(
|
||||
messages=[{"role": "system", "content": cfg.prompt}]
|
||||
)
|
||||
|
||||
greeting = await brain.greeting(cfg)
|
||||
brain.prepare_greeting_context(greeting, context)
|
||||
|
||||
self.assertIn(
|
||||
GREETING_CONTEXT_MARKER,
|
||||
context.get_messages()[1]["content"],
|
||||
)
|
||||
self.assertIn(cfg.greeting, context.get_messages()[1]["content"])
|
||||
|
||||
async def test_upstream_pipeline_error_releases_pending_tool_wait(self):
|
||||
capture = _TextCaptureProcessor(LLMContext(messages=[]))
|
||||
capture.window.pending_tools.add("call_1")
|
||||
runner_task = asyncio.create_task(asyncio.sleep(60))
|
||||
try:
|
||||
capture.fail(
|
||||
ErrorFrame(
|
||||
error=(
|
||||
"UNMOCKED_TOOL_CALL: 工具 external_lookup "
|
||||
"未配置 Mock 返回值"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "UNMOCKED_TOOL_CALL"):
|
||||
await asyncio.wait_for(
|
||||
_wait_for_quiescence(capture, runner_task, 0),
|
||||
timeout=0.1,
|
||||
)
|
||||
self.assertEqual(capture.error.code, "UNMOCKED_TOOL_CALL")
|
||||
self.assertEqual(capture.error.stage, "tool")
|
||||
self.assertFalse(capture.error.retryable)
|
||||
self.assertEqual(capture.window.pending_tools, set())
|
||||
finally:
|
||||
runner_task.cancel()
|
||||
await asyncio.gather(runner_task, return_exceptions=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user