239 lines
12 KiB
Python
239 lines
12 KiB
Python
"""Real native backend workers, with model output explicitly simulated.
|
|
|
|
No test in this module contacts a model provider or executes a remote task.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from copy import deepcopy
|
|
import unittest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from models import RuntimeTaskBackend
|
|
from pipecat.frames.frames import (
|
|
FunctionCallFromLLM,
|
|
LLMContextFrame,
|
|
LLMFullResponseEndFrame,
|
|
LLMFullResponseStartFrame,
|
|
LLMTextFrame,
|
|
)
|
|
from pipecat.services.llm_service import LLMService
|
|
from pipecat.services.settings import LLMSettings
|
|
from services.background_tasks.native import NativeTaskRuntime
|
|
from services.background_tasks.reasoner import ReasonerExecutor, create_reasoner_llm
|
|
|
|
|
|
def backend_config(**overrides):
|
|
backend = {
|
|
"id": "backend_reasoner", "type": "reasoner", "name": "Reasoner",
|
|
"config": {"reasoning_effort": "provider_default", "skills": [{"name": "核查", "content": "先检查依据，再给结论。"}]},
|
|
"model": {"id": "model_reasoner", "capability": "LLM", "interface_type": "openai-llm",
|
|
"values": {"modelId": "simulated", "apiUrl": "https://model.example.invalid/v1"},
|
|
"secrets": {"apiKey": "simulated-key"}},
|
|
**overrides,
|
|
}
|
|
return {"_backend": backend, "_task_id": "task_reasoner"}
|
|
|
|
|
|
class SimulatedLLM(LLMService):
|
|
"""Feeds declared outputs into Pipecat's actual aggregation/tool loop."""
|
|
|
|
def __init__(self, steps):
|
|
super().__init__(settings=LLMSettings(
|
|
model="simulated", temperature=None, max_tokens=None, top_p=None, top_k=None,
|
|
frequency_penalty=None, presence_penalty=None, seed=None,
|
|
filter_incomplete_user_turns=False, user_turn_completion_config=None,
|
|
))
|
|
self.steps = iter(steps)
|
|
self.contexts = []
|
|
|
|
async def process_frame(self, frame, direction):
|
|
await super().process_frame(frame, direction)
|
|
if not isinstance(frame, LLMContextFrame):
|
|
await self.push_frame(frame, direction)
|
|
return
|
|
self.contexts.append(deepcopy(frame.context.messages))
|
|
step = next(self.steps, "simulated output without explicit completion")
|
|
await self.push_frame(LLMFullResponseStartFrame())
|
|
if isinstance(step, str):
|
|
await self.push_frame(LLMTextFrame(step))
|
|
else:
|
|
await self.run_function_calls([
|
|
FunctionCallFromLLM(function_name=name, tool_call_id=f"call_{len(self.contexts)}_{index}", arguments=arguments, context=frame.context)
|
|
for index, (name, arguments) in enumerate(step)
|
|
])
|
|
await self.push_frame(LLMFullResponseEndFrame())
|
|
|
|
|
|
class ReasonerSettingsTest(unittest.TestCase):
|
|
def test_nondefault_effort_requires_explicit_model_support(self):
|
|
config = backend_config()
|
|
config["_backend"]["config"]["reasoning_effort"] = "high"
|
|
with self.assertRaisesRegex(ValueError, "未声明"):
|
|
create_reasoner_llm(RuntimeTaskBackend.model_validate(config["_backend"]))
|
|
|
|
def test_effort_is_sent_as_real_parameter_without_changing_shared_model(self):
|
|
config = backend_config()
|
|
config["_backend"]["config"]["reasoning_effort"] = "high"
|
|
config["_backend"]["model"]["values"].update(
|
|
supportsReasoningEffort=True, extraBody={"reasoning_effort": "low", "provider_option": True},
|
|
)
|
|
backend = RuntimeTaskBackend.model_validate(config["_backend"])
|
|
service = create_reasoner_llm(backend)
|
|
params = service.build_chat_completion_params({"messages": []})
|
|
self.assertEqual(params["reasoning_effort"], "high")
|
|
self.assertEqual(params["extra_body"], {"provider_option": True})
|
|
self.assertEqual(backend.model.values["extraBody"]["reasoning_effort"], "low")
|
|
|
|
|
|
class ReasonerNativeTest(unittest.IsolatedAsyncioTestCase):
|
|
async def asyncSetUp(self):
|
|
self.events = []
|
|
self.results = []
|
|
self.finished = asyncio.Event()
|
|
self.executor = None
|
|
|
|
async def execute(message, worker):
|
|
handle = await self.executor.prepare()
|
|
result = await self.executor.run(handle, "完成本次测试目标", self.emit)
|
|
self.results.append(result)
|
|
return result
|
|
|
|
async def cancel(message):
|
|
await self.executor.cancel(await self.executor.prepare())
|
|
|
|
async def on_result(message):
|
|
if message.source == self.runtime.executor.name:
|
|
self.finished.set()
|
|
|
|
self.runtime = NativeTaskRuntime(execute, cancel, self.ignore, on_result)
|
|
await self.runtime.start()
|
|
|
|
async def asyncTearDown(self):
|
|
await self.runtime.close()
|
|
|
|
async def ignore(self, message):
|
|
pass
|
|
|
|
async def emit(self, event):
|
|
self.events.append(event)
|
|
|
|
async def start_reasoner(self, steps, config=None):
|
|
self.model = SimulatedLLM(steps)
|
|
self.executor = ReasonerExecutor(config or backend_config(), {}, runtime=self.runtime)
|
|
patcher = patch("services.background_tasks.reasoner.create_reasoner_llm", return_value=self.model)
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
return await self.runtime.submit("task_reasoner", 10)
|
|
|
|
async def wait_status(self, status):
|
|
async def wait():
|
|
while not any(event.get("status") == status for event in self.events):
|
|
await asyncio.sleep(0.01)
|
|
await asyncio.wait_for(wait(), 5)
|
|
|
|
async def test_explicit_completion_uses_actual_native_backend_and_clears_context(self):
|
|
await self.start_reasoner([[('complete_task', {"status": "succeeded", "result": "显式模拟结果"})]])
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "succeeded")
|
|
self.assertEqual(self.results[0]["result"], "显式模拟结果")
|
|
self.assertIn("核查", self.model.contexts[0][0]["content"])
|
|
self.assertIsNone(self.executor._worker)
|
|
self.assertTrue(self.runtime._run and not self.runtime._run.done())
|
|
self.assertFalse(any("显式模拟结果" in event.get("summary", "") for event in self.events))
|
|
|
|
async def test_plain_output_idle_does_not_claim_success_or_expose_reasoning(self):
|
|
await self.start_reasoner(["simulated private reasoning text"])
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "failed")
|
|
self.assertIn("complete_task", self.results[0]["result"])
|
|
self.assertNotIn("private reasoning", str(self.events))
|
|
|
|
async def test_explicit_partial_failure_preserves_failed_business_result(self):
|
|
await self.start_reasoner([[('complete_task', {
|
|
"status": "failed", "result": "只完成查询；写入服务不可用，目标未完整完成。",
|
|
})]])
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "failed")
|
|
self.assertIn("报告任务未完成", self.results[0]["summary"])
|
|
self.assertIn("只完成查询", self.results[0]["result"])
|
|
self.assertIsNone(self.executor._worker)
|
|
|
|
async def test_missing_completion_status_does_not_accept_success(self):
|
|
await self.start_reasoner([
|
|
[('complete_task', {"result": "执行结束，但未报告目标是否完成"})],
|
|
"simulated model stopped without correcting completion status",
|
|
])
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "failed")
|
|
self.assertIsNone(self.executor._final)
|
|
self.assertEqual(len(self.model.contexts), 2)
|
|
self.assertIn("必须提供 succeeded 或 failed", str(self.model.contexts[1]))
|
|
|
|
async def test_invalid_completion_status_requires_model_correction(self):
|
|
await self.start_reasoner([
|
|
[('complete_task', {"status": "partial", "result": "只完成查询"})],
|
|
[('complete_task', {"status": "failed", "result": "只完成查询，目标未完成"})],
|
|
])
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "failed")
|
|
self.assertEqual(self.results[0]["result"], "只完成查询，目标未完成")
|
|
self.assertEqual(len(self.model.contexts), 2)
|
|
|
|
async def test_question_reply_continues_same_native_context(self):
|
|
await self.start_reasoner([
|
|
[('ask_user', {"question": "需要哪一天的数据？"})],
|
|
[('complete_task', {"status": "succeeded", "result": "已按用户回答完成"})],
|
|
])
|
|
await self.wait_status("waiting_input")
|
|
self.assertFalse(self.finished.is_set())
|
|
request = next(event["input_request"] for event in self.events if event.get("status") == "waiting_input")
|
|
with self.assertRaisesRegex(ValueError, "已处理"):
|
|
await self.executor.send_input({**request, "id": "other_question"}, {"answers": [["周一"]]})
|
|
await self.executor.send_input(request, {"answers": [["周一"]]})
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "succeeded")
|
|
self.assertEqual(len(self.model.contexts), 2)
|
|
self.assertIn("周一", str(self.model.contexts[1]))
|
|
|
|
async def test_cancellation_stops_only_the_task_pipeline(self):
|
|
job_id = await self.start_reasoner([[('ask_user', {"question": "等待测试回复"})]])
|
|
await self.wait_status("waiting_input")
|
|
await self.runtime.cancel(job_id)
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertIsNone(self.executor._worker)
|
|
self.assertTrue(self.runtime._run and not self.runtime._run.done())
|
|
|
|
async def test_http_tool_uses_binding_and_task_local_variable_store(self):
|
|
config = backend_config(tools=[{
|
|
"id": "tool_lookup", "name": "查询", "function_name": "lookup", "type": "http",
|
|
"definition": {"type": "http", "config": {"url": "https://tool.example.invalid", "parameters": [{"name": "customer", "required": True}]}},
|
|
}], tool_bindings=[{"tool_id": "tool_lookup", "argument_bindings": {"customer": {"source": "variable", "variable_name": "customer"}},
|
|
"result_assignments": {"answer": "answer"}}])
|
|
variables = {"customer": "local-customer", "answer": "before"}
|
|
config["_variables"] = variables
|
|
config["_variable_definitions"] = {name: {"type": "string"} for name in variables}
|
|
|
|
# Isolate the HTTP side effect while retaining real argument/result handling.
|
|
with patch("services.tool_executor.ToolExecutor._execute_http", AsyncMock(return_value={"status": "ok", "data": {"answer": "after"}})) as http:
|
|
await self.start_reasoner([[('lookup', {})], [('complete_task', {"status": "succeeded", "result": "查询完成"})]], config)
|
|
await asyncio.wait_for(self.finished.wait(), 5)
|
|
self.assertEqual(self.results[0]["status"], "succeeded")
|
|
self.assertEqual(http.await_args.args[1], {"customer": "local-customer"})
|
|
self.assertEqual(self.executor._tools.store.values["answer"], "after")
|
|
self.assertEqual(variables["answer"], "before")
|
|
|
|
async def test_client_tools_and_control_name_collisions_are_rejected_before_model_start(self):
|
|
for kind, name in [("client", "browser_action"), ("http", "complete_task")]:
|
|
config = backend_config(tools=[{"id": "bad", "name": "bad", "type": kind, "function_name": name}])
|
|
executor = ReasonerExecutor(config, {}, runtime=self.runtime)
|
|
with patch("services.background_tasks.reasoner.create_reasoner_llm") as factory:
|
|
with self.assertRaises(ValueError):
|
|
executor._build_worker()
|
|
factory.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|