"""Real native backend workers, with model output explicitly simulated. No test in this module contacts a model provider or executes a remote task. """ from __future__ import annotations import asyncio from copy import deepcopy import unittest from unittest.mock import AsyncMock, patch from models import RuntimeTaskBackend from pipecat.frames.frames import ( FunctionCallFromLLM, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, ) from pipecat.services.llm_service import LLMService from pipecat.services.settings import LLMSettings from services.background_tasks.native import NativeTaskRuntime from services.background_tasks.reasoner import ReasonerExecutor, create_reasoner_llm def backend_config(**overrides): backend = { "id": "backend_reasoner", "type": "reasoner", "name": "Reasoner", "config": {"reasoning_effort": "provider_default", "skills": [{"name": "核查", "content": "先检查依据，再给结论。"}]}, "model": {"id": "model_reasoner", "capability": "LLM", "interface_type": "openai-llm", "values": {"modelId": "simulated", "apiUrl": "https://model.example.invalid/v1"}, "secrets": {"apiKey": "simulated-key"}}, **overrides, } return {"_backend": backend, "_task_id": "task_reasoner"} class SimulatedLLM(LLMService): """Feeds declared outputs into Pipecat's actual aggregation/tool loop.""" def __init__(self, steps): super().__init__(settings=LLMSettings( model="simulated", temperature=None, max_tokens=None, top_p=None, top_k=None, frequency_penalty=None, presence_penalty=None, seed=None, filter_incomplete_user_turns=False, user_turn_completion_config=None, )) self.steps = iter(steps) self.contexts = [] async def process_frame(self, frame, direction): await super().process_frame(frame, direction) if not isinstance(frame, LLMContextFrame): await self.push_frame(frame, direction) return self.contexts.append(deepcopy(frame.context.messages)) step = next(self.steps, "simulated output without explicit completion") await self.push_frame(LLMFullResponseStartFrame()) if isinstance(step, str): await self.push_frame(LLMTextFrame(step)) else: await self.run_function_calls([ FunctionCallFromLLM(function_name=name, tool_call_id=f"call_{len(self.contexts)}_{index}", arguments=arguments, context=frame.context) for index, (name, arguments) in enumerate(step) ]) await self.push_frame(LLMFullResponseEndFrame()) class ReasonerSettingsTest(unittest.TestCase): def test_nondefault_effort_requires_explicit_model_support(self): config = backend_config() config["_backend"]["config"]["reasoning_effort"] = "high" with self.assertRaisesRegex(ValueError, "未声明"): create_reasoner_llm(RuntimeTaskBackend.model_validate(config["_backend"])) def test_effort_is_sent_as_real_parameter_without_changing_shared_model(self): config = backend_config() config["_backend"]["config"]["reasoning_effort"] = "high" config["_backend"]["model"]["values"].update( supportsReasoningEffort=True, extraBody={"reasoning_effort": "low", "provider_option": True}, ) backend = RuntimeTaskBackend.model_validate(config["_backend"]) service = create_reasoner_llm(backend) params = service.build_chat_completion_params({"messages": []}) self.assertEqual(params["reasoning_effort"], "high") self.assertEqual(params["extra_body"], {"provider_option": True}) self.assertEqual(backend.model.values["extraBody"]["reasoning_effort"], "low") class ReasonerNativeTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): self.events = [] self.results = [] self.finished = asyncio.Event() self.executor = None async def execute(message, worker): handle = await self.executor.prepare() result = await self.executor.run(handle, "完成本次测试目标", self.emit) self.results.append(result) return result async def cancel(message): await self.executor.cancel(await self.executor.prepare()) async def on_result(message): if message.source == self.runtime.executor.name: self.finished.set() self.runtime = NativeTaskRuntime(execute, cancel, self.ignore, on_result) await self.runtime.start() async def asyncTearDown(self): await self.runtime.close() async def ignore(self, message): pass async def emit(self, event): self.events.append(event) async def start_reasoner(self, steps, config=None): self.model = SimulatedLLM(steps) self.executor = ReasonerExecutor(config or backend_config(), {}, runtime=self.runtime) patcher = patch("services.background_tasks.reasoner.create_reasoner_llm", return_value=self.model) patcher.start() self.addCleanup(patcher.stop) return await self.runtime.submit("task_reasoner", 10) async def wait_status(self, status): async def wait(): while not any(event.get("status") == status for event in self.events): await asyncio.sleep(0.01) await asyncio.wait_for(wait(), 5) async def test_explicit_completion_uses_actual_native_backend_and_clears_context(self): await self.start_reasoner([[('complete_task', {"status": "succeeded", "result": "显式模拟结果"})]]) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "succeeded") self.assertEqual(self.results[0]["result"], "显式模拟结果") self.assertIn("核查", self.model.contexts[0][0]["content"]) self.assertIsNone(self.executor._worker) self.assertTrue(self.runtime._run and not self.runtime._run.done()) self.assertFalse(any("显式模拟结果" in event.get("summary", "") for event in self.events)) async def test_plain_output_idle_does_not_claim_success_or_expose_reasoning(self): await self.start_reasoner(["simulated private reasoning text"]) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "failed") self.assertIn("complete_task", self.results[0]["result"]) self.assertNotIn("private reasoning", str(self.events)) async def test_explicit_partial_failure_preserves_failed_business_result(self): await self.start_reasoner([[('complete_task', { "status": "failed", "result": "只完成查询；写入服务不可用，目标未完整完成。", })]]) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "failed") self.assertIn("报告任务未完成", self.results[0]["summary"]) self.assertIn("只完成查询", self.results[0]["result"]) self.assertIsNone(self.executor._worker) async def test_missing_completion_status_does_not_accept_success(self): await self.start_reasoner([ [('complete_task', {"result": "执行结束，但未报告目标是否完成"})], "simulated model stopped without correcting completion status", ]) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "failed") self.assertIsNone(self.executor._final) self.assertEqual(len(self.model.contexts), 2) self.assertIn("必须提供 succeeded 或 failed", str(self.model.contexts[1])) async def test_invalid_completion_status_requires_model_correction(self): await self.start_reasoner([ [('complete_task', {"status": "partial", "result": "只完成查询"})], [('complete_task', {"status": "failed", "result": "只完成查询，目标未完成"})], ]) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "failed") self.assertEqual(self.results[0]["result"], "只完成查询，目标未完成") self.assertEqual(len(self.model.contexts), 2) async def test_question_reply_continues_same_native_context(self): await self.start_reasoner([ [('ask_user', {"question": "需要哪一天的数据？"})], [('complete_task', {"status": "succeeded", "result": "已按用户回答完成"})], ]) await self.wait_status("waiting_input") self.assertFalse(self.finished.is_set()) request = next(event["input_request"] for event in self.events if event.get("status") == "waiting_input") with self.assertRaisesRegex(ValueError, "已处理"): await self.executor.send_input({**request, "id": "other_question"}, {"answers": [["周一"]]}) await self.executor.send_input(request, {"answers": [["周一"]]}) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "succeeded") self.assertEqual(len(self.model.contexts), 2) self.assertIn("周一", str(self.model.contexts[1])) async def test_cancellation_stops_only_the_task_pipeline(self): job_id = await self.start_reasoner([[('ask_user', {"question": "等待测试回复"})]]) await self.wait_status("waiting_input") await self.runtime.cancel(job_id) await asyncio.wait_for(self.finished.wait(), 5) self.assertIsNone(self.executor._worker) self.assertTrue(self.runtime._run and not self.runtime._run.done()) async def test_http_tool_uses_binding_and_task_local_variable_store(self): config = backend_config(tools=[{ "id": "tool_lookup", "name": "查询", "function_name": "lookup", "type": "http", "definition": {"type": "http", "config": {"url": "https://tool.example.invalid", "parameters": [{"name": "customer", "required": True}]}}, }], tool_bindings=[{"tool_id": "tool_lookup", "argument_bindings": {"customer": {"source": "variable", "variable_name": "customer"}}, "result_assignments": {"answer": "answer"}}]) variables = {"customer": "local-customer", "answer": "before"} config["_variables"] = variables config["_variable_definitions"] = {name: {"type": "string"} for name in variables} # Isolate the HTTP side effect while retaining real argument/result handling. with patch("services.tool_executor.ToolExecutor._execute_http", AsyncMock(return_value={"status": "ok", "data": {"answer": "after"}})) as http: await self.start_reasoner([[('lookup', {})], [('complete_task', {"status": "succeeded", "result": "查询完成"})]], config) await asyncio.wait_for(self.finished.wait(), 5) self.assertEqual(self.results[0]["status"], "succeeded") self.assertEqual(http.await_args.args[1], {"customer": "local-customer"}) self.assertEqual(self.executor._tools.store.values["answer"], "after") self.assertEqual(variables["answer"], "before") async def test_client_tools_and_control_name_collisions_are_rejected_before_model_start(self): for kind, name in [("client", "browser_action"), ("http", "complete_task")]: config = backend_config(tools=[{"id": "bad", "name": "bad", "type": kind, "function_name": name}]) executor = ReasonerExecutor(config, {}, runtime=self.runtime) with patch("services.background_tasks.reasoner.create_reasoner_llm") as factory: with self.assertRaises(ValueError): executor._build_worker() factory.assert_not_called() if __name__ == "__main__": unittest.main()