fix: harden realtime pipeline lifecycle

2026-09-07 08:27:52 +08:00
parent 43f2cc445e
commit 8025ed8aa0
12 changed files with 445 additions and 42 deletions
@@ -0,0 +1,38 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+from services.config_resolver import _agent_resource_for, _resource_for
+
+
+class DisabledResourceTest(unittest.IsolatedAsyncioTestCase):
+    async def test_disabled_binding_uses_enabled_fallback_for_models_and_agents(self):
+        for capability, interface_type in (("LLM", "openai-llm"), ("Agent", "dify")):
+            with self.subTest(capability=capability):
+                disabled = SimpleNamespace(
+                    id="disabled", capability=capability,
+                    interface_type=interface_type, enabled=False,
+                )
+                fallback = SimpleNamespace(id="enabled", enabled=True)
+                result = Mock()
+                result.scalar_one_or_none.return_value = fallback
+                session = SimpleNamespace(
+                    get=AsyncMock(side_effect=[SimpleNamespace(model_resource_id="disabled"), disabled]),
+                    execute=AsyncMock(return_value=result),
+                )
+                if capability == "Agent":
+                    selected = await _agent_resource_for(session, "assistant", interface_type)
+                else:
+                    selected = await _resource_for(session, "assistant", capability)
+                self.assertIs(selected, fallback)
+                statement = str(session.execute.await_args.args[0])
+                self.assertIn("model_resources.enabled IS true", statement)
+
+    async def test_enabled_binding_is_preserved(self):
+        resource = SimpleNamespace(id="enabled", capability="LLM", enabled=True)
+        session = SimpleNamespace(
+            get=AsyncMock(side_effect=[SimpleNamespace(model_resource_id=resource.id), resource]),
+            execute=AsyncMock(),
+        )
+        self.assertIs(await _resource_for(session, "assistant", "LLM"), resource)
+        session.execute.assert_not_awaited()
@@ -9,6 +9,91 @@ from services.conversation_history import ConversationRecorder


 class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
+    async def test_background_messages_keep_order_and_ignore_streaming_deltas(self):
+        recorder = ConversationRecorder("conv_test")
+        recorder.record_transport_message = AsyncMock()
+        events = [
+            {"type": "transcript", "content": "user"},
+            {"type": "assistant-text-end", "content": "assistant"},
+        ]
+        for event in events:
+            recorder.record_transport_message_later(event)
+        recorder.record_transport_message_later({"type": "assistant-text-delta"})
+        await recorder.flush_pending_messages()
+        self.assertEqual(
+            [call.args[0] for call in recorder.record_transport_message.await_args_list],
+            events,
+        )
+        self.assertIsNone(recorder._message_writer)
+
+    async def test_slow_database_does_not_block_audio(self):
+        from pipecat.frames.frames import EndFrame, OutputAudioRawFrame, OutputTransportMessageUrgentFrame
+        from pipecat.pipeline.pipeline import Pipeline
+        from pipecat.pipeline.worker import PipelineWorker
+        from pipecat.processors.frame_processor import FrameProcessor
+        from pipecat.workers.runner import WorkerRunner
+        from services.pipecat.processors import ConversationHistoryProcessor
+
+        entered, release, audio_received = asyncio.Event(), asyncio.Event(), asyncio.Event()
+        recorder = ConversationRecorder("conv_test")
+
+        async def slow_write(message):
+            entered.set()
+            await release.wait()
+
+        recorder.record_transport_message = slow_write
+
+        class AudioSink(FrameProcessor):
+            async def process_frame(self, frame, direction):
+                await super().process_frame(frame, direction)
+                if isinstance(frame, OutputAudioRawFrame):
+                    audio_received.set()
+                await self.push_frame(frame, direction)
+
+        worker = PipelineWorker(
+            Pipeline([ConversationHistoryProcessor(recorder), AudioSink()]), enable_rtvi=False,
+        )
+        runner = WorkerRunner(handle_sigint=False)
+        await runner.add_workers(worker)
+        task = asyncio.create_task(runner.run())
+        try:
+            await worker.queue_frame(OutputTransportMessageUrgentFrame(message={"type": "transcript"}))
+            await asyncio.wait_for(entered.wait(), 3)
+            await worker.queue_frame(OutputAudioRawFrame(audio=b"\0" * 320, sample_rate=16000, num_channels=1))
+            await asyncio.wait_for(audio_received.wait(), 1)
+            self.assertFalse(release.is_set())
+        finally:
+            release.set()
+            await recorder.flush_pending_messages()
+            await worker.queue_frame(EndFrame())
+            await asyncio.wait_for(task, 3)
+
+    async def test_history_buffer_is_bounded_and_slow_writer_is_cancelled(self):
+        started = asyncio.Event()
+        cancelled = asyncio.Event()
+        with patch("services.conversation_history.MAX_PENDING_HISTORY_MESSAGES", 2):
+            recorder = ConversationRecorder("conv_test")
+
+        async def blocked_write(message):
+            started.set()
+            try:
+                await asyncio.Event().wait()
+            finally:
+                cancelled.set()
+
+        recorder.record_transport_message = blocked_write
+        recorder.record_transport_message_later({"type": "transcript"})
+        await started.wait()
+        for _ in range(4):
+            recorder.record_transport_message_later({"type": "transcript"})
+        self.assertEqual(recorder._pending_messages.qsize(), 2)
+        with patch("services.conversation_history.HISTORY_FLUSH_TIMEOUT_SECONDS", 0.01):
+            await recorder.flush_pending_messages()
+        self.assertTrue(cancelled.is_set())
+        self.assertTrue(recorder._pending_messages.empty())
+        recorder.record_transport_message_later({"type": "transcript"})
+        self.assertIsNone(recorder._message_writer)
+
    async def test_completed_conversation_queues_enabled_analysis(self):
        conversation = SimpleNamespace(
            status="active",
@@ -0,0 +1,101 @@
+"""Regression checks for whole-pipeline assembly and terminal outcomes."""
+
+import asyncio
+import unittest
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+from models import AssistantConfig
+from pipecat.frames.frames import CancelFrame, EndFrame, ErrorFrame, StartFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.worker import PipelineWorker
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from services.pipecat import pipeline
+from services.pipecat.pipeline_runner import run_media_worker
+
+
+class PipelineAssemblyTest(unittest.IsolatedAsyncioTestCase):
+    async def test_realtime_startup_preserves_client_capabilities(self):
+        brain = SimpleNamespace(
+            spec=SimpleNamespace(supported_runtime_modes={"realtime"}),
+            system_prompt=lambda cfg: "test",
+            greeting=AsyncMock(return_value=""),
+            setup_realtime=AsyncMock(),
+            on_session_update=AsyncMock(),
+        )
+        transport = SimpleNamespace(
+            input=lambda: FrameProcessor(),
+            output=lambda: FrameProcessor(),
+            event_handler=lambda name: lambda handler: handler,
+        )
+        with (
+            patch.object(pipeline, "build_brain", return_value=brain),
+            patch.object(pipeline, "create_realtime_service", return_value=FrameProcessor()),
+            patch.object(pipeline.ConversationRecorder, "start", new=AsyncMock(return_value=None)),
+            patch.object(pipeline, "run_media_worker", new=AsyncMock()) as run,
+        ):
+            await pipeline.run_pipeline(
+                transport, AssistantConfig(runtimeMode="realtime"),
+                client_tool_names={"show_message"},
+            )
+        run.assert_awaited_once()
+        runtime = brain.setup_realtime.await_args.args[1]
+        self.assertTrue(runtime.client_tools.supports("show_message"))
+        self.assertFalse(runtime.client_tools.supports("unregistered_tool"))
+
+
+class TerminalProcessor(FrameProcessor):
+    def __init__(self, terminal):
+        super().__init__()
+        self.terminal = terminal
+
+    async def process_frame(self, frame, direction):
+        await super().process_frame(frame, direction)
+        await self.push_frame(frame, direction)
+        if isinstance(frame, StartFrame):
+            if isinstance(self.terminal, ErrorFrame):
+                await self.push_frame(self.terminal, FrameDirection.UPSTREAM)
+            else:
+                await self.push_frame(self.terminal)
+
+
+class PipelineOutcomeTest(unittest.IsolatedAsyncioTestCase):
+    async def test_fatal_error_is_failed_even_when_runner_returns_normally(self):
+        worker = PipelineWorker(
+            Pipeline([TerminalProcessor(ErrorFrame(error="provider failed", fatal=True))]),
+            enable_rtvi=False,
+        )
+        recorder = SimpleNamespace(finish=AsyncMock())
+        await asyncio.wait_for(run_media_worker(worker, recorder), 3)
+        recorder.finish.assert_awaited_once_with(status="failed")
+
+    async def test_normal_end_and_client_disconnect_are_completed(self):
+        for terminal in (EndFrame(), CancelFrame(reason="client_disconnected")):
+            with self.subTest(terminal=type(terminal).__name__):
+                worker = PipelineWorker(Pipeline([Passthrough()]), enable_rtvi=False)
+                recorder = SimpleNamespace(finish=AsyncMock())
+                await worker.queue_frame(terminal)
+                await asyncio.wait_for(run_media_worker(worker, recorder), 3)
+                recorder.finish.assert_awaited_once_with(status="completed")
+
+    async def test_server_task_cancellation_remains_failed_and_propagates(self):
+        worker = PipelineWorker(Pipeline([Passthrough()]), enable_rtvi=False)
+        started = asyncio.Event()
+
+        @worker.event_handler("on_pipeline_started")
+        async def on_started(_worker, _frame):
+            started.set()
+
+        recorder = SimpleNamespace(finish=AsyncMock())
+        task = asyncio.create_task(run_media_worker(worker, recorder))
+        await asyncio.wait_for(started.wait(), 3)
+        task.cancel()
+        with self.assertRaises(asyncio.CancelledError):
+            await asyncio.wait_for(task, 3)
+        recorder.finish.assert_awaited_once_with(status="failed")
+
+
+class Passthrough(FrameProcessor):
+    async def process_frame(self, frame, direction):
+        await super().process_frame(frame, direction)
+        await self.push_frame(frame, direction)
@@ -8,12 +8,16 @@ from unittest.mock import AsyncMock, patch

 from fastapi import HTTPException
 from routes import voice_webrtc
+from services.realtime import lifecycle
 from starlette.requests import Request


 class FakeConnection:
    pc_id = "pc_test"

+    def __init__(self):
+        self.disconnect = AsyncMock()
+

 def encoded_header_json(value: object) -> str:
    payload = json.dumps(value).encode("utf-8")
@@ -81,6 +85,16 @@ class WebRTCOfferMetadataTest(unittest.IsolatedAsyncioTestCase):

        self.assertEqual(raised.exception.status_code, 400)

+    async def test_invalid_assistant_returns_actionable_bad_request(self):
+        with patch.object(
+            voice_webrtc, "_handle_offer_payload",
+            new=AsyncMock(side_effect=ValueError("助手不存在: missing")),
+        ):
+            with self.assertRaises(HTTPException) as raised:
+                await voice_webrtc.http_offer(offer_request(), {"type": "offer"})
+        self.assertEqual(raised.exception.status_code, 400)
+        self.assertIn("助手不存在", raised.exception.detail)
+

 class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase):
    async def asyncTearDown(self):
@@ -89,6 +103,9 @@ class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase):
            task.cancel()
        if tasks:
            await asyncio.gather(*tasks, return_exceptions=True)
+        await asyncio.sleep(0)
+        if lifecycle.connection_cleanup_tasks:
+            await asyncio.gather(*list(lifecycle.connection_cleanup_tasks))
        voice_webrtc._pipeline_tasks.clear()
        voice_webrtc._connection_tasks.clear()
        voice_webrtc._active_connections.clear()
@@ -111,6 +128,54 @@ class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase):
        self.assertNotIn(task, voice_webrtc._pipeline_tasks)
        self.assertNotIn(connection, voice_webrtc._connection_tasks)

+    async def test_failed_startup_disconnects_and_releases_connection(self):
+        connection = FakeConnection()
+
+        async def pipeline():
+            raise ValueError("provider construction failed")
+
+        task = voice_webrtc._start_pipeline_task(connection, pipeline())
+        with self.assertRaises(ValueError):
+            await task
+        await asyncio.sleep(0)
+        if lifecycle.connection_cleanup_tasks:
+            await asyncio.gather(*list(lifecycle.connection_cleanup_tasks))
+        connection.disconnect.assert_awaited_once()
+        self.assertNotIn(connection, voice_webrtc._active_connections)
+        self.assertNotIn(connection, voice_webrtc._connection_tasks)
+
+    async def test_cleanup_closed_handler_can_wait_for_completed_pipeline(self):
+        connection = FakeConnection()
+
+        async def pipeline():
+            return None
+
+        task = voice_webrtc._start_pipeline_task(connection, pipeline())
+
+        async def disconnect():
+            await lifecycle.wait_for_pipeline_close(
+                task, connection_id=connection.pc_id, timeout=0.01,
+            )
+
+        connection.disconnect.side_effect = disconnect
+        await task
+        await asyncio.sleep(0)
+        if lifecycle.connection_cleanup_tasks:
+            await asyncio.wait_for(
+                asyncio.gather(*list(lifecycle.connection_cleanup_tasks)), 0.5,
+            )
+        connection.disconnect.assert_awaited_once()
+
+    async def test_already_closed_connection_is_not_disconnected_twice(self):
+        connection = FakeConnection()
+        release = asyncio.Event()
+        task = voice_webrtc._start_pipeline_task(connection, release.wait())
+        voice_webrtc._active_connections.discard(connection)
+        release.set()
+        await task
+        await asyncio.sleep(0)
+        connection.disconnect.assert_not_awaited()
+
    async def test_stuck_pipeline_is_cancelled_after_close_grace_period(self):
        connection = FakeConnection()
        cancelled = asyncio.Event()