fix: harden realtime pipeline lifecycle

backend/services/config_resolver.py

@@ -34,7 +34,7 @@ async def _resource_for(
     binding = await session.get(AssistantModelBinding, (assistant_id, capability))
     resource_id = binding.model_resource_id if binding else None
     resource = await session.get(ModelResource, resource_id) if resource_id else None
-    if resource and resource.capability != capability:
+    if resource and (not resource.enabled or resource.capability != capability):
         resource = None
     if resource is None:
         resource = (
@@ -60,6 +60,7 @@ async def _agent_resource_for(
     resource = await session.get(ModelResource, resource_id) if resource_id else None
     if (
         resource
+        and resource.enabled
         and resource.capability == "Agent"
         and resource.interface_type == interface_type
     ):

backend/services/conversation_history.py

@@ -16,6 +16,8 @@ from services.object_storage import delete_object, put_object, storage_uri
 
 
 MAX_WORKFLOW_TRACE_EVENTS = 1000
+MAX_PENDING_HISTORY_MESSAGES = 256
+HISTORY_FLUSH_TIMEOUT_SECONDS = 3.0
 
 
 def _parse_timestamp(value: object) -> datetime:
@@ -39,6 +41,63 @@ class ConversationRecorder:
         self._lock = asyncio.Lock()
         self._seen_events: set[str] = set()
         self._pending_artifacts: set[asyncio.Task[None]] = set()
+        self._pending_messages: asyncio.Queue[dict] = asyncio.Queue(
+            maxsize=MAX_PENDING_HISTORY_MESSAGES,
+        )
+        self._message_writer: asyncio.Task[None] | None = None
+        self._accepting_messages = True
+
+    def record_transport_message_later(self, message: object) -> None:
+        """Keep database latency off the media path, with bounded buffering."""
+        if (
+            not self._accepting_messages
+            or not isinstance(message, dict)
+            or message.get("type") not in {
+                "transcript", "assistant-text-end", "workflow-event",
+            }
+        ):
+            return
+        try:
+            self._pending_messages.put_nowait(deepcopy(message))
+        except asyncio.QueueFull:
+            logger.warning(f"对话历史写入积压，跳过本条记录: session={self.session_id}")
+            return
+        if self._message_writer is None:
+            self._message_writer = asyncio.create_task(
+                self._write_pending_messages(),
+                name=f"conversation-history-writer:{self.session_id}",
+            )
+
+    async def _write_pending_messages(self) -> None:
+        while True:
+            message = await self._pending_messages.get()
+            try:
+                await self.record_transport_message(message)
+            except Exception as exc:
+                logger.warning(f"对话历史写入失败: {type(exc).__name__}")
+            finally:
+                self._pending_messages.task_done()
+
+    async def flush_pending_messages(self) -> None:
+        """Drain final transcripts before finishing the conversation row."""
+        self._accepting_messages = False
+        writer = self._message_writer
+        if writer is None:
+            return
+        try:
+            await asyncio.wait_for(
+                self._pending_messages.join(),
+                timeout=HISTORY_FLUSH_TIMEOUT_SECONDS,
+            )
+        except TimeoutError:
+            logger.warning(f"对话历史收尾超时，丢弃剩余缓冲: session={self.session_id}")
+        finally:
+            writer.cancel()
+            await asyncio.gather(writer, return_exceptions=True)
+            self._message_writer = None
+            while not self._pending_messages.empty():
+                self._pending_messages.get_nowait()
+                self._pending_messages.task_done()
 
     @classmethod
     async def start(
@@ -288,6 +347,7 @@ class ConversationRecorder:
             raise
 
     async def _finish(self, *, status: str) -> None:
+        await self.flush_pending_messages()
         pending_artifacts = list(self._pending_artifacts)
         if pending_artifacts:
             await asyncio.gather(*pending_artifacts, return_exceptions=True)

backend/services/pipecat/pipeline.py

@@ -103,7 +103,7 @@ from services.realtime.protocol import (
     PipelineProtocolRuntime,
     RealtimeProviderControlFrame,
 )
-from pipecat.workers.runner import WorkerRunner
+from services.pipecat.pipeline_runner import run_media_worker
 
 
 KNOWLEDGE_TOOL_NAME = "search_knowledge_base"
@@ -260,6 +260,7 @@ async def run_pipeline(
             assistant_id=assistant_id,
             channel=channel,
             protocol_adapter=protocol_adapter,
+            client_tool_names=client_tool_names,
         )
         return
 
@@ -987,25 +988,7 @@ async def run_pipeline(
         submit_user_input=submit_user_input,
         client_tools=client_tools,
     )
-    runner = WorkerRunner(handle_sigint=False)
-    run_status = "completed"
-    can_finalize_recorder = True
-    try:
-        await runner.add_workers(worker)
-        await runner.run()
-    except GeneratorExit:
-        # A coroutine being synchronously closed cannot perform async cleanup.
-        # The WebRTC route now prevents this by retaining every pipeline task;
-        # keep this guard so a future unmanaged caller cannot corrupt asyncpg.
-        run_status = "failed"
-        can_finalize_recorder = False
-        raise
-    except Exception:
-        run_status = "failed"
-        raise
-    finally:
-        if recorder and can_finalize_recorder:
-            await recorder.finish(status=run_status)
+    await run_media_worker(worker, recorder)
     logger.info("管线已结束")
 
 
@@ -1018,6 +1001,7 @@ async def run_realtime_pipeline(
     assistant_id: str | None = None,
     channel: str = "webrtc",
     protocol_adapter: PipelineProtocolAdapter | None = None,
+    client_tool_names: set[str] | None = None,
 ) -> None:
     """Run a speech-to-speech model that owns ASR, reasoning, and synthesis."""
     realtime = create_realtime_service(
@@ -1239,20 +1223,5 @@ async def run_realtime_pipeline(
         vision_enabled=vision_enabled,
         vision_state=vision_state,
     )
-    runner = WorkerRunner(handle_sigint=False)
-    run_status = "completed"
-    can_finalize_recorder = True
-    try:
-        await runner.add_workers(worker)
-        await runner.run()
-    except GeneratorExit:
-        run_status = "failed"
-        can_finalize_recorder = False
-        raise
-    except Exception:
-        run_status = "failed"
-        raise
-    finally:
-        if recorder and can_finalize_recorder:
-            await recorder.finish(status=run_status)
+    await run_media_worker(worker, recorder)
     logger.info("Realtime 管线已结束")

backend/services/pipecat/pipeline_runner.py

@@ -0,0 +1,56 @@
+"""Run a media worker and persist its actual terminal outcome."""
+
+import asyncio
+
+from pipecat.frames.frames import CancelFrame, EndFrame, ErrorFrame
+from pipecat.pipeline.worker import PipelineWorker
+from pipecat.workers.runner import WorkerRunner
+
+from services.conversation_history import ConversationRecorder
+
+
+async def run_media_worker(
+    worker: PipelineWorker,
+    recorder: ConversationRecorder | None,
+) -> None:
+    # A runner can return normally after a fatal ErrorFrame or a failed worker.
+    # Only an observed, expected terminal frame counts as normal completion.
+    status = "failed"
+    fatal_error = False
+
+    @worker.event_handler("on_pipeline_error")
+    async def on_pipeline_error(_worker, frame: ErrorFrame):
+        nonlocal status, fatal_error
+        if frame.fatal:
+            fatal_error = True
+            status = "failed"
+
+    @worker.event_handler("on_pipeline_finished")
+    async def on_pipeline_finished(_worker, frame):
+        nonlocal status
+        normal_end = isinstance(frame, EndFrame) or (
+            isinstance(frame, CancelFrame)
+            and frame.reason in {"client_disconnected", "idle timeout"}
+        )
+        if normal_end and not fatal_error:
+            status = "completed"
+
+    runner = WorkerRunner(handle_sigint=False)
+    can_finalize = True
+    try:
+        await runner.add_workers(worker)
+        await runner.run()
+        # Pipecat's runner consumes cancellation while draining its workers.
+        # Preserve the owning server task's cancellation after that cleanup.
+        owner = asyncio.current_task()
+        if owner is not None and owner.cancelling():
+            raise asyncio.CancelledError
+    except GeneratorExit:
+        can_finalize = False  # Synchronously closed coroutines cannot await.
+        raise
+    except BaseException:
+        status = "failed"
+        raise
+    finally:
+        if recorder and can_finalize:
+            await recorder.finish(status=status)

backend/services/pipecat/processors.py

@@ -657,7 +657,7 @@ class ConversationHistoryProcessor(FrameProcessor):
         await super().process_frame(frame, direction)
         await self.push_frame(frame, direction)
         if self._recorder and isinstance(frame, OutputTransportMessageUrgentFrame):
-            await self._recorder.record_transport_message(frame.message)
+            self._recorder.record_transport_message_later(frame.message)
 
 
 class KnowledgeRetrievalProcessor(FrameProcessor):

backend/services/realtime/lifecycle.py

@@ -12,13 +12,33 @@ from loguru import logger
 active_connections: set[object] = set()
 pipeline_tasks: set[asyncio.Task[None]] = set()
 connection_tasks: dict[object, asyncio.Task[None]] = {}
+connection_cleanup_tasks: set[asyncio.Task[None]] = set()
 DEFAULT_CLOSE_GRACE_SECONDS = 10.0
 
 
+async def _disconnect_connection(connection: object) -> None:
+    """Release media even when construction failed before transport startup."""
+    try:
+        async with asyncio.timeout(DEFAULT_CLOSE_GRACE_SECONDS):
+            await connection.disconnect()
+    except Exception as exc:
+        logger.warning(f"Realtime 连接清理失败: {type(exc).__name__}")
+
+
 def _consume_pipeline_result(task: asyncio.Task[None], connection: object) -> None:
     pipeline_tasks.discard(task)
     if connection_tasks.get(connection) is task:
         connection_tasks.pop(connection, None)
+    if connection in active_connections:
+        active_connections.discard(connection)
+        # Run after the pipeline task is done: closed handlers can safely wait
+        # for that task without making pipeline shutdown wait on itself.
+        cleanup = asyncio.create_task(
+            _disconnect_connection(connection),
+            name=f"realtime-disconnect:{getattr(connection, 'pc_id', id(connection))}",
+        )
+        connection_cleanup_tasks.add(cleanup)
+        cleanup.add_done_callback(connection_cleanup_tasks.discard)
     try:
         error = task.exception()
     except asyncio.CancelledError:
@@ -86,11 +106,15 @@ async def shutdown_active_sessions(
 ) -> None:
     connections = list(active_connections)
     if connections:
+        active_connections.difference_update(connections)
         await asyncio.gather(
-            *(connection.disconnect() for connection in connections),
+            *(_disconnect_connection(connection) for connection in connections),
             return_exceptions=True,
         )
-        active_connections.difference_update(connections)
+
+    cleanup_tasks = list(connection_cleanup_tasks)
+    if cleanup_tasks:
+        await asyncio.gather(*cleanup_tasks, return_exceptions=True)
 
     tasks = list(pipeline_tasks)
     if not tasks:
@@ -106,4 +130,3 @@ async def shutdown_active_sessions(
     logger.info(
         f"Realtime 会话清理完成: normal={len(done)} cancelled={len(cancelled)}"
     )
-