feat: add optional bot backchannels to prompt pipelines

backend/services/pipecat/backchannel.py

@@ -0,0 +1,100 @@
+"""Optional bot acknowledgments; never invoke the LLM or change user turns.
+
+Placed immediately before TTS. A pending timer is cancelled on silence or a
+reply. Once handed to TTS, the very short acknowledgment may finish normally.
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Callable
+
+from pipecat.frames.frames import (
+    BotBackchannelFrame, BotStartedSpeakingFrame, BotStoppedSpeakingFrame,
+    CancelFrame, EndFrame, Frame, InterruptionFrame, LLMFullResponseStartFrame,
+    TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame,
+    VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+
+class BotBackchannelProcessor(FrameProcessor):
+    """One short acknowledgment per long user turn, with a global cooldown."""
+
+    def __init__(self, *, delay_secs: float = 4, can_speak: Callable[[], bool]):
+        super().__init__()
+        self._delay = delay_secs
+        self._can_speak = can_speak
+        self._user_turn = False
+        self._voice_active = False
+        self._used = False
+        self._bot_busy = False
+        self._closed = False
+        self._last_sent = float("-inf")
+        self._timer: asyncio.Task | None = None
+
+    async def _cancel_pending(self):
+        timer, self._timer = self._timer, None
+        if timer and timer is not asyncio.current_task():
+            await self.cancel_task(timer)
+
+    def _schedule(self):
+        if (not self._timer and self._user_turn and self._voice_active
+                and not self._used and not self._bot_busy and not self._closed):
+            self._timer = self.create_task(self._acknowledge())
+
+    async def _acknowledge(self):
+        try:
+            await asyncio.sleep(self._delay)
+            if (self._user_turn and self._voice_active and not self._used
+                    and not self._bot_busy and not self._closed and self._can_speak()
+                    and time.monotonic() - self._last_sent >= 10):
+                self._used = True
+                self._last_sent = time.monotonic()
+                # Reserve output before synthesis starts; playback events follow later.
+                self._bot_busy = True
+                await self.push_frame(BotBackchannelFrame(text="嗯。"))
+        finally:
+            self._timer = None
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        # Playback boundaries travel upstream; user/VAD frames can be broadcast
+        # both ways. Handle user signals only on the downstream path.
+        if isinstance(frame, BotStartedSpeakingFrame):
+            self._bot_busy = True
+            await self._cancel_pending()
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            self._bot_busy = False
+            self._schedule()
+        elif isinstance(frame, (EndFrame, CancelFrame)):
+            self._closed = True
+            await self._cancel_pending()
+        elif direction == FrameDirection.DOWNSTREAM:
+            if isinstance(frame, UserStartedSpeakingFrame):
+                self._user_turn, self._used = True, False
+                self._schedule()
+            elif isinstance(frame, VADUserStartedSpeakingFrame):
+                self._voice_active = True
+                self._schedule()
+            elif isinstance(frame, VADUserStoppedSpeakingFrame):
+                self._voice_active = False
+                await self._cancel_pending()
+            elif isinstance(frame, UserStoppedSpeakingFrame):
+                self._user_turn = False
+                await self._cancel_pending()
+            elif isinstance(frame, (LLMFullResponseStartFrame, TTSSpeakFrame)):
+                self._bot_busy = True
+                await self._cancel_pending()
+            elif isinstance(frame, InterruptionFrame):
+                # User-start strategies broadcast an interruption after the
+                # turn-start signal, even when no bot audio was playing.
+                await self._cancel_pending()
+                self._bot_busy = False
+                self._schedule()
+        await self.push_frame(frame, direction)
+
+    async def cleanup(self):
+        self._closed = True
+        await self._cancel_pending()
+        await super().cleanup()

backend/services/pipecat/pipeline.py

@@ -14,6 +14,7 @@ from models import AssistantConfig
 from services.brains import Brain, BrainRuntime, build_brain
 from services.brains.base import RealtimeBrainRuntime
 from services.conversation_history import ConversationRecorder
+from services.pipecat.backchannel import BotBackchannelProcessor
 from services.pipecat.call_lifecycle import (
     CallEndCoordinator,
     EndCallAfterSpeechProcessor,
@@ -722,6 +723,14 @@ async def run_pipeline(
         protocol_adapter.inference_processors() if protocol_adapter else []
     )
     protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
+    backchannel_config = cfg.turnConfig.get("bot_backchannel", cfg.turnConfig.get("botBackchannel", {}))
+    backchannel_processors = []
+    if cfg.type == "prompt" and backchannel_config.get("enabled", False):
+        backchannel_processors.append(BotBackchannelProcessor(
+            delay_secs=float(backchannel_config.get("delay_secs", backchannel_config.get("delaySecs", 4))),
+            can_speak=lambda: (call_end.accepts_input and not call_end.ending
+                               and input_state["enabled"] and not tool_interruption_strategy.is_muted),
+        ))
     pipeline = Pipeline(
         [
             transport.input(),
@@ -740,6 +749,7 @@ async def run_pipeline(
             # Pipecat commits the generated prefix immediately instead of
             # waiting for a TTS provider to emit spoken-text/timestamp frames.
             assistant_aggregator,
+            *backchannel_processors,
             tts_processor,
             EndCallAfterSpeechProcessor(call_end),
             ConversationHistoryProcessor(recorder),