feat: add optional bot backchannels to prompt pipelines

backend/contracts/openapi.json

@@ -1198,6 +1198,24 @@
         "title": "Body_upload_test_image_api_test_assets_images_post",
         "type": "object"
       },
+      "BotBackchannelConfig": {
+        "properties": {
+          "delaySecs": {
+            "default": 4.0,
+            "maximum": 15.0,
+            "minimum": 2.0,
+            "title": "Delaysecs",
+            "type": "number"
+          },
+          "enabled": {
+            "default": false,
+            "title": "Enabled",
+            "type": "boolean"
+          }
+        },
+        "title": "BotBackchannelConfig",
+        "type": "object"
+      },
       "BuiltInToolBase": {
         "properties": {
           "description": {
@@ -6406,6 +6424,9 @@
           "bargeIn": {
             "$ref": "#/components/schemas/BargeInConfig"
           },
+          "botBackchannel": {
+            "$ref": "#/components/schemas/BotBackchannelConfig"
+          },
           "turnDetection": {
             "$ref": "#/components/schemas/TurnDetectionConfig"
           },
@@ -6612,6 +6633,69 @@
         ],
         "title": "VariableValueSource",
         "type": "object"
+      },
+      "VoiceRequest": {
+        "properties": {
+          "action": {
+            "enum": [
+              "create_voice",
+              "query_voice"
+            ],
+            "title": "Action",
+            "type": "string"
+          },
+          "apiKey": {
+            "default": "",
+            "maxLength": 4096,
+            "title": "Apikey",
+            "type": "string"
+          },
+          "apiUrl": {
+            "maxLength": 2048,
+            "title": "Apiurl",
+            "type": "string"
+          },
+          "audio": {
+            "default": "",
+            "maxLength": 14000000,
+            "title": "Audio",
+            "type": "string"
+          },
+          "model": {
+            "title": "Model",
+            "type": "string"
+          },
+          "prefix": {
+            "default": "assistant",
+            "pattern": "^[A-Za-z0-9]{1,10}$",
+            "title": "Prefix",
+            "type": "string"
+          },
+          "resourceId": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Resourceid"
+          },
+          "voiceId": {
+            "default": "",
+            "maxLength": 512,
+            "title": "Voiceid",
+            "type": "string"
+          }
+        },
+        "required": [
+          "apiUrl",
+          "model",
+          "action"
+        ],
+        "title": "VoiceRequest",
+        "type": "object"
       }
     },
     "securitySchemes": {
@@ -8483,6 +8567,46 @@
         ]
       }
     },
+    "/api/model-resources/dashscope/voice": {
+      "post": {
+        "description": "Create/query a voice using draft settings and server-held credentials.",
+        "operationId": "manage_dashscope_voice_api_model_resources_dashscope_voice_post",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/VoiceRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {}
+              }
+            },
+            "description": "Successful Response"
+          },
+          "422": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            },
+            "description": "Validation Error"
+          }
+        },
+        "summary": "Manage Dashscope Voice",
+        "tags": [
+          "model-registry"
+        ]
+      }
+    },
     "/api/model-resources/test": {
       "post": {
         "operationId": "test_new_model_resource_api_model_resources_test_post",

backend/schemas.py

@@ -68,7 +68,13 @@ class TurnDetectionConfig(CamelModel):
     silence_timeout_secs: float = Field(default=0.6, ge=0.2, le=5.0)
 
 
+class BotBackchannelConfig(CamelModel):
+    enabled: bool = False
+    delay_secs: float = Field(default=4.0, ge=2.0, le=15.0)
+
+
 class TurnConfig(CamelModel):
+    bot_backchannel: BotBackchannelConfig = Field(default_factory=BotBackchannelConfig)
     barge_in: BargeInConfig = Field(default_factory=BargeInConfig)
     vad: VadConfig = Field(default_factory=VadConfig)
     turn_detection: TurnDetectionConfig = Field(default_factory=TurnDetectionConfig)

backend/services/pipecat/backchannel.py

@@ -0,0 +1,100 @@
+"""Optional bot acknowledgments; never invoke the LLM or change user turns.
+
+Placed immediately before TTS. A pending timer is cancelled on silence or a
+reply. Once handed to TTS, the very short acknowledgment may finish normally.
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Callable
+
+from pipecat.frames.frames import (
+    BotBackchannelFrame, BotStartedSpeakingFrame, BotStoppedSpeakingFrame,
+    CancelFrame, EndFrame, Frame, InterruptionFrame, LLMFullResponseStartFrame,
+    TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame,
+    VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+
+class BotBackchannelProcessor(FrameProcessor):
+    """One short acknowledgment per long user turn, with a global cooldown."""
+
+    def __init__(self, *, delay_secs: float = 4, can_speak: Callable[[], bool]):
+        super().__init__()
+        self._delay = delay_secs
+        self._can_speak = can_speak
+        self._user_turn = False
+        self._voice_active = False
+        self._used = False
+        self._bot_busy = False
+        self._closed = False
+        self._last_sent = float("-inf")
+        self._timer: asyncio.Task | None = None
+
+    async def _cancel_pending(self):
+        timer, self._timer = self._timer, None
+        if timer and timer is not asyncio.current_task():
+            await self.cancel_task(timer)
+
+    def _schedule(self):
+        if (not self._timer and self._user_turn and self._voice_active
+                and not self._used and not self._bot_busy and not self._closed):
+            self._timer = self.create_task(self._acknowledge())
+
+    async def _acknowledge(self):
+        try:
+            await asyncio.sleep(self._delay)
+            if (self._user_turn and self._voice_active and not self._used
+                    and not self._bot_busy and not self._closed and self._can_speak()
+                    and time.monotonic() - self._last_sent >= 10):
+                self._used = True
+                self._last_sent = time.monotonic()
+                # Reserve output before synthesis starts; playback events follow later.
+                self._bot_busy = True
+                await self.push_frame(BotBackchannelFrame(text="嗯。"))
+        finally:
+            self._timer = None
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        # Playback boundaries travel upstream; user/VAD frames can be broadcast
+        # both ways. Handle user signals only on the downstream path.
+        if isinstance(frame, BotStartedSpeakingFrame):
+            self._bot_busy = True
+            await self._cancel_pending()
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            self._bot_busy = False
+            self._schedule()
+        elif isinstance(frame, (EndFrame, CancelFrame)):
+            self._closed = True
+            await self._cancel_pending()
+        elif direction == FrameDirection.DOWNSTREAM:
+            if isinstance(frame, UserStartedSpeakingFrame):
+                self._user_turn, self._used = True, False
+                self._schedule()
+            elif isinstance(frame, VADUserStartedSpeakingFrame):
+                self._voice_active = True
+                self._schedule()
+            elif isinstance(frame, VADUserStoppedSpeakingFrame):
+                self._voice_active = False
+                await self._cancel_pending()
+            elif isinstance(frame, UserStoppedSpeakingFrame):
+                self._user_turn = False
+                await self._cancel_pending()
+            elif isinstance(frame, (LLMFullResponseStartFrame, TTSSpeakFrame)):
+                self._bot_busy = True
+                await self._cancel_pending()
+            elif isinstance(frame, InterruptionFrame):
+                # User-start strategies broadcast an interruption after the
+                # turn-start signal, even when no bot audio was playing.
+                await self._cancel_pending()
+                self._bot_busy = False
+                self._schedule()
+        await self.push_frame(frame, direction)
+
+    async def cleanup(self):
+        self._closed = True
+        await self._cancel_pending()
+        await super().cleanup()

backend/services/pipecat/pipeline.py

@@ -14,6 +14,7 @@ from models import AssistantConfig
 from services.brains import Brain, BrainRuntime, build_brain
 from services.brains.base import RealtimeBrainRuntime
 from services.conversation_history import ConversationRecorder
+from services.pipecat.backchannel import BotBackchannelProcessor
 from services.pipecat.call_lifecycle import (
     CallEndCoordinator,
     EndCallAfterSpeechProcessor,
@@ -722,6 +723,14 @@ async def run_pipeline(
         protocol_adapter.inference_processors() if protocol_adapter else []
     )
     protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
+    backchannel_config = cfg.turnConfig.get("bot_backchannel", cfg.turnConfig.get("botBackchannel", {}))
+    backchannel_processors = []
+    if cfg.type == "prompt" and backchannel_config.get("enabled", False):
+        backchannel_processors.append(BotBackchannelProcessor(
+            delay_secs=float(backchannel_config.get("delay_secs", backchannel_config.get("delaySecs", 4))),
+            can_speak=lambda: (call_end.accepts_input and not call_end.ending
+                               and input_state["enabled"] and not tool_interruption_strategy.is_muted),
+        ))
     pipeline = Pipeline(
         [
             transport.input(),
@@ -740,6 +749,7 @@ async def run_pipeline(
             # Pipecat commits the generated prefix immediately instead of
             # waiting for a TTS provider to emit spoken-text/timestamp frames.
             assistant_aggregator,
+            *backchannel_processors,
             tts_processor,
             EndCallAfterSpeechProcessor(call_end),
             ConversationHistoryProcessor(recorder),

backend/tests/test_bot_backchannel.py

@@ -0,0 +1,156 @@
+import asyncio
+import unittest
+from unittest.mock import AsyncMock, patch
+
+from pipecat.frames.frames import (
+    BotBackchannelFrame, BotStoppedSpeakingFrame, CancelFrame,
+    InterruptionFrame, LLMFullResponseStartFrame, TTSSpeakFrame,
+    UserStartedSpeakingFrame, UserStoppedSpeakingFrame,
+    VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from schemas import TurnConfig
+from services.pipecat.backchannel import BotBackchannelProcessor
+
+
+class BotBackchannelTests(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self):
+        self.allowed = True
+        self.frames = []
+        self.processor = BotBackchannelProcessor(delay_secs=0.01, can_speak=lambda: self.allowed)
+        self.processor.create_task = asyncio.create_task
+        async def cancel(task):
+            task.cancel()
+            try:
+                await task
+            except asyncio.CancelledError:
+                pass
+        self.processor.cancel_task = cancel
+        async def push(frame, direction=FrameDirection.DOWNSTREAM):
+            self.frames.append(frame)
+        self.processor.push_frame = push
+        self.patch = patch.object(FrameProcessor, "process_frame", AsyncMock())
+        self.patch.start()
+
+    async def asyncTearDown(self):
+        await self.processor.cleanup()
+        self.patch.stop()
+
+    async def send(self, frame):
+        await self.processor.process_frame(frame, FrameDirection.DOWNSTREAM)
+
+    async def start_turn(self):
+        await self.send(VADUserStartedSpeakingFrame())
+        await self.send(UserStartedSpeakingFrame())
+        await self.send(InterruptionFrame())
+
+    def acknowledgments(self):
+        return [frame for frame in self.frames if isinstance(frame, BotBackchannelFrame)]
+
+    async def test_long_turn_once_and_excludes_context(self):
+        await self.start_turn()
+        await asyncio.sleep(0.03)
+        self.assertEqual(len(self.acknowledgments()), 1)
+        self.assertFalse(self.acknowledgments()[0].append_to_context)
+        self.assertFalse(self.acknowledgments()[0].interruptible)
+        await self.send(BotStoppedSpeakingFrame())
+        await self.send(VADUserStoppedSpeakingFrame())
+        await self.send(VADUserStartedSpeakingFrame())
+        await asyncio.sleep(0.03)
+        self.assertEqual(len(self.acknowledgments()), 1)
+
+    async def test_short_turn_does_not_acknowledge(self):
+        await self.start_turn()
+        await self.send(UserStoppedSpeakingFrame())
+        await asyncio.sleep(0.03)
+        self.assertFalse(self.acknowledgments())
+
+    async def test_silence_cancels_pending(self):
+        await self.start_turn()
+        await self.send(VADUserStoppedSpeakingFrame())
+        await asyncio.sleep(0.03)
+        self.assertFalse(self.acknowledgments())
+
+    async def test_formal_reply_cancels_pending(self):
+        await self.start_turn()
+        await self.send(LLMFullResponseStartFrame())
+        await asyncio.sleep(0.03)
+        self.assertFalse(self.acknowledgments())
+
+    async def test_greeting_reserves_output(self):
+        await self.send(TTSSpeakFrame("欢迎"))
+        await self.send(UserStartedSpeakingFrame())
+        await self.send(VADUserStartedSpeakingFrame())
+        await asyncio.sleep(0.03)
+        self.assertFalse(self.acknowledgments())
+
+    async def test_closing_or_muted_blocks_timer(self):
+        await self.start_turn()
+        self.allowed = False
+        await asyncio.sleep(0.03)
+        self.assertFalse(self.acknowledgments())
+
+    async def test_cancel_closes_controller(self):
+        await self.start_turn()
+        await self.send(CancelFrame())
+        await asyncio.sleep(0.03)
+        self.assertFalse(self.acknowledgments())
+        self.assertIsNone(self.processor._timer)
+
+    async def test_cooldown_applies_between_turns(self):
+        await self.start_turn()
+        await asyncio.sleep(0.03)
+        await self.send(UserStoppedSpeakingFrame())
+        await self.send(BotStoppedSpeakingFrame())
+        await self.start_turn()
+        await asyncio.sleep(0.03)
+        self.assertEqual(len(self.acknowledgments()), 1)
+
+    def test_old_configuration_defaults_off_and_alias_roundtrip(self):
+        self.assertFalse(TurnConfig().bot_backchannel.enabled)
+        config = TurnConfig.model_validate({"botBackchannel": {"enabled": True, "delaySecs": 5}})
+        self.assertEqual(config.bot_backchannel.delay_secs, 5)
+
+
+class BackchannelPipelineTest(unittest.IsolatedAsyncioTestCase):
+    async def test_native_pipeline_delivers_backchannel_without_assistant_context(self):
+        """Exercise real frame queues and the project's pre-TTS aggregator."""
+        from pipecat.frames.frames import EndFrame
+        from pipecat.pipeline.pipeline import Pipeline
+        from pipecat.pipeline.worker import PipelineWorker
+        from pipecat.processors.aggregators.llm_context import LLMContext
+        from services.pipecat.processors import PassthroughLLMAssistantAggregator
+        from services.pipecat.pipeline_runner import run_media_worker
+
+        received = asyncio.Event()
+        context = LLMContext([])
+        class Sink(FrameProcessor):
+            async def process_frame(self, frame, direction):
+                await super().process_frame(frame, direction)
+                if isinstance(frame, BotBackchannelFrame):
+                    received.set()
+                await self.push_frame(frame, direction)
+
+        controller = BotBackchannelProcessor(delay_secs=0.01, can_speak=lambda: True)
+        worker = PipelineWorker(Pipeline([
+            PassthroughLLMAssistantAggregator(context), controller, Sink(),
+        ]), enable_rtvi=False)
+        @worker.event_handler("on_pipeline_started")
+        async def start(_worker, _frame):
+            for frame in (VADUserStartedSpeakingFrame(), UserStartedSpeakingFrame(), InterruptionFrame()):
+                await worker.queue_frame(frame)
+
+        run = asyncio.create_task(run_media_worker(worker, None))
+        try:
+            await asyncio.wait_for(received.wait(), 3)
+            self.assertEqual(context.messages, [])
+            await worker.queue_frame(EndFrame())
+            await asyncio.wait_for(run, 3)
+            self.assertIsNone(controller._timer)
+        finally:
+            if not run.done():
+                run.cancel()
+                try:
+                    await run
+                except asyncio.CancelledError:
+                    pass