feat: implement OpenAI-compatible Realtime API with authentication and management features

- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections.
- Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys.
- Implemented authentication mechanisms for API keys and client secrets.
- Enhanced environment configuration with new secrets for Realtime API.
- Created OpenAIRealtime session management and event processing for real-time interactions.
- Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.

backend/services/openai_realtime/__init__.py

@@ -0,0 +1 @@
+"""OpenAI Realtime-compatible northbound protocol adapter."""

backend/services/openai_realtime/auth.py

@@ -0,0 +1,181 @@
+"""Authentication for the public OpenAI-compatible Realtime API."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import secrets
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+from uuid import uuid4
+
+import settings
+from db.models import RealtimeApiKey
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+
+API_KEY_PREFIX = "sk-rt-"
+CLIENT_SECRET_PREFIX = "ek-rt-"
+
+
+class RealtimeAuthError(ValueError):
+    """Raised when a public Realtime credential cannot be accepted."""
+
+
+@dataclass(frozen=True)
+class RealtimeCredential:
+    api_key_id: str
+    assistant_id: str | None = None
+    session: dict[str, Any] | None = None
+    safety_identifier_hash: str | None = None
+    ephemeral: bool = False
+
+
+def _b64encode(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
+
+
+def _b64decode(value: str) -> bytes:
+    padding = "=" * (-len(value) % 4)
+    return base64.urlsafe_b64decode(f"{value}{padding}".encode("ascii"))
+
+
+def _peppered_digest(value: str) -> str:
+    return hmac.new(
+        settings.REALTIME_TOKEN_SECRET.encode("utf-8"),
+        value.encode("utf-8"),
+        hashlib.sha256,
+    ).hexdigest()
+
+
+def _token_signature(encoded_payload: str) -> str:
+    digest = hmac.new(
+        settings.REALTIME_TOKEN_SECRET.encode("utf-8"),
+        encoded_payload.encode("ascii"),
+        hashlib.sha256,
+    ).digest()
+    return _b64encode(digest)
+
+
+def hash_safety_identifier(value: str | None) -> str | None:
+    normalized = (value or "").strip()
+    if not normalized:
+        return None
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+
+
+def create_api_key_value() -> tuple[str, str, str, str]:
+    key_id = f"rtkey_{uuid4().hex}"
+    random_secret = secrets.token_urlsafe(32)
+    value = f"{API_KEY_PREFIX}{key_id[6:18]}.{random_secret}"
+    return key_id, value, value[:24], _peppered_digest(value)
+
+
+def create_client_secret(
+    *,
+    api_key_id: str,
+    assistant_id: str,
+    session: dict[str, Any],
+    safety_identifier_hash: str | None,
+) -> tuple[str, int]:
+    now = int(time.time())
+    expires_at = now + settings.REALTIME_CLIENT_SECRET_TTL_SECONDS
+    payload = {
+        "sub": api_key_id,
+        "assistant_id": assistant_id,
+        "session": session,
+        "safety_identifier_hash": safety_identifier_hash,
+        "iat": now,
+        "exp": expires_at,
+        "jti": f"rtcs_{uuid4().hex}",
+    }
+    encoded = _b64encode(
+        json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode(
+            "utf-8"
+        )
+    )
+    return f"{CLIENT_SECRET_PREFIX}{encoded}.{_token_signature(encoded)}", expires_at
+
+
+async def _active_api_key(
+    session: AsyncSession,
+    *,
+    key_id: str | None = None,
+    key_prefix: str | None = None,
+) -> RealtimeApiKey | None:
+    if key_id:
+        row = await session.get(RealtimeApiKey, key_id)
+    elif key_prefix:
+        row = (
+            await session.execute(
+                select(RealtimeApiKey).where(
+                    RealtimeApiKey.key_prefix == key_prefix
+                )
+            )
+        ).scalar_one_or_none()
+    else:
+        return None
+    now = datetime.now(UTC)
+    if row is None or row.revoked_at is not None:
+        return None
+    if row.expires_at is not None and row.expires_at <= now:
+        return None
+    return row
+
+
+async def authenticate_bearer(
+    session: AsyncSession,
+    token: str,
+) -> RealtimeCredential:
+    if token.startswith(API_KEY_PREFIX):
+        prefix = token[:24]
+        row = await _active_api_key(session, key_prefix=prefix)
+        if row is None or not hmac.compare_digest(
+            row.key_hash,
+            _peppered_digest(token),
+        ):
+            raise RealtimeAuthError("Invalid or expired API key")
+        row.last_used_at = datetime.now(UTC)
+        await session.commit()
+        return RealtimeCredential(api_key_id=row.id)
+
+    if not token.startswith(CLIENT_SECRET_PREFIX):
+        raise RealtimeAuthError("Unsupported Realtime credential")
+    compact = token[len(CLIENT_SECRET_PREFIX) :]
+    try:
+        encoded, signature = compact.rsplit(".", 1)
+    except ValueError as exc:
+        raise RealtimeAuthError("Invalid client secret") from exc
+    if not hmac.compare_digest(_token_signature(encoded), signature):
+        raise RealtimeAuthError("Invalid client secret")
+    try:
+        payload = json.loads(_b64decode(encoded))
+    except (ValueError, json.JSONDecodeError) as exc:
+        raise RealtimeAuthError("Invalid client secret") from exc
+    if int(payload.get("exp", 0)) < int(time.time()):
+        raise RealtimeAuthError("Client secret expired")
+    api_key_id = str(payload.get("sub") or "")
+    row = await _active_api_key(session, key_id=api_key_id)
+    if row is None:
+        raise RealtimeAuthError("Parent API key is no longer active")
+    row.last_used_at = datetime.now(UTC)
+    await session.commit()
+    return RealtimeCredential(
+        api_key_id=api_key_id,
+        assistant_id=str(payload.get("assistant_id") or "") or None,
+        session=(payload.get("session") if isinstance(payload.get("session"), dict) else None),
+        safety_identifier_hash=str(payload.get("safety_identifier_hash") or "") or None,
+        ephemeral=True,
+    )
+
+
+def bearer_from_authorization(value: str | None) -> str:
+    scheme, _, token = (value or "").partition(" ")
+    if scheme.lower() != "bearer" or not token.strip():
+        raise RealtimeAuthError("Missing Bearer credential")
+    return token.strip()

backend/services/openai_realtime/bridge.py

@@ -0,0 +1,916 @@
+"""Translate OpenAI Realtime events to the project's neutral pipeline messages."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import binascii
+import json
+from typing import Any
+from uuid import uuid4
+
+from pipecat.audio.utils import create_stream_resampler
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    InputAudioRawFrame,
+    InputTransportMessageFrame,
+    LLMContextFrame,
+    OutputAudioRawFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
+    StartFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+    VADParamsUpdateFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+from services.input_assets import store_input_image
+from services.openai_realtime.events import (
+    SUPPORTED_CAPABILITIES,
+    RealtimeEventError,
+    error_event,
+    normalize_turn_detection,
+    require_event,
+)
+from services.openai_realtime.session import OpenAIRealtimeSession
+from services.pipecat.turn_config import create_vad_params
+from services.realtime.protocol import PipelineProtocolRuntime
+
+
+MAX_BUFFERED_AUDIO_BYTES = 24000 * 2 * 120
+
+
+def _event_id() -> str:
+    return f"event_{uuid4().hex}"
+
+
+def _server_event(event_type: str, **payload: Any) -> dict[str, Any]:
+    return {"type": event_type, "event_id": _event_id(), **payload}
+
+
+def _decode_base64(value: object, *, param: str) -> bytes:
+    if not isinstance(value, str) or not value:
+        raise RealtimeEventError(f"{param} must be non-empty Base64", param=param)
+    try:
+        return base64.b64decode(value, validate=True)
+    except (ValueError, binascii.Error) as exc:
+        raise RealtimeEventError(f"{param} is not valid Base64", param=param) from exc
+
+
+def _decode_data_url(value: object) -> bytes:
+    if not isinstance(value, str) or not value.startswith("data:image/"):
+        raise RealtimeEventError(
+            "input_image only supports image Data URLs",
+            code="unsupported_content_type",
+            param="item.content",
+        )
+    header, separator, encoded = value.partition(",")
+    if not separator or ";base64" not in header:
+        raise RealtimeEventError(
+            "input_image Data URL must use Base64 encoding",
+            param="item.content",
+        )
+    return _decode_base64(encoded, param="item.content")
+
+
+class OpenAIRealtimeInputProcessor(FrameProcessor):
+    def __init__(self, bridge: "OpenAIRealtimeBridge") -> None:
+        super().__init__()
+        self._bridge = bridge
+        self._started = False
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        if isinstance(frame, StartFrame) and not self._started:
+            self._started = True
+            # Initialize every downstream processor/serializer before the first
+            # business event is sent to the client.
+            await self.push_frame(frame, direction)
+            await self._bridge.emit(
+                _server_event(
+                    "session.created",
+                    session=self._bridge.session.public_value(),
+                )
+            )
+            return
+
+        if isinstance(frame, InputAudioRawFrame):
+            if (
+                self._bridge.session.external_turn_control
+                and frame.transport_source != "openai-committed"
+            ):
+                try:
+                    self._bridge.buffer_audio(
+                        frame.audio,
+                        sample_rate=frame.sample_rate,
+                        num_channels=frame.num_channels,
+                    )
+                except RealtimeEventError as exc:
+                    await self._bridge.emit(error_event(exc))
+                return
+            await self.push_frame(frame, direction)
+            return
+
+        if not isinstance(frame, InputTransportMessageFrame):
+            await self.push_frame(frame, direction)
+            return
+        if isinstance(frame.message, dict) and frame.message.pop(
+            "_pipeline_internal", False
+        ):
+            await self.push_frame(frame, direction)
+            return
+        if not self._bridge.client_ready_sent:
+            self._bridge.client_ready_sent = True
+            await self.push_frame(
+                InputTransportMessageFrame(
+                    message={"type": "client-ready"}
+                ),
+                direction,
+            )
+        try:
+            await self._bridge.handle_client_event(require_event(frame.message))
+        except RealtimeEventError as exc:
+            await self._bridge.emit(error_event(exc))
+        except Exception as exc:  # noqa: BLE001 - event errors must stay connection-local
+            client_event_id = (
+                str(frame.message.get("event_id") or "") or None
+                if isinstance(frame.message, dict)
+                else None
+            )
+            await self._bridge.emit(
+                error_event(
+                    RealtimeEventError(
+                        f"Event processing failed: {exc}",
+                        code="event_processing_error",
+                        event_id=client_event_id,
+                    )
+                )
+            )
+
+
+class OpenAIRealtimeOutputProcessor(FrameProcessor):
+    def __init__(self, bridge: "OpenAIRealtimeBridge") -> None:
+        super().__init__()
+        self._bridge = bridge
+        self._audio_resampler = create_stream_resampler()
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        if isinstance(frame, BotStartedSpeakingFrame):
+            if not self._bridge.session.active_response_id:
+                for event in self._bridge._start_assistant_output():
+                    await self.push_frame(
+                        OutputTransportMessageUrgentFrame(message=event)
+                    )
+            await self.push_frame(frame, direction)
+            return
+        if isinstance(frame, BotStoppedSpeakingFrame):
+            if self._bridge.session.active_response_id:
+                for event in self._bridge._end_assistant_output(False):
+                    await self.push_frame(
+                        OutputTransportMessageUrgentFrame(message=event)
+                    )
+            await self.push_frame(frame, direction)
+            return
+        if isinstance(frame, UserStartedSpeakingFrame):
+            item_id = f"item_{uuid4().hex}"
+            self._bridge.session.active_input_item_id = item_id
+            await self.push_frame(
+                OutputTransportMessageUrgentFrame(
+                    message=_server_event(
+                        "input_audio_buffer.speech_started",
+                        audio_start_ms=0,
+                        item_id=item_id,
+                    )
+                )
+            )
+            await self.push_frame(frame, direction)
+            return
+        if isinstance(frame, UserStoppedSpeakingFrame):
+            item_id = (
+                self._bridge.session.active_input_item_id
+                or f"item_{uuid4().hex}"
+            )
+            self._bridge.session.active_input_item_id = None
+            await self.push_frame(
+                OutputTransportMessageUrgentFrame(
+                    message=_server_event(
+                        "input_audio_buffer.speech_stopped",
+                        audio_end_ms=0,
+                        item_id=item_id,
+                    )
+                )
+            )
+            await self.push_frame(frame, direction)
+            return
+        if isinstance(frame, OutputAudioRawFrame):
+            if not self._bridge.session.output_is_audio:
+                return
+            if self._bridge.channel == "websocket":
+                if not self._bridge.session.active_response_id:
+                    for event in self._bridge._start_assistant_output():
+                        await self.push_frame(
+                            OutputTransportMessageUrgentFrame(message=event)
+                        )
+                response_id, item_id = self._bridge.session.begin_response()
+                audio = frame.audio
+                if frame.sample_rate != 24000:
+                    audio = await self._audio_resampler.resample(
+                        audio,
+                        frame.sample_rate,
+                        24000,
+                    )
+                await self.push_frame(
+                    OutputTransportMessageUrgentFrame(
+                        message=_server_event(
+                            "response.output_audio.delta",
+                            response_id=response_id,
+                            item_id=item_id,
+                            output_index=0,
+                            content_index=0,
+                            delta=base64.b64encode(audio).decode("ascii"),
+                        )
+                    )
+                )
+                return
+            await self.push_frame(frame, direction)
+            return
+
+        if not isinstance(
+            frame,
+            (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame),
+        ):
+            await self.push_frame(frame, direction)
+            return
+        message = frame.message
+        if not isinstance(message, dict):
+            return
+        translated = self._bridge.translate_server_message(message)
+        for event in translated:
+            await self.push_frame(
+                OutputTransportMessageUrgentFrame(message=event),
+                direction,
+            )
+
+
+class OpenAIResponseGateProcessor(FrameProcessor):
+    """Hold automatic inference when create_response is disabled or PTT is active."""
+
+    def __init__(self, bridge: "OpenAIRealtimeBridge") -> None:
+        super().__init__()
+        self._bridge = bridge
+        self._held: list[tuple[Any, FrameDirection]] = []
+        self._allow_next = False
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        if direction != FrameDirection.DOWNSTREAM or not isinstance(
+            frame, LLMContextFrame
+        ):
+            await self.push_frame(frame, direction)
+            return
+        turn = self._bridge.session.turn_detection
+        auto_response = bool(turn and turn.get("create_response", True))
+        if auto_response or self._allow_next:
+            self._allow_next = False
+            await self.push_frame(frame, direction)
+            return
+        self._held.append((frame, direction))
+
+    async def allow_one_response(self) -> None:
+        if self._held:
+            held, self._held = self._held, []
+            for frame, direction in held:
+                await self.push_frame(frame, direction)
+            return
+        self._allow_next = True
+
+
+class OpenAIRealtimeBridge:
+    """One adapter instance is owned by exactly one public Realtime session."""
+
+    def __init__(self, session: OpenAIRealtimeSession, *, channel: str) -> None:
+        self.session = session
+        self.channel = channel
+        self._runtime: PipelineProtocolRuntime | None = None
+        self.client_ready_sent = False
+        self._input = OpenAIRealtimeInputProcessor(self)
+        self._inference = OpenAIResponseGateProcessor(self)
+        self._output = OpenAIRealtimeOutputProcessor(self)
+
+    def input_processors(self) -> list[FrameProcessor]:
+        return [self._input]
+
+    def output_processors(self) -> list[FrameProcessor]:
+        return [self._output]
+
+    def inference_processors(self) -> list[FrameProcessor]:
+        return [self._inference]
+
+    async def bind(self, runtime: PipelineProtocolRuntime) -> None:
+        self._runtime = runtime
+        await runtime.set_external_turn_control(self.session.external_turn_control)
+        turn = self.session.turn_detection
+        if turn:
+            await runtime.set_response_interruption(
+                bool(turn.get("interrupt_response", True))
+            )
+
+    @property
+    def runtime(self) -> PipelineProtocolRuntime:
+        if self._runtime is None:
+            raise RuntimeError("OpenAI Realtime bridge is not bound to a pipeline")
+        return self._runtime
+
+    async def emit(self, event: dict[str, Any]) -> None:
+        await self._input.push_frame(
+            OutputTransportMessageUrgentFrame(message=event)
+        )
+
+    async def queue_internal_message(self, message: dict[str, Any]) -> None:
+        await self.runtime.queue_frame(
+            InputTransportMessageFrame(
+                message={**message, "_pipeline_internal": True}
+            )
+        )
+
+    def buffer_audio(
+        self,
+        audio: bytes,
+        *,
+        sample_rate: int,
+        num_channels: int,
+    ) -> None:
+        if self.session.buffered_audio_bytes + len(audio) > MAX_BUFFERED_AUDIO_BYTES:
+            self.clear_buffered_audio()
+            raise RealtimeEventError(
+                "input_audio_buffer exceeds the 120 second MVP limit",
+                code="input_audio_buffer_too_large",
+            )
+        self.session.audio_chunks.append((audio, sample_rate, num_channels))
+        self.session.buffered_audio_bytes += len(audio)
+
+    def clear_buffered_audio(self) -> None:
+        self.session.audio_chunks.clear()
+        self.session.buffered_audio_bytes = 0
+        self.session.audio_committed = False
+
+    async def handle_client_event(self, event: dict[str, Any]) -> None:
+        event_type = str(event["type"])
+        if event_type == "session.update":
+            await self._update_session(event)
+        elif event_type == "conversation.item.create":
+            await self._create_item(event)
+        elif event_type == "conversation.item.truncate":
+            await self.runtime.cancel_response()
+            await self.emit(
+                _server_event(
+                    "conversation.item.truncated",
+                    item_id=event.get("item_id"),
+                    content_index=int(event.get("content_index") or 0),
+                    audio_end_ms=int(event.get("audio_end_ms") or 0),
+                )
+            )
+        elif event_type == "input_audio_buffer.append":
+            await self._append_audio(event)
+        elif event_type == "input_audio_buffer.commit":
+            await self._commit_audio()
+        elif event_type == "input_audio_buffer.clear":
+            self.clear_buffered_audio()
+            await self.runtime.clear_audio()
+            await self.emit(_server_event("input_audio_buffer.cleared"))
+        elif event_type == "output_audio_buffer.clear":
+            await self.runtime.cancel_response()
+            await self._finish_response(status="cancelled")
+            await self.emit(_server_event("output_audio_buffer.cleared"))
+        elif event_type == "response.create":
+            await self._create_response(event)
+        elif event_type == "response.cancel":
+            await self.runtime.cancel_response()
+            await self._finish_response(status="cancelled")
+        elif event_type == "x.interactive_media.capabilities.update":
+            await self._update_capabilities(event)
+        elif event_type == "x.interactive_media.session.variables.update":
+            await self._update_variables(event)
+
+    async def _update_session(self, event: dict[str, Any]) -> None:
+        update = event.get("session")
+        if not isinstance(update, dict):
+            raise RealtimeEventError("session.update requires session", param="session")
+        locked = {"model", "instructions", "voice", "tools", "tool_choice"}
+        changed_locked = sorted(locked.intersection(update))
+        audio = update.get("audio")
+        if isinstance(audio, dict):
+            output = audio.get("output")
+            if isinstance(output, dict) and "voice" in output:
+                changed_locked.append("audio.output.voice")
+        if changed_locked:
+            raise RealtimeEventError(
+                f"Assistant-owned session fields cannot be changed: {', '.join(changed_locked)}",
+                code="immutable_session_field",
+                param=changed_locked[0],
+                event_id=str(event.get("event_id") or "") or None,
+            )
+
+        modalities = update.get("output_modalities")
+        if modalities is not None:
+            if modalities not in (["audio"], ["text"]):
+                raise RealtimeEventError(
+                    'output_modalities must be ["audio"] or ["text"]',
+                    param="session.output_modalities",
+                )
+            self.session.output_modalities = list(modalities)
+
+        marker = object()
+        turn_detection: object = marker
+        if isinstance(audio, dict) and isinstance(audio.get("input"), dict):
+            turn_detection = audio["input"].get("turn_detection", marker)
+        if turn_detection is marker:
+            turn_detection = update.get("turn_detection", marker)
+        if turn_detection is not marker:
+            self.session.turn_detection = self._validate_turn_detection(turn_detection)
+            await self.runtime.set_external_turn_control(
+                self.session.external_turn_control
+            )
+            if self.session.turn_detection:
+                await self.runtime.set_response_interruption(
+                    bool(self.session.turn_detection.get("interrupt_response", True))
+                )
+            if self.session.turn_detection:
+                turn = self.session.turn_detection
+                config = {
+                    "vad": {
+                        "confidence": turn["threshold"],
+                        "start_secs": max(0.05, turn["prefix_padding_ms"] / 1000),
+                        "stop_secs": 0.2,
+                    },
+                    "turn_detection": {
+                        "strategy": "silence",
+                        "silence_timeout_secs": turn["silence_duration_ms"] / 1000,
+                    },
+                }
+                await self.runtime.queue_frame(
+                    VADParamsUpdateFrame(params=create_vad_params(config))
+                )
+        await self.emit(
+            _server_event("session.updated", session=self.session.public_value())
+        )
+
+    def _validate_turn_detection(self, value: object) -> dict[str, Any] | None:
+        return normalize_turn_detection(value)
+
+    async def _create_item(self, event: dict[str, Any]) -> None:
+        item = event.get("item")
+        if not isinstance(item, dict):
+            raise RealtimeEventError("conversation.item.create requires item", param="item")
+        item_type = item.get("type")
+        if item_type == "function_call_output":
+            call_id = str(item.get("call_id") or "")
+            if not call_id:
+                raise RealtimeEventError("function_call_output requires call_id", param="item.call_id")
+            raw_output = item.get("output")
+            try:
+                data = json.loads(raw_output) if isinstance(raw_output, str) else raw_output
+            except json.JSONDecodeError:
+                data = raw_output
+            await self.queue_internal_message(
+                {
+                    "type": "client-tool-result",
+                    "tool_call_id": call_id,
+                    "status": "ok",
+                    "data": data,
+                }
+            )
+            await self._emit_item_ack(item)
+            return
+        if item_type != "message" or item.get("role") != "user":
+            raise RealtimeEventError(
+                "Only user messages and function_call_output items are accepted",
+                code="unsupported_item_type",
+                param="item.type",
+            )
+
+        wire_parts: list[dict[str, Any]] = []
+        for part in item.get("content") or []:
+            if not isinstance(part, dict):
+                continue
+            if part.get("type") == "input_text":
+                text = str(part.get("text") or "").strip()
+                if text:
+                    wire_parts.append({"type": "input_text", "text": text})
+            elif part.get("type") == "input_image":
+                if self.session.config.runtimeMode == "realtime":
+                    raise RealtimeEventError(
+                        "input_image is not supported by the assistant's realtime runtime",
+                        code="unsupported_content_type",
+                    )
+                if not self.session.vision_enabled:
+                    raise RealtimeEventError(
+                        "This assistant has not enabled image input",
+                        code="unsupported_content_type",
+                    )
+                data = _decode_data_url(part.get("image_url"))
+                stored = await asyncio.to_thread(store_input_image, data)
+                wire_parts.append(
+                    {
+                        "type": "input_image",
+                        "source": {
+                            "type": "uploaded_asset",
+                            "asset_token": stored.token,
+                        },
+                    }
+                )
+            else:
+                raise RealtimeEventError(
+                    f"Unsupported content part: {part.get('type')}",
+                    code="unsupported_content_type",
+                    param="item.content",
+                )
+        if not wire_parts:
+            raise RealtimeEventError("User item has no supported content", param="item.content")
+        item_id = str(item.get("id") or f"item_{uuid4().hex}")
+        public_item = {**item, "id": item_id, "status": "completed"}
+        self.session.pending_item = {
+            "type": "user-input",
+            "schema_version": 1,
+            "input_id": item_id,
+            "parts": wire_parts,
+            "options": {
+                "run_immediately": self.session.config.runtimeMode != "realtime",
+                "interrupt": True,
+            },
+        }
+        await self._emit_item_ack(public_item)
+
+    async def _emit_item_ack(self, item: dict[str, Any]) -> None:
+        await self.emit(
+            _server_event(
+                "conversation.item.added",
+                previous_item_id=None,
+                item=item,
+            )
+        )
+        await self.emit(_server_event("conversation.item.done", item=item))
+
+    async def _append_audio(self, event: dict[str, Any]) -> None:
+        audio = _decode_base64(event.get("audio"), param="audio")
+        if len(audio) % 2:
+            raise RealtimeEventError(
+                "WebSocket audio must contain complete PCM16 samples",
+                code="invalid_audio_format",
+                param="audio",
+            )
+        if self.channel != "websocket":
+            raise RealtimeEventError(
+                "input_audio_buffer.append is only used by WebSocket audio",
+                code="invalid_event",
+            )
+        if self.session.external_turn_control:
+            self.buffer_audio(audio, sample_rate=24000, num_channels=1)
+            return
+        await self.runtime.queue_frame(
+            InputAudioRawFrame(audio=audio, sample_rate=24000, num_channels=1)
+        )
+
+    async def _commit_audio(self) -> None:
+        if not self.session.audio_chunks and self.session.external_turn_control:
+            raise RealtimeEventError(
+                "input_audio_buffer is empty",
+                code="input_audio_buffer_commit_empty",
+            )
+        self.session.audio_committed = True
+        item_id = f"item_{uuid4().hex}"
+        await self.emit(
+            _server_event("input_audio_buffer.committed", item_id=item_id)
+        )
+
+    async def _create_response(self, event: dict[str, Any]) -> None:
+        response = event.get("response")
+        if isinstance(response, dict):
+            forbidden = {"instructions", "voice", "tools", "tool_choice", "model"}
+            changed = forbidden.intersection(response)
+            if changed:
+                raise RealtimeEventError(
+                    "Per-response assistant configuration is locked",
+                    code="immutable_session_field",
+                    param=f"response.{sorted(changed)[0]}",
+                )
+        if (
+            self.session.external_turn_control
+            and self.session.audio_chunks
+            and not self.session.audio_committed
+        ):
+            raise RealtimeEventError(
+                "Commit input_audio_buffer before response.create",
+                code="input_audio_buffer_not_committed",
+            )
+        await self._inference.allow_one_response()
+        submitted_item = bool(self.session.pending_item)
+        if self.session.pending_item:
+            pending = self.session.pending_item
+            self.session.pending_item = None
+            await self.queue_internal_message(pending)
+        if self.session.external_turn_control and self.session.audio_chunks:
+            await self.runtime.queue_frame(UserStartedSpeakingFrame())
+            for audio, sample_rate, num_channels in self.session.audio_chunks:
+                await self.runtime.queue_frame(
+                    InputAudioRawFrame(
+                        audio=audio,
+                        sample_rate=sample_rate,
+                        num_channels=num_channels,
+                        transport_source="openai-committed",
+                    )
+                )
+            await self.runtime.queue_frame(UserStoppedSpeakingFrame())
+            await self.runtime.commit_audio()
+            self.clear_buffered_audio()
+        if not submitted_item or self.session.config.runtimeMode == "realtime":
+            await self.runtime.request_response()
+
+    async def _finish_response(self, *, status: str) -> None:
+        response_id, _item_id, text = self.session.finish_response()
+        if not response_id:
+            return
+        await self.emit(
+            _server_event(
+                "response.done",
+                response={
+                    "id": response_id,
+                    "object": "realtime.response",
+                    "status": status,
+                    "output": [],
+                    "output_text": text,
+                },
+            )
+        )
+
+    async def _update_capabilities(self, event: dict[str, Any]) -> None:
+        requested = event.get("capabilities") or []
+        if not isinstance(requested, list) or not all(isinstance(x, str) for x in requested):
+            raise RealtimeEventError("capabilities must be a string array", param="capabilities")
+        enabled = set(requested).intersection(SUPPORTED_CAPABILITIES)
+        if "video_track" in enabled and not self.session.vision_enabled:
+            enabled.remove("video_track")
+        self.session.capabilities = enabled
+        await self.emit(
+            _server_event(
+                "x.interactive_media.capabilities.updated",
+                capabilities=sorted(enabled),
+                rejected=sorted(set(requested) - enabled),
+            )
+        )
+
+    async def _update_variables(self, event: dict[str, Any]) -> None:
+        if "dynamic_variables" not in self.session.capabilities:
+            raise RealtimeEventError(
+                "dynamic_variables capability was not negotiated",
+                code="extension_not_negotiated",
+            )
+        variables = event.get("variables")
+        if not isinstance(variables, dict) or not variables:
+            raise RealtimeEventError("variables must be a non-empty object", param="variables")
+        await self.queue_internal_message(
+            {
+                "type": "session-update",
+                "schema_version": 1,
+                "update_id": str(event.get("event_id") or f"update_{uuid4().hex}"),
+                "dynamic_variables": variables,
+            }
+        )
+
+    def translate_server_message(self, message: dict[str, Any]) -> list[dict[str, Any]]:
+        message_type = str(message.get("type") or "")
+        if message_type == "error":
+            return [message]
+        if "." in message_type:
+            return [message]
+        if message_type == "transcript":
+            return self._translate_transcript(message)
+        if message_type == "assistant-text-start":
+            return self._start_assistant_output()
+        if message_type == "assistant-text-delta":
+            return self._assistant_delta(str(message.get("delta") or ""))
+        if message_type == "assistant-text-end":
+            return self._end_assistant_output(bool(message.get("interrupted")))
+        if message_type == "client-tool-call":
+            return self._client_tool_call(message)
+        if message_type == "user-input-result" and message.get("status") == "error":
+            return [
+                error_event(
+                    RealtimeEventError(
+                        str(message.get("message") or "User input failed"),
+                        code="input_error",
+                    )
+                )
+            ]
+        extension = self._extension_event(message)
+        return [extension] if extension else []
+
+    def _translate_transcript(self, message: dict[str, Any]) -> list[dict[str, Any]]:
+        role = message.get("role")
+        text = str(message.get("content") or "")
+        item_id = f"item_{uuid4().hex}"
+        if role == "user":
+            return [
+                _server_event(
+                    "conversation.item.input_audio_transcription.completed",
+                    item_id=item_id,
+                    content_index=0,
+                    transcript=text,
+                )
+            ]
+        response_id, output_item_id = self.session.begin_response()
+        item = {
+            "id": output_item_id,
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": text}],
+        }
+        self.session.response_text += text
+        response = {
+            "id": response_id,
+            "object": "realtime.response",
+            "status": "completed",
+            "output": [item],
+        }
+        self.session.finish_response()
+        return [
+            _server_event("response.created", response={**response, "status": "in_progress", "output": []}),
+            _server_event("response.output_item.added", response_id=response_id, output_index=0, item=item),
+            _server_event("response.output_item.done", response_id=response_id, output_index=0, item=item),
+            _server_event("response.done", response=response),
+        ]
+
+    def _start_assistant_output(self) -> list[dict[str, Any]]:
+        if self.session.active_response_id:
+            return []
+        response_id, item_id = self.session.begin_response()
+        content_type = "audio" if self.session.output_is_audio else "text"
+        item = {
+            "id": item_id,
+            "type": "message",
+            "role": "assistant",
+            "status": "in_progress",
+            "content": [{"type": content_type}],
+        }
+        return [
+            _server_event(
+                "response.created",
+                response={
+                    "id": response_id,
+                    "object": "realtime.response",
+                    "status": "in_progress",
+                    "output": [],
+                },
+            ),
+            _server_event(
+                "response.output_item.added",
+                response_id=response_id,
+                output_index=0,
+                item=item,
+            ),
+            _server_event(
+                "response.content_part.added",
+                response_id=response_id,
+                item_id=item_id,
+                output_index=0,
+                content_index=0,
+                part={"type": content_type},
+            ),
+        ]
+
+    def _assistant_delta(self, delta: str) -> list[dict[str, Any]]:
+        events = self._start_assistant_output()
+        response_id, item_id = self.session.begin_response()
+        self.session.response_text += delta
+        event_type = (
+            "response.output_audio_transcript.delta"
+            if self.session.output_is_audio
+            else "response.output_text.delta"
+        )
+        events.append(
+            _server_event(
+                event_type,
+                response_id=response_id,
+                item_id=item_id,
+                output_index=0,
+                content_index=0,
+                delta=delta,
+            )
+        )
+        return events
+
+    def _end_assistant_output(self, interrupted: bool) -> list[dict[str, Any]]:
+        response_id, item_id, text = self.session.finish_response()
+        if not response_id or not item_id:
+            return []
+        content_type = "audio" if self.session.output_is_audio else "text"
+        done_type = (
+            "response.output_audio_transcript.done"
+            if self.session.output_is_audio
+            else "response.output_text.done"
+        )
+        item = {
+            "id": item_id,
+            "type": "message",
+            "role": "assistant",
+            "status": "incomplete" if interrupted else "completed",
+            "content": [{"type": content_type, "transcript" if self.session.output_is_audio else "text": text}],
+        }
+        status = "cancelled" if interrupted else "completed"
+        events = [
+            _server_event(done_type, response_id=response_id, item_id=item_id, output_index=0, content_index=0, transcript=text, text=text),
+            _server_event("response.content_part.done", response_id=response_id, item_id=item_id, output_index=0, content_index=0, part=item["content"][0]),
+            _server_event("response.output_item.done", response_id=response_id, output_index=0, item=item),
+            _server_event(
+                "response.done",
+                response={
+                    "id": response_id,
+                    "object": "realtime.response",
+                    "status": status,
+                    "output": [item],
+                },
+            ),
+        ]
+        if self.session.output_is_audio:
+            events.insert(
+                1,
+                _server_event(
+                    "response.output_audio.done",
+                    response_id=response_id,
+                    item_id=item_id,
+                    output_index=0,
+                    content_index=0,
+                ),
+            )
+        return events
+
+    def _client_tool_call(self, message: dict[str, Any]) -> list[dict[str, Any]]:
+        events: list[dict[str, Any]] = []
+        if not self.session.active_response_id:
+            response_id, _unused_item_id = self.session.begin_response()
+            events.append(
+                _server_event(
+                    "response.created",
+                    response={
+                        "id": response_id,
+                        "object": "realtime.response",
+                        "status": "in_progress",
+                        "output": [],
+                    },
+                )
+            )
+        else:
+            response_id = self.session.active_response_id
+        item_id = f"item_{uuid4().hex}"
+        call_id = str(message.get("tool_call_id") or "")
+        arguments = json.dumps(message.get("arguments") or {}, ensure_ascii=False)
+        item = {
+            "id": item_id,
+            "type": "function_call",
+            "status": "completed",
+            "name": str(message.get("function_name") or ""),
+            "call_id": call_id,
+            "arguments": arguments,
+        }
+        events.extend([
+            _server_event("response.output_item.added", response_id=response_id, output_index=0, item=item),
+            _server_event("response.function_call_arguments.delta", response_id=response_id, item_id=item_id, output_index=0, call_id=call_id, delta=arguments),
+            _server_event("response.function_call_arguments.done", response_id=response_id, item_id=item_id, output_index=0, call_id=call_id, name=item["name"], arguments=arguments),
+            _server_event("response.output_item.done", response_id=response_id, output_index=0, item=item),
+        ])
+        self.session.active_output_item_id = None
+        return events
+
+    def _extension_event(self, message: dict[str, Any]) -> dict[str, Any] | None:
+        kind = str(message.get("type") or "")
+        if kind == "session-update-result" and "dynamic_variables" in self.session.capabilities:
+            return _server_event(
+                "x.interactive_media.session.variables.updated",
+                update_id=message.get("update_id"),
+                status=message.get("status"),
+                message=message.get("message"),
+            )
+        if kind in {"node-active", "workflow-event", "workflow-variables", "workflow-error"}:
+            if "workflow_events" not in self.session.capabilities:
+                return None
+            names = {
+                "node-active": "x.interactive_media.workflow.node_active",
+                "workflow-event": "x.interactive_media.workflow.event",
+                "workflow-variables": "x.interactive_media.workflow.variables.updated",
+                "workflow-error": "x.interactive_media.workflow.error",
+            }
+            return _server_event(names[kind], **{k: v for k, v in message.items() if k != "type"})
+        if kind in {"handoff-requested", "call-ended"}:
+            if "handoff" not in self.session.capabilities:
+                return None
+            event_type = (
+                "x.interactive_media.call.handoff_requested"
+                if kind == "handoff-requested"
+                else "x.interactive_media.call.ended"
+            )
+            return _server_event(event_type, **{k: v for k, v in message.items() if k != "type"})
+        return None

backend/services/openai_realtime/events.py

@@ -0,0 +1,115 @@
+"""Small, explicit validation helpers for the supported Realtime event subset."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+from uuid import uuid4
+
+
+SUPPORTED_CLIENT_EVENTS = {
+    "session.update",
+    "conversation.item.create",
+    "conversation.item.truncate",
+    "input_audio_buffer.append",
+    "input_audio_buffer.commit",
+    "input_audio_buffer.clear",
+    "output_audio_buffer.clear",
+    "response.create",
+    "response.cancel",
+    "x.interactive_media.capabilities.update",
+    "x.interactive_media.session.variables.update",
+}
+
+SUPPORTED_CAPABILITIES = {
+    "dynamic_variables",
+    "workflow_events",
+    "handoff",
+    "video_track",
+}
+
+
+@dataclass
+class RealtimeEventError(ValueError):
+    message: str
+    code: str = "invalid_request_error"
+    param: str | None = None
+    event_id: str | None = None
+
+    def __str__(self) -> str:
+        return self.message
+
+
+def require_event(message: object) -> dict[str, Any]:
+    if not isinstance(message, dict):
+        raise RealtimeEventError("Event must be a JSON object")
+    event_type = str(message.get("type") or "")
+    if event_type not in SUPPORTED_CLIENT_EVENTS:
+        raise RealtimeEventError(
+            f"Unsupported client event: {event_type or '<missing>'}",
+            code="invalid_event",
+            param="type",
+            event_id=str(message.get("event_id") or "") or None,
+        )
+    return message
+
+
+def assistant_id_from_model(model: object) -> str:
+    value = str(model or "").strip()
+    if not value.startswith("assistant:") or not value.removeprefix("assistant:"):
+        raise RealtimeEventError(
+            "model must use assistant:asst_xxx",
+            code="invalid_model",
+            param="session.model",
+        )
+    return value.removeprefix("assistant:")
+
+
+def normalize_turn_detection(value: object) -> dict[str, Any] | None:
+    if value is None:
+        return None
+    if not isinstance(value, dict) or value.get("type") != "server_vad":
+        raise RealtimeEventError(
+            "turn_detection must be server_vad or null",
+            param="session.audio.input.turn_detection",
+        )
+    try:
+        threshold = float(value.get("threshold", 0.7))
+        prefix_ms = int(value.get("prefix_padding_ms", 200))
+        silence_ms = int(value.get("silence_duration_ms", 600))
+    except (TypeError, ValueError) as exc:
+        raise RealtimeEventError(
+            "Invalid server_vad threshold, padding, or silence duration",
+            param="session.audio.input.turn_detection",
+        ) from exc
+    if (
+        not 0 <= threshold <= 1
+        or not 0 <= prefix_ms <= 5000
+        or not 100 <= silence_ms <= 10000
+    ):
+        raise RealtimeEventError(
+            "Invalid server_vad threshold, padding, or silence duration",
+            param="session.audio.input.turn_detection",
+        )
+    return {
+        "type": "server_vad",
+        "threshold": threshold,
+        "prefix_padding_ms": prefix_ms,
+        "silence_duration_ms": silence_ms,
+        "create_response": bool(value.get("create_response", True)),
+        "interrupt_response": bool(value.get("interrupt_response", True)),
+    }
+
+
+def error_event(error: RealtimeEventError) -> dict[str, Any]:
+    return {
+        "type": "error",
+        "event_id": f"event_{uuid4().hex}",
+        "error": {
+            "type": "invalid_request_error",
+            "code": error.code,
+            "message": error.message,
+            "param": error.param,
+            "event_id": error.event_id,
+        },
+    }

backend/services/openai_realtime/session.py

@@ -0,0 +1,141 @@
+"""Connection-local state for the OpenAI-compatible Realtime wire protocol."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+from uuid import uuid4
+
+from models import AssistantConfig
+from services.openai_realtime.events import normalize_turn_detection
+
+
+def _id(prefix: str) -> str:
+    return f"{prefix}_{uuid4().hex}"
+
+
+@dataclass
+class OpenAIRealtimeSession:
+    assistant_id: str
+    config: AssistantConfig
+    vision_enabled: bool
+    safety_identifier_hash: str | None = None
+    id: str = field(default_factory=lambda: _id("sess"))
+    output_modalities: list[str] = field(default_factory=lambda: ["audio"])
+    turn_detection: dict[str, Any] | None = field(
+        default_factory=lambda: {
+            "type": "server_vad",
+            "threshold": 0.7,
+            "prefix_padding_ms": 200,
+            "silence_duration_ms": 600,
+            "create_response": True,
+            "interrupt_response": True,
+        }
+    )
+    capabilities: set[str] = field(default_factory=set)
+    pending_item: dict[str, Any] | None = None
+    audio_chunks: list[tuple[bytes, int, int]] = field(default_factory=list)
+    buffered_audio_bytes: int = 0
+    audio_committed: bool = False
+    active_response_id: str | None = None
+    active_output_item_id: str | None = None
+    active_input_item_id: str | None = None
+    response_text: str = ""
+
+    @property
+    def model(self) -> str:
+        return f"assistant:{self.assistant_id}"
+
+    @property
+    def external_turn_control(self) -> bool:
+        return self.turn_detection is None
+
+    @property
+    def output_is_audio(self) -> bool:
+        return self.output_modalities == ["audio"]
+
+    def client_tools(self) -> list[dict[str, Any]]:
+        result = []
+        for tool in self.config.tools:
+            if tool.type != "client":
+                continue
+            definition = tool.definition or {}
+            parameters = (definition.get("config") or {}).get("parameters") or []
+            properties: dict[str, Any] = {}
+            required: list[str] = []
+            for parameter in parameters:
+                name = str(parameter.get("name") or "")
+                if not name:
+                    continue
+                properties[name] = {
+                    "type": parameter.get("type") or "string",
+                    **(
+                        {"description": parameter["description"]}
+                        if parameter.get("description")
+                        else {}
+                    ),
+                }
+                if parameter.get("required"):
+                    required.append(name)
+            result.append(
+                {
+                    "type": "function",
+                    "name": tool.function_name,
+                    "description": tool.description,
+                    "parameters": {
+                        "type": "object",
+                        "properties": properties,
+                        "required": required,
+                    },
+                }
+            )
+        return result
+
+    def public_value(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "type": "realtime",
+            "model": self.model,
+            "output_modalities": self.output_modalities,
+            "audio": {
+                "input": {
+                    "format": {"type": "audio/pcm", "rate": 24000},
+                    "turn_detection": self.turn_detection,
+                },
+                "output": {
+                    "format": {"type": "audio/pcm", "rate": 24000},
+                },
+            },
+            "tools": self.client_tools(),
+            "tool_choice": "auto",
+        }
+
+    def apply_initial_options(self, value: dict[str, Any]) -> None:
+        modalities = value.get("output_modalities")
+        if modalities in (["audio"], ["text"]):
+            self.output_modalities = list(modalities)
+        audio = value.get("audio")
+        if isinstance(audio, dict) and isinstance(audio.get("input"), dict):
+            turn_detection = audio["input"].get("turn_detection", self.turn_detection)
+        else:
+            turn_detection = value.get("turn_detection", self.turn_detection)
+        self.turn_detection = normalize_turn_detection(turn_detection)
+
+    def begin_response(self) -> tuple[str, str]:
+        if not self.active_response_id:
+            self.active_response_id = _id("resp")
+        if not self.active_output_item_id:
+            self.active_output_item_id = _id("item")
+            self.response_text = ""
+        return self.active_response_id, str(self.active_output_item_id)
+
+    def finish_response(self) -> tuple[str | None, str | None, str]:
+        result = (
+            self.active_response_id,
+            self.active_output_item_id,
+            self.response_text,
+        )
+        self.active_response_id = None
+        self.active_output_item_id = None
+        self.response_text = ""
+        return result

backend/services/openai_realtime/webrtc.py

@@ -0,0 +1,26 @@
+"""Project-local SmallWebRTC specialization for the OpenAI event channel."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
+
+
+class OpenAIRealtimeWebRTCConnection(SmallWebRTCConnection):
+    """Prevent Pipecat's private signalling envelope from leaking to clients."""
+
+    data_channel_label = "oai-events"
+
+    def _setup_listeners(self):
+        super()._setup_listeners()
+
+        @self._pc.on("datachannel")
+        def require_openai_event_channel(channel):
+            if channel.label != self.data_channel_label:
+                channel.close()
+
+    def send_app_message(self, message: Any):
+        if isinstance(message, dict) and message.get("type") == "signalling":
+            return
+        super().send_app_message(message)

backend/services/openai_realtime/websocket.py

@@ -0,0 +1,51 @@
+"""UTF-8 JSON serializer and transport builder for public Realtime WebSockets."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from fastapi import WebSocket
+from pipecat.frames.frames import (
+    Frame,
+    InputTransportMessageFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
+)
+from pipecat.serializers.base_serializer import FrameSerializer
+
+from services.pipecat.transports import build_serialized_ws_transport
+
+
+class OpenAIRealtimeJSONSerializer(FrameSerializer):
+    async def serialize(self, frame: Frame) -> str | bytes | None:
+        if self.should_ignore_frame(frame):
+            return None
+        if isinstance(
+            frame,
+            (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame),
+        ) and isinstance(frame.message, dict):
+            return json.dumps(frame.message, ensure_ascii=False, separators=(",", ":"))
+        return None
+
+    async def deserialize(self, data: str | bytes) -> Frame | None:
+        if isinstance(data, bytes):
+            message: Any = {
+                "type": "_invalid_binary_frame",
+                "event_id": None,
+            }
+        else:
+            try:
+                message = json.loads(data)
+            except json.JSONDecodeError:
+                message = {"type": "_invalid_json", "event_id": None}
+        return InputTransportMessageFrame(message=message)
+
+
+def build_openai_websocket_transport(websocket: WebSocket):
+    return build_serialized_ws_transport(
+        websocket,
+        serializer=OpenAIRealtimeJSONSerializer(),
+        sample_rate=24000,
+    )
+

backend/services/pipecat/pipeline.py

@@ -47,6 +47,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.flows import FlowsFunctionSchema
 from pipecat.frames.frames import (
     EndFrame,
+    InterruptionFrame,
     LLMMessagesAppendFrame,
     OutputTransportMessageUrgentFrame,
     UserImageRawFrame,
@@ -94,6 +95,11 @@ from services.pipecat.pipeline_events import (
     bind_cascade_pipeline_events,
     bind_realtime_pipeline_events,
 )
+from services.realtime.protocol import (
+    PipelineProtocolAdapter,
+    PipelineProtocolRuntime,
+    RealtimeProviderControlFrame,
+)
 from pipecat.workers.runner import WorkerRunner
 
 
@@ -201,6 +207,7 @@ async def run_pipeline(
     vision_enabled: bool = False,
     assistant_id: str | None = None,
     channel: str = "webrtc",
+    protocol_adapter: PipelineProtocolAdapter | None = None,
 ) -> None:
     """在给定 transport 上构建并运行管线,直到连接结束。
 
@@ -234,6 +241,7 @@ async def run_pipeline(
             vision_enabled=vision_enabled,
             assistant_id=assistant_id,
             channel=channel,
+            protocol_adapter=protocol_adapter,
         )
         return
 
@@ -626,15 +634,22 @@ async def run_pipeline(
         analysis_plan=cfg.analysis_config,
         extra=(workflow_engine.session_metadata() if workflow_engine else None),
     )
+    protocol_inputs = protocol_adapter.input_processors() if protocol_adapter else []
+    protocol_inference = (
+        protocol_adapter.inference_processors() if protocol_adapter else []
+    )
+    protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
     pipeline = Pipeline(
         [
             transport.input(),
+            *protocol_inputs,
             client_tools,
             session_update,
             vision_capture,
             user_input,
             stt_processor,
             user_aggregator,
+            *protocol_inference,
             user_turn_router,
             knowledge_retrieval,
             llm,
@@ -645,6 +660,7 @@ async def run_pipeline(
             tts_processor,
             EndCallAfterSpeechProcessor(call_end),
             ConversationHistoryProcessor(recorder),
+            *protocol_outputs,
             transport.output(),
         ]
     )
@@ -657,6 +673,47 @@ async def run_pipeline(
         enable_rtvi=False,
     )
     worker_holder["worker"] = worker
+    protocol_turn_state: dict[str, bool | None] = {
+        "external": False,
+        "interrupt_response": None,
+    }
+    if protocol_adapter:
+
+        async def set_external_turn_control(enabled: bool) -> None:
+            protocol_turn_state["external"] = enabled
+            await user_aggregator.apply_external_turn_control(enabled)
+
+        async def request_response() -> None:
+            # Cascade inference is triggered when the adapter submits user-input.
+            return None
+
+        async def commit_audio() -> None:
+            return None
+
+        async def clear_audio() -> None:
+            return None
+
+        async def set_response_interruption(enabled: bool) -> None:
+            protocol_turn_state["interrupt_response"] = enabled
+            await user_aggregator.apply_turn_strategies(
+                cfg.turnConfig,
+                enable_interruptions=enabled,
+            )
+
+        async def cancel_response() -> None:
+            await worker.queue_frame(InterruptionFrame())
+
+        await protocol_adapter.bind(
+            PipelineProtocolRuntime(
+                queue_frame=worker.queue_frame,
+                set_external_turn_control=set_external_turn_control,
+                set_response_interruption=set_response_interruption,
+                commit_audio=commit_audio,
+                clear_audio=clear_audio,
+                request_response=request_response,
+                cancel_response=cancel_response,
+            )
+        )
     service_controller = WorkflowServiceController(
         worker=worker,
         llm_services=llm_services,
@@ -685,6 +742,15 @@ async def run_pipeline(
             normalized,
             enable_interruptions=enable_interrupt,
         )
+        if protocol_turn_state["external"]:
+            await user_aggregator.apply_external_turn_control(True)
+        elif protocol_turn_state["interrupt_response"] is not None:
+            await user_aggregator.apply_turn_strategies(
+                normalized,
+                enable_interruptions=bool(
+                    protocol_turn_state["interrupt_response"]
+                ),
+            )
         await worker.queue_frame(
             VADParamsUpdateFrame(params=create_vad_params(normalized))
         )
@@ -872,6 +938,7 @@ async def run_realtime_pipeline(
     vision_enabled: bool = False,
     assistant_id: str | None = None,
     channel: str = "webrtc",
+    protocol_adapter: PipelineProtocolAdapter | None = None,
 ) -> None:
     """Run a speech-to-speech model that owns ASR, reasoning, and synthesis."""
     realtime = create_realtime_service(
@@ -960,18 +1027,26 @@ async def run_realtime_pipeline(
             else None
         ),
     )
+    protocol_inputs = protocol_adapter.input_processors() if protocol_adapter else []
+    protocol_inference = (
+        protocol_adapter.inference_processors() if protocol_adapter else []
+    )
+    protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
     pipeline = Pipeline(
         [
             transport.input(),
+            *protocol_inputs,
             vision_capture,
             client_tools,
             session_update,
             user_input,
             input_gate,
+            *protocol_inference,
             realtime,
             dynamic_variables,
             EndCallAfterSpeechProcessor(call_end),
             ConversationHistoryProcessor(recorder),
+            *protocol_outputs,
             transport.output(),
         ]
     )
@@ -985,6 +1060,46 @@ async def run_realtime_pipeline(
         enable_rtvi=False,
     )
     worker_holder["worker"] = worker
+    if protocol_adapter:
+
+        async def set_external_turn_control(enabled: bool) -> None:
+            update = getattr(realtime, "update_turn_detection", None)
+            if callable(update):
+                await update(None if enabled else cfg.turnConfig)
+
+        async def request_response() -> None:
+            await worker.queue_frame(
+                RealtimeProviderControlFrame(action="request_response")
+            )
+
+        async def commit_audio() -> None:
+            await worker.queue_frame(
+                RealtimeProviderControlFrame(action="commit_audio")
+            )
+
+        async def clear_audio() -> None:
+            await worker.queue_frame(
+                RealtimeProviderControlFrame(action="clear_audio")
+            )
+
+        async def cancel_response() -> None:
+            await realtime.interrupt()
+
+        async def set_response_interruption(_enabled: bool) -> None:
+            # Provider-side settings vary; explicit response.cancel remains portable.
+            return None
+
+        await protocol_adapter.bind(
+            PipelineProtocolRuntime(
+                queue_frame=worker.queue_frame,
+                set_external_turn_control=set_external_turn_control,
+                set_response_interruption=set_response_interruption,
+                commit_audio=commit_audio,
+                clear_audio=clear_audio,
+                request_response=request_response,
+                cancel_response=cancel_response,
+            )
+        )
 
     def set_input_enabled(enabled: bool) -> None:
         input_state["enabled"] = enabled

backend/services/pipecat/qwen_audio_realtime.py

@@ -41,13 +41,14 @@ from services.pipecat.realtime_tools import (
     RealtimeToolDispatcher,
     RealtimeToolSession,
 )
+from services.realtime.protocol import RealtimeProviderControlFrame
 
 
 DEFAULT_QWEN_AUDIO_REALTIME_MODEL = "qwen-audio-3.0-realtime-flash"
 DEFAULT_QWEN_AUDIO_REALTIME_VOICE = "longanqian"
 QWEN_INPUT_SAMPLE_RATE = 16_000
 QWEN_OUTPUT_SAMPLE_RATE = 24_000
-SUPPORTED_TURN_DETECTION_MODES = frozenset({"server_vad", "smart_turn"})
+SUPPORTED_TURN_DETECTION_MODES = frozenset({"none", "server_vad", "smart_turn"})
 
 ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None]
 SpeechStartedHandler = Callable[[], Awaitable[None]]
@@ -161,6 +162,15 @@ class QwenAudioRealtimeService(AIService):
     async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
         await super().process_frame(frame, direction)
 
+        if isinstance(frame, RealtimeProviderControlFrame):
+            if frame.action == "commit_audio":
+                await self.commit_audio_buffer()
+            elif frame.action == "clear_audio":
+                await self.clear_audio_buffer()
+            else:
+                await self.request_response()
+            return
+
         if isinstance(frame, InputAudioRawFrame):
             if (
                 frame.sample_rate != self._input_sample_rate
@@ -234,6 +244,20 @@ class QwenAudioRealtimeService(AIService):
             )
         await self._send_event({"type": "response.create"})
 
+    async def commit_audio_buffer(self) -> None:
+        await self._send_event({"type": "input_audio_buffer.commit"})
+
+    async def clear_audio_buffer(self) -> None:
+        await self._send_event({"type": "input_audio_buffer.clear"})
+
+    async def update_turn_detection(self, config: dict[str, Any] | None) -> None:
+        """Apply OpenAI PTT before provider startup; reject unsafe live changes."""
+
+        mode = "none" if config is None else "server_vad"
+        if self._session_ready.is_set():
+            raise ValueError("Qwen Realtime 不支持会话建立后切换轮次检测")
+        self._turn_detection_mode = mode
+
     async def wait_for_response_boundary(self) -> None:
         """Wait until Qwen has finished or cancelled the current response."""
         await self._response_done.wait()
@@ -355,7 +379,9 @@ class QwenAudioRealtimeService(AIService):
             "tool_choice": "auto",
         }
 
-    def _turn_detection_config(self) -> dict[str, Any]:
+    def _turn_detection_config(self) -> dict[str, Any] | None:
+        if self._turn_detection_mode == "none":
+            return None
         if self._turn_detection_mode == "smart_turn":
             return {"type": "smart_turn"}
         return {

backend/services/pipecat/stepfun_realtime.py

@@ -35,6 +35,7 @@ from services.pipecat.realtime_tools import (
     RealtimeToolDispatcher,
     RealtimeToolSession,
 )
+from services.realtime.protocol import RealtimeProviderControlFrame
 
 DEFAULT_STEPFUN_REALTIME_URL = "wss://api.stepfun.com/v1/realtime"
 SpeechStartedHandler = Callable[[], Awaitable[None]]
@@ -69,6 +70,7 @@ class StepFunRealtimeService(AIService):
         self._prefix_padding_ms = prefix_padding_ms
         self._silence_duration_ms = silence_duration_ms
         self._energy_awakeness_threshold = energy_awakeness_threshold
+        self._turn_detection_enabled = True
         self._warned_input_sample_rate = False
         self._websocket = None
         self._receive_task: asyncio.Task | None = None
@@ -115,6 +117,15 @@ class StepFunRealtimeService(AIService):
     async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
         await super().process_frame(frame, direction)
 
+        if isinstance(frame, RealtimeProviderControlFrame):
+            if frame.action == "commit_audio":
+                await self.commit_audio_buffer()
+            elif frame.action == "clear_audio":
+                await self.clear_audio_buffer()
+            else:
+                await self.request_response()
+            return
+
         if isinstance(frame, InputAudioRawFrame):
             if (
                 frame.sample_rate != self._input_sample_rate
@@ -183,6 +194,23 @@ class StepFunRealtimeService(AIService):
             )
         await self._send_event({"type": "response.create"})
 
+    async def commit_audio_buffer(self) -> None:
+        await self._send_event({"type": "input_audio_buffer.commit"})
+
+    async def clear_audio_buffer(self) -> None:
+        await self._send_event({"type": "input_audio_buffer.clear"})
+
+    async def update_turn_detection(self, config: dict[str, Any] | None) -> None:
+        self._turn_detection_enabled = config is not None
+        if self._session_ready.is_set():
+            await self._send_event(
+                {
+                    "type": "session.update",
+                    "session": {"turn_detection": self._turn_detection_config()},
+                },
+                wait_until_ready=False,
+            )
+
     async def wait_for_response_boundary(self) -> None:
         """Wait until StepFun has finished or cancelled the current response."""
         await self._response_done.wait()
@@ -386,12 +414,7 @@ class StepFunRealtimeService(AIService):
                     "voice": self._voice,
                     "input_audio_format": "pcm16",
                     "output_audio_format": "pcm16",
-                    "turn_detection": {
-                        "type": "server_vad",
-                        "prefix_padding_ms": self._prefix_padding_ms,
-                        "silence_duration_ms": self._silence_duration_ms,
-                        "energy_awakeness_threshold": self._energy_awakeness_threshold,
-                    },
+                    "turn_detection": self._turn_detection_config(),
                     "tools": [tool.provider_schema() for tool in self._tools],
                     "tool_choice": "auto",
                 },
@@ -399,6 +422,16 @@ class StepFunRealtimeService(AIService):
             wait_until_ready=False,
         )
 
+    def _turn_detection_config(self) -> dict[str, Any] | None:
+        if not self._turn_detection_enabled:
+            return None
+        return {
+            "type": "server_vad",
+            "prefix_padding_ms": self._prefix_padding_ms,
+            "silence_duration_ms": self._silence_duration_ms,
+            "energy_awakeness_threshold": self._energy_awakeness_threshold,
+        }
+
     async def update_instructions(self, instructions: str) -> None:
         """Refresh model instructions without rebuilding the realtime session."""
         self._instructions = instructions

backend/services/pipecat/transports.py

@@ -25,6 +25,7 @@ from pipecat.transports.websocket.fastapi import (
     FastAPIWebsocketTransport,
     FastAPIWebsocketParams,
 )
+from pipecat.serializers.base_serializer import FrameSerializer
 from pipecat.serializers.protobuf import ProtobufFrameSerializer
 
 from services.pipecat.call_lifecycle import FixedSpeechPlaybackMarkerFrame
@@ -103,10 +104,33 @@ def build_ws_transport(websocket: WebSocket) -> FastAPIWebsocketTransport:
     """裸 WS 输出。序列化用 protobuf(自定义客户端用同款解码);
     若对接电话商,把 serializer 换成对应的 TwilioFrameSerializer 等即可。
     """
+    return build_serialized_ws_transport(
+        websocket,
+        serializer=ProtobufFrameSerializer(),
+    )
+
+
+def build_serialized_ws_transport(
+    websocket: WebSocket,
+    *,
+    serializer: FrameSerializer,
+    sample_rate: int | None = None,
+) -> FastAPIWebsocketTransport:
+    """Build a text/binary WS transport without coupling it to one protocol."""
+
+    sample_rates = (
+        {
+            "audio_in_sample_rate": sample_rate,
+            "audio_out_sample_rate": sample_rate,
+        }
+        if sample_rate
+        else {}
+    )
     return _WebsocketTransport(
         websocket=websocket,
         params=FastAPIWebsocketParams(
-            serializer=ProtobufFrameSerializer(),
+            serializer=serializer,
             **_base_params(),
+            **sample_rates,
         ),
     )

backend/services/pipecat/turn_config.py

@@ -20,7 +20,10 @@ from pipecat.turns.user_stop import (
     SpeechTimeoutUserTurnStopStrategy,
     TurnAnalyzerUserTurnStopStrategy,
 )
-from pipecat.turns.user_turn_strategies import UserTurnStrategies
+from pipecat.turns.user_turn_strategies import (
+    ExternalUserTurnStrategies,
+    UserTurnStrategies,
+)
 
 
 DEFAULT_VAD = {
@@ -126,3 +129,13 @@ class ConfigurableLLMUserAggregator(LLMUserAggregator):
             enable_interruptions=enable_interruptions,
         )
         await self._user_turn_controller.update_strategies(strategies)
+
+    async def apply_external_turn_control(self, enabled: bool) -> None:
+        """Switch between client-controlled PTT and the configured VAD."""
+
+        strategies = (
+            ExternalUserTurnStrategies()
+            if enabled
+            else self._params.user_turn_strategies
+        )
+        await self._user_turn_controller.update_strategies(strategies)

backend/services/realtime/__init__.py

@@ -0,0 +1,2 @@
+"""Protocol-neutral helpers for realtime transports."""
+

backend/services/realtime/launcher.py

@@ -0,0 +1,195 @@
+"""Resolve and validate an assistant before creating a media connection."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from db.session import SessionLocal
+from models import AssistantConfig
+from services.config_resolver import resolve_runtime_config
+from services.node_specs import graph_references
+from services.runtime_variables import prepare_dynamic_config
+from services.workflow_engine import WorkflowEngine
+
+
+async def resolve_assistant_config(
+    assistant_id: str,
+    *,
+    dynamic_variables: dict[str, Any] | None = None,
+) -> AssistantConfig:
+    async with SessionLocal() as session:
+        config = await resolve_runtime_config(session, assistant_id)
+    return prepare_dynamic_config(
+        config,
+        dynamic_variables or {},
+        assistant_id=assistant_id,
+    )
+
+
+def validate_visual_runtime(config: AssistantConfig) -> bool:
+    """Return the authoritative video-input permission or fail before connect."""
+
+    if config.type == "workflow":
+        return WorkflowEngine(config.graph).uses_vision()
+
+    vision_enabled = config.vision_enabled
+    if not vision_enabled:
+        return False
+    has_native_vision = (
+        not config.vision_model_resource_id and config.llm_support_image_input
+    )
+    has_aux_vision_model = (
+        bool(config.vision_model_resource_id)
+        and config.vision_llm_support_image_input
+    )
+    if not (has_native_vision or has_aux_vision_model):
+        raise ValueError(
+            "当前模型不支持图片输入,请在模型资源中选择支持图片输入的视觉模型"
+        )
+    return True
+
+
+def _require_values(labels: list[tuple[str, Any]]) -> None:
+    missing = [label for label, value in labels if not value]
+    if missing:
+        raise ValueError(f"助手运行配置不完整: {', '.join(missing)}")
+
+
+def _validate_voice_resource(
+    capability: str,
+    *,
+    interface_type: str,
+    values: dict[str, Any],
+    secrets: dict[str, Any],
+) -> None:
+    labels: list[tuple[str, Any]] = []
+    if capability == "ASR":
+        if interface_type not in {"openai-asr", "dashscope-asr", "xfyun-asr"}:
+            raise ValueError(f"不支持的 ASR 接口类型: {interface_type}")
+        if interface_type == "xfyun-asr":
+            labels.extend(
+                (f"ASR {key}", secrets.get(key))
+                for key in ("appId", "apiKey", "apiSecret")
+            )
+        else:
+            labels.extend(
+                [
+                    ("ASR modelId", values.get("modelId")),
+                    ("ASR apiUrl", values.get("apiUrl")),
+                    ("ASR apiKey", secrets.get("apiKey")),
+                ]
+            )
+    elif capability == "TTS":
+        if interface_type not in {
+            "openai-tts",
+            "dashscope-tts",
+            "xfyun-tts",
+            "xfyun-super-tts",
+        }:
+            raise ValueError(f"不支持的 TTS 接口类型: {interface_type}")
+        labels.append(("TTS voice", values.get("voice")))
+        if interface_type in {"xfyun-tts", "xfyun-super-tts"}:
+            labels.extend(
+                (f"TTS {key}", secrets.get(key))
+                for key in ("appId", "apiKey", "apiSecret")
+            )
+        else:
+            labels.extend(
+                [
+                    ("TTS modelId", values.get("modelId")),
+                    ("TTS apiUrl", values.get("apiUrl")),
+                    ("TTS apiKey", secrets.get("apiKey")),
+                ]
+            )
+    elif capability == "LLM":
+        if interface_type not in {"openai-llm", "dashscope-llm"}:
+            raise ValueError(f"不支持的 LLM 接口类型: {interface_type}")
+        labels.extend(
+            [
+                ("LLM modelId", values.get("modelId")),
+                ("LLM apiUrl", values.get("apiUrl")),
+                ("LLM apiKey", secrets.get("apiKey")),
+            ]
+        )
+    _require_values(labels)
+
+
+def validate_runtime_requirements(config: AssistantConfig) -> None:
+    """Reject structurally incomplete assistants before media negotiation."""
+
+    if config.type not in {"prompt", "workflow", "dify", "fastgpt"}:
+        raise ValueError(f"当前助手类型不支持 Realtime API: {config.type}")
+    if config.runtimeMode == "realtime":
+        if config.type not in {"prompt", "workflow"}:
+            raise ValueError(f"助手类型 {config.type} 不支持 realtime 运行模式")
+        if config.realtime_interface_type not in {
+            "qwen-audio-realtime",
+            "stepfun-realtime",
+        }:
+            raise ValueError(
+                f"不支持的 Realtime 接口类型: {config.realtime_interface_type}"
+            )
+        _require_values(
+            [
+                ("Realtime interfaceType", config.realtime_interface_type),
+                ("Realtime modelId", config.realtimeModel),
+                ("Realtime apiUrl", config.realtime_base_url),
+                ("Realtime apiKey", config.realtime_api_key),
+            ],
+        )
+        return
+
+    _validate_voice_resource(
+        "ASR",
+        interface_type=config.stt_interface_type,
+        values=config.stt_values,
+        secrets=config.stt_secrets,
+    )
+    _validate_voice_resource(
+        "TTS",
+        interface_type=config.tts_interface_type,
+        values=config.tts_values,
+        secrets=config.tts_secrets,
+    )
+    if config.type == "prompt":
+        _validate_voice_resource(
+            "LLM",
+            interface_type=config.llm_interface_type,
+            values=config.llm_values,
+            secrets=config.llm_secrets,
+        )
+    elif config.type == "dify":
+        _require_values(
+            [
+                ("Dify apiUrl", config.dify_api_url),
+                ("Dify apiKey", config.dify_api_key),
+            ],
+        )
+    elif config.type == "fastgpt":
+        _require_values(
+            [
+                ("FastGPT apiUrl", config.fastgpt_api_url),
+                ("FastGPT apiKey", config.fastgpt_api_key),
+            ],
+        )
+
+    if config.type != "workflow":
+        return
+    references = graph_references(config.graph)
+    missing_models = references["model_resources"] - set(
+        config.workflow_model_resources
+    )
+    missing_knowledge = references["knowledge_bases"] - set(
+        config.workflow_knowledge_bases
+    )
+    if missing_models or missing_knowledge:
+        missing = sorted([*missing_models, *missing_knowledge])
+        raise ValueError(f"Workflow 引用了不可用资源: {', '.join(missing)}")
+    for resource in config.workflow_model_resources.values():
+        if resource.capability in {"ASR", "TTS", "LLM"}:
+            _validate_voice_resource(
+                resource.capability,
+                interface_type=resource.interface_type,
+                values=resource.values,
+                secrets=resource.secrets,
+            )

backend/services/realtime/lifecycle.py

@@ -0,0 +1,109 @@
+"""Shared ownership of peer connections and their pipeline tasks."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Coroutine
+from typing import Any
+
+from loguru import logger
+
+
+active_connections: set[object] = set()
+pipeline_tasks: set[asyncio.Task[None]] = set()
+connection_tasks: dict[object, asyncio.Task[None]] = {}
+DEFAULT_CLOSE_GRACE_SECONDS = 10.0
+
+
+def _consume_pipeline_result(task: asyncio.Task[None], connection: object) -> None:
+    pipeline_tasks.discard(task)
+    if connection_tasks.get(connection) is task:
+        connection_tasks.pop(connection, None)
+    try:
+        error = task.exception()
+    except asyncio.CancelledError:
+        logger.info(f"Realtime pipeline 已取消: task={task.get_name()}")
+        return
+    if error is not None:
+        logger.opt(exception=error).error(
+            f"Realtime pipeline 异常结束: task={task.get_name()}"
+        )
+
+
+def start_pipeline_task(
+    connection: object,
+    coroutine: Coroutine[Any, Any, None],
+    *,
+    protocol: str,
+) -> asyncio.Task[None]:
+    connection_id = str(getattr(connection, "pc_id", id(connection)))
+    task = asyncio.create_task(
+        coroutine,
+        name=f"{protocol}-pipeline:{connection_id}",
+    )
+    active_connections.add(connection)
+    pipeline_tasks.add(task)
+    connection_tasks[connection] = task
+    task.add_done_callback(
+        lambda completed, connection=connection: _consume_pipeline_result(
+            completed,
+            connection,
+        )
+    )
+    return task
+
+
+async def wait_for_pipeline_close(
+    task: asyncio.Task[None] | None,
+    *,
+    connection_id: str,
+    timeout: float = DEFAULT_CLOSE_GRACE_SECONDS,
+) -> None:
+    if task is None:
+        return
+    try:
+        await asyncio.wait_for(asyncio.shield(task), timeout=timeout)
+        return
+    except TimeoutError:
+        logger.warning(
+            f"Realtime pipeline 关闭超过 {timeout:g} 秒，执行取消: "
+            f"connection_id={connection_id}"
+        )
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        return
+
+    task.cancel()
+    done, _pending = await asyncio.wait({task}, timeout=timeout)
+    if not done:
+        logger.error(f"Realtime pipeline 取消后仍未退出: connection_id={connection_id}")
+
+
+async def shutdown_active_sessions(
+    *,
+    timeout: float = DEFAULT_CLOSE_GRACE_SECONDS,
+) -> None:
+    connections = list(active_connections)
+    if connections:
+        await asyncio.gather(
+            *(connection.disconnect() for connection in connections),
+            return_exceptions=True,
+        )
+        active_connections.difference_update(connections)
+
+    tasks = list(pipeline_tasks)
+    if not tasks:
+        return
+    done, pending = await asyncio.wait(tasks, timeout=timeout)
+    for task in pending:
+        task.cancel()
+    cancelled: set[asyncio.Task[None]] = set()
+    if pending:
+        cancelled, stuck = await asyncio.wait(pending, timeout=timeout)
+        if stuck:
+            logger.error(f"应用关闭时仍有 {len(stuck)} 个 Realtime pipeline 未退出")
+    logger.info(
+        f"Realtime 会话清理完成: normal={len(done)} cancelled={len(cancelled)}"
+    )
+

backend/services/realtime/protocol.py

@@ -0,0 +1,42 @@
+"""Neutral extension point between a transport protocol and the voice pipeline."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Literal, Protocol
+
+from pipecat.frames.frames import Frame, SystemFrame
+from pipecat.processors.frame_processor import FrameProcessor
+
+
+@dataclass(frozen=True)
+class PipelineProtocolRuntime:
+    """Operations a wire-protocol adapter may request from a running pipeline."""
+
+    queue_frame: Callable[[Frame], Awaitable[None]]
+    set_external_turn_control: Callable[[bool], Awaitable[None]]
+    set_response_interruption: Callable[[bool], Awaitable[None]]
+    commit_audio: Callable[[], Awaitable[None]]
+    clear_audio: Callable[[], Awaitable[None]]
+    request_response: Callable[[], Awaitable[None]]
+    cancel_response: Callable[[], Awaitable[None]]
+
+
+@dataclass
+class RealtimeProviderControlFrame(SystemFrame):
+    """Ordered control sent to a speech-to-speech provider after prior input."""
+
+    action: Literal["commit_audio", "clear_audio", "request_response"]
+
+
+class PipelineProtocolAdapter(Protocol):
+    """A protocol adapter only translates frames; it does not own the assistant."""
+
+    def input_processors(self) -> list[FrameProcessor]: ...
+
+    def inference_processors(self) -> list[FrameProcessor]: ...
+
+    def output_processors(self) -> list[FrameProcessor]: ...
+
+    async def bind(self, runtime: PipelineProtocolRuntime) -> None: ...