ai-video-fullstack/backend/routes/openai_realtime.py

"""Public OpenAI-compatible Realtime HTTP, WebRTC, and WebSocket entries."""

from __future__ import annotations

import json
from dataclasses import dataclass, field
from typing import Any
from uuid import uuid4

from db.session import SessionLocal
from fastapi import APIRouter, Depends, Request, WebSocket
from fastapi.responses import JSONResponse, Response
from loguru import logger
from services.openai_api import (
    PUBLIC_API_BEARER,
    PUBLIC_ERROR_RESPONSES,
    REQUEST_ID_RESPONSE_HEADERS,
    openai_error_response,
)
from starlette.websockets import WebSocketState

from services.openai_realtime.auth import (
    RealtimeAuthError,
    RealtimeCredential,
    authenticate_bearer,
    bearer_from_authorization,
    create_client_secret,
    hash_safety_identifier,
)
from services.openai_realtime.bridge import OpenAIRealtimeBridge
from services.openai_realtime.events import (
    RealtimeEventError,
    assistant_id_from_model,
)
from services.openai_realtime.session import OpenAIRealtimeSession
from services.openai_realtime.schemas import (
    RealtimeClientSecretBody,
    RealtimeClientSecretResponse,
)
from services.openai_realtime.webrtc import OpenAIRealtimeWebRTCConnection
from services.openai_realtime.websocket import build_openai_websocket_transport
from services.pipecat.pipeline import run_pipeline
from services.pipecat.transports import build_webrtc_transport
from services.realtime import lifecycle
from services.realtime.launcher import (
    resolve_assistant_config,
    validate_runtime_requirements,
    validate_visual_runtime,
)
from services.webrtc_ice import aiortc_ice_servers


router = APIRouter(prefix="/v1/realtime", tags=["openai-realtime"])
_webrtc_peers: dict[str, OpenAIRealtimeWebRTCConnection] = {}


def _api_error(
    message: str,
    *,
    code: str,
    status_code: int,
    param: str | None = None,
) -> JSONResponse:
    return openai_error_response(
        message,
        code=code,
        status_code=status_code,
        param=param,
    )


async def _credential(request: Request) -> RealtimeCredential:
    token = bearer_from_authorization(request.headers.get("authorization"))
    async with SessionLocal() as db:
        return await authenticate_bearer(db, token)


def _session_payload(value: object) -> dict[str, Any]:
    if not isinstance(value, dict):
        raise RealtimeEventError("session must be an object", param="session")
    session = value.get("session", value)
    if not isinstance(session, dict):
        raise RealtimeEventError("session must be an object", param="session")
    return session


def _reject_locked_creation_fields(session: dict[str, Any]) -> None:
    locked = {"instructions", "voice", "tools", "tool_choice"}.intersection(session)
    audio = session.get("audio")
    if isinstance(audio, dict) and isinstance(audio.get("output"), dict):
        if "voice" in audio["output"]:
            locked.add("audio.output.voice")
    if locked:
        field = sorted(locked)[0]
        raise RealtimeEventError(
            f"{field} is owned by the selected assistant",
            code="immutable_session_field",
            param=f"session.{field}",
        )
    modalities = session.get("output_modalities")
    if modalities is not None and modalities not in (["audio"], ["text"]):
        raise RealtimeEventError(
            'output_modalities must be ["audio"] or ["text"]',
            param="session.output_modalities",
        )


async def _build_session(
    credential: RealtimeCredential,
    requested: dict[str, Any] | None,
    *,
    safety_identifier_hash: str | None,
) -> OpenAIRealtimeSession:
    session_value = dict(credential.session or requested or {})
    _reject_locked_creation_fields(session_value)
    requested_assistant_id = assistant_id_from_model(session_value.get("model"))
    if credential.assistant_id and requested_assistant_id != credential.assistant_id:
        raise RealtimeEventError(
            "Client secret is bound to another assistant",
            code="invalid_model",
            param="session.model",
        )
    config = await resolve_assistant_config(requested_assistant_id)
    validate_runtime_requirements(config)
    vision_enabled = validate_visual_runtime(config)
    state = OpenAIRealtimeSession(
        assistant_id=requested_assistant_id,
        config=config,
        vision_enabled=vision_enabled,
        safety_identifier_hash=(
            credential.safety_identifier_hash or safety_identifier_hash
        ),
    )
    state.apply_initial_options(session_value)
    return state


@router.post(
    "/client_secrets",
    response_model=RealtimeClientSecretResponse,
    responses={
        200: {"headers": REQUEST_ID_RESPONSE_HEADERS},
        **PUBLIC_ERROR_RESPONSES,
    },
    dependencies=[Depends(PUBLIC_API_BEARER)],
    summary="Create a short-lived Realtime client secret",
)
async def create_realtime_client_secret(
    body: RealtimeClientSecretBody,
    request: Request,
):
    try:
        credential = await _credential(request)
        if credential.ephemeral:
            raise RealtimeAuthError("A long-lived API key is required")
        session = _session_payload(body.model_dump(exclude_unset=True))
        _reject_locked_creation_fields(session)
        assistant_id = assistant_id_from_model(session.get("model"))
        # Resolve now so invalid models/configurations fail before a browser gets a token.
        config = await resolve_assistant_config(assistant_id)
        validate_runtime_requirements(config)
        validate_visual_runtime(config)
        safety_hash = hash_safety_identifier(
            request.headers.get("openai-safety-identifier")
        )
        value, expires_at = create_client_secret(
            api_key_id=credential.api_key_id,
            assistant_id=assistant_id,
            session=session,
            safety_identifier_hash=safety_hash,
        )
        return {
            "value": value,
            "expires_at": expires_at,
            "session": session,
        }
    except RealtimeAuthError as exc:
        return _api_error(str(exc), code="invalid_api_key", status_code=401)
    except RealtimeEventError as exc:
        return _api_error(
            str(exc), code=exc.code, status_code=400, param=exc.param
        )
    except ValueError as exc:
        code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session"
        return _api_error(str(exc), code=code, status_code=400)


@router.post(
    "/calls",
    response_class=Response,
    responses={
        200: {
            "description": "WebRTC SDP answer.",
            "headers": {
                **REQUEST_ID_RESPONSE_HEADERS,
                "Location": {
                    "description": "Relative URL of the created Realtime call.",
                    "schema": {"type": "string"},
                },
            },
            "content": {
                "application/sdp": {
                    "schema": {"type": "string", "format": "sdp"},
                    "example": "v=0\\r\\no=- ...",
                }
            },
        },
        **PUBLIC_ERROR_RESPONSES,
    },
    dependencies=[Depends(PUBLIC_API_BEARER)],
    openapi_extra={
        "requestBody": {
            "required": True,
            "content": {
                "multipart/form-data": {
                    "schema": {
                        "type": "object",
                        "required": ["sdp", "session"],
                        "properties": {
                            "sdp": {"type": "string", "format": "sdp"},
                            "session": {
                                "type": "string",
                                "description": "JSON-encoded Realtime session.",
                            },
                        },
                    }
                },
                "application/sdp": {
                    "schema": {"type": "string", "format": "sdp"}
                },
            },
        }
    },
    summary="Create a Realtime WebRTC call",
)
async def create_realtime_call(request: Request):
    try:
        credential = await _credential(request)
        content_type = request.headers.get("content-type", "").lower()
        if content_type.startswith("multipart/form-data"):
            if credential.ephemeral:
                raise RealtimeAuthError(
                    "Client secrets must send an application/sdp offer"
                )
            form = await request.form()
            sdp = str(form.get("sdp") or "")
            raw_session = str(form.get("session") or "{}")
            requested = _session_payload(json.loads(raw_session))
        elif content_type.startswith("application/sdp"):
            if not credential.ephemeral:
                raise RealtimeAuthError(
                    "application/sdp requires a short-lived client secret"
                )
            sdp = (await request.body()).decode("utf-8")
            requested = None
        else:
            raise RealtimeEventError(
                "Use multipart/form-data or application/sdp",
                code="unsupported_content_type",
            )
        if not sdp.strip():
            raise RealtimeEventError("SDP offer is empty", param="sdp")
        state = await _build_session(
            credential,
            requested,
            safety_identifier_hash=hash_safety_identifier(
                request.headers.get("openai-safety-identifier")
            ),
        )
        answer = await _start_webrtc(sdp, state)
        return Response(
            content=answer,
            media_type="application/sdp",
            headers={"Location": f"/v1/realtime/calls/{state.id}"},
        )
    except RealtimeAuthError as exc:
        return _api_error(str(exc), code="invalid_api_key", status_code=401)
    except (RealtimeEventError, json.JSONDecodeError, UnicodeDecodeError) as exc:
        code = exc.code if isinstance(exc, RealtimeEventError) else "invalid_request_error"
        param = exc.param if isinstance(exc, RealtimeEventError) else None
        return _api_error(str(exc), code=code, status_code=400, param=param)
    except ValueError as exc:
        code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session"
        return _api_error(str(exc), code=code, status_code=400)
    except Exception as exc:  # noqa: BLE001 - no internal details in public response
        logger.exception(f"OpenAI Realtime WebRTC 启动失败: {exc}")
        return _api_error(
            "Realtime connection could not be established",
            code="connection_error",
            status_code=500,
        )


async def _start_webrtc(sdp: str, state: OpenAIRealtimeSession) -> str:
    connection = OpenAIRealtimeWebRTCConnection(
        ice_servers=aiortc_ice_servers()
    )
    await connection.initialize(sdp=sdp, type="offer")
    _webrtc_peers[connection.pc_id] = connection
    bridge = OpenAIRealtimeBridge(state, channel="webrtc")
    transport = build_webrtc_transport(
        connection,
        video_in_enabled=state.vision_enabled,
    )
    task = lifecycle.start_pipeline_task(
        connection,
        run_pipeline(
            transport,
            state.config,
            vision_enabled=state.vision_enabled,
            assistant_id=state.assistant_id,
            channel="openai-webrtc",
            protocol_adapter=bridge,
        ),
        protocol="openai-webrtc",
    )

    @connection.event_handler("closed")
    async def on_closed(conn: OpenAIRealtimeWebRTCConnection):
        _webrtc_peers.pop(conn.pc_id, None)
        lifecycle.active_connections.discard(conn)
        await lifecycle.wait_for_pipeline_close(
            task,
            connection_id=conn.pc_id,
        )

    answer = connection.get_answer()
    if not answer:
        raise RuntimeError("WebRTC answer was not created")
    return str(answer["sdp"])


def _websocket_token(websocket: WebSocket) -> tuple[str, str | None]:
    authorization = websocket.headers.get("authorization")
    if authorization:
        return bearer_from_authorization(authorization), None
    protocols = [
        item.strip()
        for item in websocket.headers.get("sec-websocket-protocol", "").split(",")
        if item.strip()
    ]
    for protocol in protocols:
        prefix = "openai-insecure-api-key."
        if protocol.startswith(prefix):
            return protocol.removeprefix(prefix), "realtime" if "realtime" in protocols else None
    raise RealtimeAuthError("Missing Bearer credential")


@dataclass(eq=False)
class _ManagedWebSocket:
    websocket: WebSocket
    pc_id: str = field(default_factory=lambda: f"ws_{uuid4().hex}")

    async def disconnect(self) -> None:
        if self.websocket.application_state == WebSocketState.CONNECTED:
            await self.websocket.close(code=1001)


@router.websocket("")
async def realtime_websocket(websocket: WebSocket):
    managed: _ManagedWebSocket | None = None
    task = None
    try:
        token, accepted_subprotocol = _websocket_token(websocket)
        async with SessionLocal() as db:
            credential = await authenticate_bearer(db, token)
        requested = (
            credential.session
            if credential.ephemeral
            else {"model": websocket.query_params.get("model")}
        )
        state = await _build_session(
            credential,
            requested,
            safety_identifier_hash=hash_safety_identifier(
                websocket.headers.get("openai-safety-identifier")
            ),
        )
        await websocket.accept(subprotocol=accepted_subprotocol)
        managed = _ManagedWebSocket(websocket)
        transport = build_openai_websocket_transport(websocket)
        bridge = OpenAIRealtimeBridge(state, channel="websocket")
        task = lifecycle.start_pipeline_task(
            managed,
            run_pipeline(
                transport,
                state.config,
                vision_enabled=False,
                assistant_id=state.assistant_id,
                channel="openai-websocket",
                protocol_adapter=bridge,
            ),
            protocol="openai-websocket",
        )
        await task
    except (RealtimeAuthError, RealtimeEventError, ValueError) as exc:
        logger.warning(f"拒绝 OpenAI Realtime WebSocket: {exc}")
        if websocket.application_state == WebSocketState.CONNECTED:
            await websocket.close(code=1008, reason=str(exc)[:120])
        else:
            await websocket.close(code=1008)
    except Exception as exc:  # noqa: BLE001 - pipeline callback logs full exception
        logger.warning(f"OpenAI Realtime WebSocket 已关闭: {type(exc).__name__}")
        if websocket.application_state == WebSocketState.CONNECTED:
            await websocket.close(code=1011)
    finally:
        if managed:
            lifecycle.active_connections.discard(managed)
        if task and not task.done():
            await lifecycle.wait_for_pipeline_close(
                task,
                connection_id=managed.pc_id if managed else "websocket",
            )