"""Public OpenAI-compatible Realtime HTTP, WebRTC, and WebSocket entries.""" from __future__ import annotations import json from dataclasses import dataclass, field from typing import Any from uuid import uuid4 from db.session import SessionLocal from fastapi import APIRouter, Depends, Request, WebSocket from fastapi.responses import JSONResponse, Response from loguru import logger from services.openai_api import ( PUBLIC_API_BEARER, PUBLIC_ERROR_RESPONSES, REQUEST_ID_RESPONSE_HEADERS, openai_error_response, ) from starlette.websockets import WebSocketState from services.openai_realtime.auth import ( RealtimeAuthError, RealtimeCredential, authenticate_bearer, bearer_from_authorization, create_client_secret, hash_safety_identifier, ) from services.openai_realtime.bridge import OpenAIRealtimeBridge from services.openai_realtime.events import ( RealtimeEventError, assistant_id_from_model, ) from services.openai_realtime.session import OpenAIRealtimeSession from services.openai_realtime.schemas import ( RealtimeClientSecretBody, RealtimeClientSecretResponse, ) from services.openai_realtime.webrtc import OpenAIRealtimeWebRTCConnection from services.openai_realtime.websocket import build_openai_websocket_transport from services.pipecat.pipeline import run_pipeline from services.pipecat.transports import build_webrtc_transport from services.realtime import lifecycle from services.realtime.launcher import ( resolve_assistant_config, validate_runtime_requirements, validate_visual_runtime, ) from services.webrtc_ice import aiortc_ice_servers router = APIRouter(prefix="/v1/realtime", tags=["openai-realtime"]) _webrtc_peers: dict[str, OpenAIRealtimeWebRTCConnection] = {} def _api_error( message: str, *, code: str, status_code: int, param: str | None = None, ) -> JSONResponse: return openai_error_response( message, code=code, status_code=status_code, param=param, ) async def _credential(request: Request) -> RealtimeCredential: token = bearer_from_authorization(request.headers.get("authorization")) async with SessionLocal() as db: return await authenticate_bearer(db, token) def _session_payload(value: object) -> dict[str, Any]: if not isinstance(value, dict): raise RealtimeEventError("session must be an object", param="session") session = value.get("session", value) if not isinstance(session, dict): raise RealtimeEventError("session must be an object", param="session") return session def _reject_locked_creation_fields(session: dict[str, Any]) -> None: locked = {"instructions", "voice", "tools", "tool_choice"}.intersection(session) audio = session.get("audio") if isinstance(audio, dict) and isinstance(audio.get("output"), dict): if "voice" in audio["output"]: locked.add("audio.output.voice") if locked: field = sorted(locked)[0] raise RealtimeEventError( f"{field} is owned by the selected assistant", code="immutable_session_field", param=f"session.{field}", ) modalities = session.get("output_modalities") if modalities is not None and modalities not in (["audio"], ["text"]): raise RealtimeEventError( 'output_modalities must be ["audio"] or ["text"]', param="session.output_modalities", ) async def _build_session( credential: RealtimeCredential, requested: dict[str, Any] | None, *, safety_identifier_hash: str | None, ) -> OpenAIRealtimeSession: session_value = dict(credential.session or requested or {}) _reject_locked_creation_fields(session_value) requested_assistant_id = assistant_id_from_model(session_value.get("model")) if credential.assistant_id and requested_assistant_id != credential.assistant_id: raise RealtimeEventError( "Client secret is bound to another assistant", code="invalid_model", param="session.model", ) config = await resolve_assistant_config(requested_assistant_id) validate_runtime_requirements(config) vision_enabled = validate_visual_runtime(config) state = OpenAIRealtimeSession( assistant_id=requested_assistant_id, config=config, vision_enabled=vision_enabled, safety_identifier_hash=( credential.safety_identifier_hash or safety_identifier_hash ), ) state.apply_initial_options(session_value) return state @router.post( "/client_secrets", response_model=RealtimeClientSecretResponse, responses={ 200: {"headers": REQUEST_ID_RESPONSE_HEADERS}, **PUBLIC_ERROR_RESPONSES, }, dependencies=[Depends(PUBLIC_API_BEARER)], summary="Create a short-lived Realtime client secret", ) async def create_realtime_client_secret( body: RealtimeClientSecretBody, request: Request, ): try: credential = await _credential(request) if credential.ephemeral: raise RealtimeAuthError("A long-lived API key is required") session = _session_payload(body.model_dump(exclude_unset=True)) _reject_locked_creation_fields(session) assistant_id = assistant_id_from_model(session.get("model")) # Resolve now so invalid models/configurations fail before a browser gets a token. config = await resolve_assistant_config(assistant_id) validate_runtime_requirements(config) validate_visual_runtime(config) safety_hash = hash_safety_identifier( request.headers.get("openai-safety-identifier") ) value, expires_at = create_client_secret( api_key_id=credential.api_key_id, assistant_id=assistant_id, session=session, safety_identifier_hash=safety_hash, ) return { "value": value, "expires_at": expires_at, "session": session, } except RealtimeAuthError as exc: return _api_error(str(exc), code="invalid_api_key", status_code=401) except RealtimeEventError as exc: return _api_error( str(exc), code=exc.code, status_code=400, param=exc.param ) except ValueError as exc: code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session" return _api_error(str(exc), code=code, status_code=400) @router.post( "/calls", response_class=Response, responses={ 200: { "description": "WebRTC SDP answer.", "headers": { **REQUEST_ID_RESPONSE_HEADERS, "Location": { "description": "Relative URL of the created Realtime call.", "schema": {"type": "string"}, }, }, "content": { "application/sdp": { "schema": {"type": "string", "format": "sdp"}, "example": "v=0\\r\\no=- ...", } }, }, **PUBLIC_ERROR_RESPONSES, }, dependencies=[Depends(PUBLIC_API_BEARER)], openapi_extra={ "requestBody": { "required": True, "content": { "multipart/form-data": { "schema": { "type": "object", "required": ["sdp", "session"], "properties": { "sdp": {"type": "string", "format": "sdp"}, "session": { "type": "string", "description": "JSON-encoded Realtime session.", }, }, } }, "application/sdp": { "schema": {"type": "string", "format": "sdp"} }, }, } }, summary="Create a Realtime WebRTC call", ) async def create_realtime_call(request: Request): try: credential = await _credential(request) content_type = request.headers.get("content-type", "").lower() if content_type.startswith("multipart/form-data"): if credential.ephemeral: raise RealtimeAuthError( "Client secrets must send an application/sdp offer" ) form = await request.form() sdp = str(form.get("sdp") or "") raw_session = str(form.get("session") or "{}") requested = _session_payload(json.loads(raw_session)) elif content_type.startswith("application/sdp"): if not credential.ephemeral: raise RealtimeAuthError( "application/sdp requires a short-lived client secret" ) sdp = (await request.body()).decode("utf-8") requested = None else: raise RealtimeEventError( "Use multipart/form-data or application/sdp", code="unsupported_content_type", ) if not sdp.strip(): raise RealtimeEventError("SDP offer is empty", param="sdp") state = await _build_session( credential, requested, safety_identifier_hash=hash_safety_identifier( request.headers.get("openai-safety-identifier") ), ) answer = await _start_webrtc(sdp, state) return Response( content=answer, media_type="application/sdp", headers={"Location": f"/v1/realtime/calls/{state.id}"}, ) except RealtimeAuthError as exc: return _api_error(str(exc), code="invalid_api_key", status_code=401) except (RealtimeEventError, json.JSONDecodeError, UnicodeDecodeError) as exc: code = exc.code if isinstance(exc, RealtimeEventError) else "invalid_request_error" param = exc.param if isinstance(exc, RealtimeEventError) else None return _api_error(str(exc), code=code, status_code=400, param=param) except ValueError as exc: code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session" return _api_error(str(exc), code=code, status_code=400) except Exception as exc: # noqa: BLE001 - no internal details in public response logger.exception(f"OpenAI Realtime WebRTC 启动失败: {exc}") return _api_error( "Realtime connection could not be established", code="connection_error", status_code=500, ) async def _start_webrtc(sdp: str, state: OpenAIRealtimeSession) -> str: connection = OpenAIRealtimeWebRTCConnection( ice_servers=aiortc_ice_servers() ) await connection.initialize(sdp=sdp, type="offer") _webrtc_peers[connection.pc_id] = connection bridge = OpenAIRealtimeBridge(state, channel="webrtc") transport = build_webrtc_transport( connection, video_in_enabled=state.vision_enabled, ) task = lifecycle.start_pipeline_task( connection, run_pipeline( transport, state.config, vision_enabled=state.vision_enabled, assistant_id=state.assistant_id, channel="openai-webrtc", protocol_adapter=bridge, ), protocol="openai-webrtc", ) @connection.event_handler("closed") async def on_closed(conn: OpenAIRealtimeWebRTCConnection): _webrtc_peers.pop(conn.pc_id, None) lifecycle.active_connections.discard(conn) await lifecycle.wait_for_pipeline_close( task, connection_id=conn.pc_id, ) answer = connection.get_answer() if not answer: raise RuntimeError("WebRTC answer was not created") return str(answer["sdp"]) def _websocket_token(websocket: WebSocket) -> tuple[str, str | None]: authorization = websocket.headers.get("authorization") if authorization: return bearer_from_authorization(authorization), None protocols = [ item.strip() for item in websocket.headers.get("sec-websocket-protocol", "").split(",") if item.strip() ] for protocol in protocols: prefix = "openai-insecure-api-key." if protocol.startswith(prefix): return protocol.removeprefix(prefix), "realtime" if "realtime" in protocols else None raise RealtimeAuthError("Missing Bearer credential") @dataclass(eq=False) class _ManagedWebSocket: websocket: WebSocket pc_id: str = field(default_factory=lambda: f"ws_{uuid4().hex}") async def disconnect(self) -> None: if self.websocket.application_state == WebSocketState.CONNECTED: await self.websocket.close(code=1001) @router.websocket("") async def realtime_websocket(websocket: WebSocket): managed: _ManagedWebSocket | None = None task = None try: token, accepted_subprotocol = _websocket_token(websocket) async with SessionLocal() as db: credential = await authenticate_bearer(db, token) requested = ( credential.session if credential.ephemeral else {"model": websocket.query_params.get("model")} ) state = await _build_session( credential, requested, safety_identifier_hash=hash_safety_identifier( websocket.headers.get("openai-safety-identifier") ), ) await websocket.accept(subprotocol=accepted_subprotocol) managed = _ManagedWebSocket(websocket) transport = build_openai_websocket_transport(websocket) bridge = OpenAIRealtimeBridge(state, channel="websocket") task = lifecycle.start_pipeline_task( managed, run_pipeline( transport, state.config, vision_enabled=False, assistant_id=state.assistant_id, channel="openai-websocket", protocol_adapter=bridge, ), protocol="openai-websocket", ) await task except (RealtimeAuthError, RealtimeEventError, ValueError) as exc: logger.warning(f"拒绝 OpenAI Realtime WebSocket: {exc}") if websocket.application_state == WebSocketState.CONNECTED: await websocket.close(code=1008, reason=str(exc)[:120]) else: await websocket.close(code=1008) except Exception as exc: # noqa: BLE001 - pipeline callback logs full exception logger.warning(f"OpenAI Realtime WebSocket 已关闭: {type(exc).__name__}") if websocket.application_state == WebSocketState.CONNECTED: await websocket.close(code=1011) finally: if managed: lifecycle.active_connections.discard(managed) if task and not task.done(): await lifecycle.wait_for_pipeline_close( task, connection_id=managed.pc_id if managed else "websocket", )