feat: implement OpenAI-compatible Realtime API with authentication and management features
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
This commit is contained in:
Xin Wang
@@ -0,0 +1,341 @@
|
||||
"""Public OpenAI-compatible Realtime HTTP, WebRTC, and WebSocket entries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from db.session import SessionLocal
|
||||
from fastapi import APIRouter, Request, WebSocket
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from loguru import logger
|
||||
from starlette.websockets import WebSocketState
|
||||
|
||||
from services.openai_realtime.auth import (
|
||||
RealtimeAuthError,
|
||||
RealtimeCredential,
|
||||
authenticate_bearer,
|
||||
bearer_from_authorization,
|
||||
create_client_secret,
|
||||
hash_safety_identifier,
|
||||
)
|
||||
from services.openai_realtime.bridge import OpenAIRealtimeBridge
|
||||
from services.openai_realtime.events import (
|
||||
RealtimeEventError,
|
||||
assistant_id_from_model,
|
||||
)
|
||||
from services.openai_realtime.session import OpenAIRealtimeSession
|
||||
from services.openai_realtime.webrtc import OpenAIRealtimeWebRTCConnection
|
||||
from services.openai_realtime.websocket import build_openai_websocket_transport
|
||||
from services.pipecat.pipeline import run_pipeline
|
||||
from services.pipecat.transports import build_webrtc_transport
|
||||
from services.realtime import lifecycle
|
||||
from services.realtime.launcher import (
|
||||
resolve_assistant_config,
|
||||
validate_runtime_requirements,
|
||||
validate_visual_runtime,
|
||||
)
|
||||
from services.webrtc_ice import aiortc_ice_servers
|
||||
|
||||
|
||||
router = APIRouter(prefix="/v1/realtime", tags=["openai-realtime"])
|
||||
_webrtc_peers: dict[str, OpenAIRealtimeWebRTCConnection] = {}
|
||||
|
||||
|
||||
def _api_error(message: str, *, code: str, status_code: int) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": code,
|
||||
"message": message,
|
||||
"param": None,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _credential(request: Request) -> RealtimeCredential:
|
||||
token = bearer_from_authorization(request.headers.get("authorization"))
|
||||
async with SessionLocal() as db:
|
||||
return await authenticate_bearer(db, token)
|
||||
|
||||
|
||||
def _session_payload(value: object) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise RealtimeEventError("session must be an object", param="session")
|
||||
session = value.get("session", value)
|
||||
if not isinstance(session, dict):
|
||||
raise RealtimeEventError("session must be an object", param="session")
|
||||
return session
|
||||
|
||||
|
||||
def _reject_locked_creation_fields(session: dict[str, Any]) -> None:
|
||||
locked = {"instructions", "voice", "tools", "tool_choice"}.intersection(session)
|
||||
audio = session.get("audio")
|
||||
if isinstance(audio, dict) and isinstance(audio.get("output"), dict):
|
||||
if "voice" in audio["output"]:
|
||||
locked.add("audio.output.voice")
|
||||
if locked:
|
||||
field = sorted(locked)[0]
|
||||
raise RealtimeEventError(
|
||||
f"{field} is owned by the selected assistant",
|
||||
code="immutable_session_field",
|
||||
param=f"session.{field}",
|
||||
)
|
||||
modalities = session.get("output_modalities")
|
||||
if modalities is not None and modalities not in (["audio"], ["text"]):
|
||||
raise RealtimeEventError(
|
||||
'output_modalities must be ["audio"] or ["text"]',
|
||||
param="session.output_modalities",
|
||||
)
|
||||
|
||||
|
||||
async def _build_session(
|
||||
credential: RealtimeCredential,
|
||||
requested: dict[str, Any] | None,
|
||||
*,
|
||||
safety_identifier_hash: str | None,
|
||||
) -> OpenAIRealtimeSession:
|
||||
session_value = dict(credential.session or requested or {})
|
||||
_reject_locked_creation_fields(session_value)
|
||||
requested_assistant_id = assistant_id_from_model(session_value.get("model"))
|
||||
if credential.assistant_id and requested_assistant_id != credential.assistant_id:
|
||||
raise RealtimeEventError(
|
||||
"Client secret is bound to another assistant",
|
||||
code="invalid_model",
|
||||
param="session.model",
|
||||
)
|
||||
config = await resolve_assistant_config(requested_assistant_id)
|
||||
validate_runtime_requirements(config)
|
||||
vision_enabled = validate_visual_runtime(config)
|
||||
state = OpenAIRealtimeSession(
|
||||
assistant_id=requested_assistant_id,
|
||||
config=config,
|
||||
vision_enabled=vision_enabled,
|
||||
safety_identifier_hash=(
|
||||
credential.safety_identifier_hash or safety_identifier_hash
|
||||
),
|
||||
)
|
||||
state.apply_initial_options(session_value)
|
||||
return state
|
||||
|
||||
|
||||
@router.post("/client_secrets")
|
||||
async def create_realtime_client_secret(request: Request):
|
||||
try:
|
||||
credential = await _credential(request)
|
||||
if credential.ephemeral:
|
||||
raise RealtimeAuthError("A long-lived API key is required")
|
||||
body = await request.json()
|
||||
session = _session_payload(body)
|
||||
_reject_locked_creation_fields(session)
|
||||
assistant_id = assistant_id_from_model(session.get("model"))
|
||||
# Resolve now so invalid models/configurations fail before a browser gets a token.
|
||||
config = await resolve_assistant_config(assistant_id)
|
||||
validate_runtime_requirements(config)
|
||||
validate_visual_runtime(config)
|
||||
safety_hash = hash_safety_identifier(
|
||||
request.headers.get("openai-safety-identifier")
|
||||
)
|
||||
value, expires_at = create_client_secret(
|
||||
api_key_id=credential.api_key_id,
|
||||
assistant_id=assistant_id,
|
||||
session=session,
|
||||
safety_identifier_hash=safety_hash,
|
||||
)
|
||||
return {
|
||||
"value": value,
|
||||
"expires_at": expires_at,
|
||||
"session": session,
|
||||
}
|
||||
except RealtimeAuthError as exc:
|
||||
return _api_error(str(exc), code="invalid_api_key", status_code=401)
|
||||
except RealtimeEventError as exc:
|
||||
return _api_error(str(exc), code=exc.code, status_code=400)
|
||||
except ValueError as exc:
|
||||
code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session"
|
||||
return _api_error(str(exc), code=code, status_code=400)
|
||||
|
||||
|
||||
@router.post("/calls")
|
||||
async def create_realtime_call(request: Request):
|
||||
try:
|
||||
credential = await _credential(request)
|
||||
content_type = request.headers.get("content-type", "").lower()
|
||||
if content_type.startswith("multipart/form-data"):
|
||||
if credential.ephemeral:
|
||||
raise RealtimeAuthError(
|
||||
"Client secrets must send an application/sdp offer"
|
||||
)
|
||||
form = await request.form()
|
||||
sdp = str(form.get("sdp") or "")
|
||||
raw_session = str(form.get("session") or "{}")
|
||||
requested = _session_payload(json.loads(raw_session))
|
||||
elif content_type.startswith("application/sdp"):
|
||||
if not credential.ephemeral:
|
||||
raise RealtimeAuthError(
|
||||
"application/sdp requires a short-lived client secret"
|
||||
)
|
||||
sdp = (await request.body()).decode("utf-8")
|
||||
requested = None
|
||||
else:
|
||||
raise RealtimeEventError(
|
||||
"Use multipart/form-data or application/sdp",
|
||||
code="unsupported_content_type",
|
||||
)
|
||||
if not sdp.strip():
|
||||
raise RealtimeEventError("SDP offer is empty", param="sdp")
|
||||
state = await _build_session(
|
||||
credential,
|
||||
requested,
|
||||
safety_identifier_hash=hash_safety_identifier(
|
||||
request.headers.get("openai-safety-identifier")
|
||||
),
|
||||
)
|
||||
answer = await _start_webrtc(sdp, state)
|
||||
return Response(
|
||||
content=answer,
|
||||
media_type="application/sdp",
|
||||
headers={"Location": f"/v1/realtime/calls/{state.id}"},
|
||||
)
|
||||
except RealtimeAuthError as exc:
|
||||
return _api_error(str(exc), code="invalid_api_key", status_code=401)
|
||||
except (RealtimeEventError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
code = exc.code if isinstance(exc, RealtimeEventError) else "invalid_request_error"
|
||||
return _api_error(str(exc), code=code, status_code=400)
|
||||
except ValueError as exc:
|
||||
code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session"
|
||||
return _api_error(str(exc), code=code, status_code=400)
|
||||
except Exception as exc: # noqa: BLE001 - no internal details in public response
|
||||
logger.exception(f"OpenAI Realtime WebRTC 启动失败: {exc}")
|
||||
return _api_error(
|
||||
"Realtime connection could not be established",
|
||||
code="connection_error",
|
||||
status_code=500,
|
||||
)
|
||||
|
||||
|
||||
async def _start_webrtc(sdp: str, state: OpenAIRealtimeSession) -> str:
|
||||
connection = OpenAIRealtimeWebRTCConnection(
|
||||
ice_servers=aiortc_ice_servers()
|
||||
)
|
||||
await connection.initialize(sdp=sdp, type="offer")
|
||||
_webrtc_peers[connection.pc_id] = connection
|
||||
bridge = OpenAIRealtimeBridge(state, channel="webrtc")
|
||||
transport = build_webrtc_transport(
|
||||
connection,
|
||||
video_in_enabled=state.vision_enabled,
|
||||
)
|
||||
task = lifecycle.start_pipeline_task(
|
||||
connection,
|
||||
run_pipeline(
|
||||
transport,
|
||||
state.config,
|
||||
vision_enabled=state.vision_enabled,
|
||||
assistant_id=state.assistant_id,
|
||||
channel="openai-webrtc",
|
||||
protocol_adapter=bridge,
|
||||
),
|
||||
protocol="openai-webrtc",
|
||||
)
|
||||
|
||||
@connection.event_handler("closed")
|
||||
async def on_closed(conn: OpenAIRealtimeWebRTCConnection):
|
||||
_webrtc_peers.pop(conn.pc_id, None)
|
||||
lifecycle.active_connections.discard(conn)
|
||||
await lifecycle.wait_for_pipeline_close(
|
||||
task,
|
||||
connection_id=conn.pc_id,
|
||||
)
|
||||
|
||||
answer = connection.get_answer()
|
||||
if not answer:
|
||||
raise RuntimeError("WebRTC answer was not created")
|
||||
return str(answer["sdp"])
|
||||
|
||||
|
||||
def _websocket_token(websocket: WebSocket) -> tuple[str, str | None]:
|
||||
authorization = websocket.headers.get("authorization")
|
||||
if authorization:
|
||||
return bearer_from_authorization(authorization), None
|
||||
protocols = [
|
||||
item.strip()
|
||||
for item in websocket.headers.get("sec-websocket-protocol", "").split(",")
|
||||
if item.strip()
|
||||
]
|
||||
for protocol in protocols:
|
||||
prefix = "openai-insecure-api-key."
|
||||
if protocol.startswith(prefix):
|
||||
return protocol.removeprefix(prefix), "realtime" if "realtime" in protocols else None
|
||||
raise RealtimeAuthError("Missing Bearer credential")
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class _ManagedWebSocket:
|
||||
websocket: WebSocket
|
||||
pc_id: str = field(default_factory=lambda: f"ws_{uuid4().hex}")
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.websocket.application_state == WebSocketState.CONNECTED:
|
||||
await self.websocket.close(code=1001)
|
||||
|
||||
|
||||
@router.websocket("")
|
||||
async def realtime_websocket(websocket: WebSocket):
|
||||
managed: _ManagedWebSocket | None = None
|
||||
task = None
|
||||
try:
|
||||
token, accepted_subprotocol = _websocket_token(websocket)
|
||||
async with SessionLocal() as db:
|
||||
credential = await authenticate_bearer(db, token)
|
||||
requested = (
|
||||
credential.session
|
||||
if credential.ephemeral
|
||||
else {"model": websocket.query_params.get("model")}
|
||||
)
|
||||
state = await _build_session(
|
||||
credential,
|
||||
requested,
|
||||
safety_identifier_hash=hash_safety_identifier(
|
||||
websocket.headers.get("openai-safety-identifier")
|
||||
),
|
||||
)
|
||||
await websocket.accept(subprotocol=accepted_subprotocol)
|
||||
managed = _ManagedWebSocket(websocket)
|
||||
transport = build_openai_websocket_transport(websocket)
|
||||
bridge = OpenAIRealtimeBridge(state, channel="websocket")
|
||||
task = lifecycle.start_pipeline_task(
|
||||
managed,
|
||||
run_pipeline(
|
||||
transport,
|
||||
state.config,
|
||||
vision_enabled=False,
|
||||
assistant_id=state.assistant_id,
|
||||
channel="openai-websocket",
|
||||
protocol_adapter=bridge,
|
||||
),
|
||||
protocol="openai-websocket",
|
||||
)
|
||||
await task
|
||||
except (RealtimeAuthError, RealtimeEventError, ValueError) as exc:
|
||||
logger.warning(f"拒绝 OpenAI Realtime WebSocket: {exc}")
|
||||
if websocket.application_state == WebSocketState.CONNECTED:
|
||||
await websocket.close(code=1008, reason=str(exc)[:120])
|
||||
else:
|
||||
await websocket.close(code=1008)
|
||||
except Exception as exc: # noqa: BLE001 - pipeline callback logs full exception
|
||||
logger.warning(f"OpenAI Realtime WebSocket 已关闭: {type(exc).__name__}")
|
||||
if websocket.application_state == WebSocketState.CONNECTED:
|
||||
await websocket.close(code=1011)
|
||||
finally:
|
||||
if managed:
|
||||
lifecycle.active_connections.discard(managed)
|
||||
if task and not task.done():
|
||||
await lifecycle.wait_for_pipeline_close(
|
||||
task,
|
||||
connection_id=managed.pc_id if managed else "websocket",
|
||||
)
|
||||
Reference in New Issue
Block a user