feat: implement OpenAI-compatible Realtime API with authentication and management features
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
This commit is contained in:
Xin Wang
+24
-116
@@ -8,18 +8,20 @@
|
||||
server → {type:"error", payload:{message}}
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from typing import Any
|
||||
|
||||
from db.session import SessionLocal
|
||||
from fastapi import APIRouter, Body, Depends, Request, WebSocket
|
||||
from loguru import logger
|
||||
from models import AssistantConfig, SignalingOffer
|
||||
from services.auth import require_admin, require_admin_websocket
|
||||
from services.config_resolver import resolve_runtime_config
|
||||
from services.realtime import lifecycle as realtime_lifecycle
|
||||
from services.realtime.launcher import (
|
||||
resolve_assistant_config,
|
||||
validate_visual_runtime,
|
||||
)
|
||||
from services.runtime_variables import DynamicVariableError, prepare_dynamic_config
|
||||
from starlette.websockets import WebSocketDisconnect, WebSocketState
|
||||
|
||||
@@ -29,116 +31,42 @@ from services.webrtc_ice import aiortc_ice_servers, client_ice_servers
|
||||
|
||||
router = APIRouter(tags=["voice"])
|
||||
_http_peers: dict[str, object] = {}
|
||||
_active_connections: set[object] = set()
|
||||
_pipeline_tasks: set[asyncio.Task[None]] = set()
|
||||
_connection_tasks: dict[object, asyncio.Task[None]] = {}
|
||||
_active_connections = realtime_lifecycle.active_connections
|
||||
_pipeline_tasks = realtime_lifecycle.pipeline_tasks
|
||||
_connection_tasks = realtime_lifecycle.connection_tasks
|
||||
PIPELINE_CLOSE_GRACE_SECONDS = 10.0
|
||||
|
||||
|
||||
def _consume_pipeline_result(task: asyncio.Task[None], connection: object) -> None:
|
||||
"""Retrieve background exceptions and release the task's strong reference."""
|
||||
_pipeline_tasks.discard(task)
|
||||
if _connection_tasks.get(connection) is task:
|
||||
_connection_tasks.pop(connection, None)
|
||||
try:
|
||||
error = task.exception()
|
||||
except asyncio.CancelledError:
|
||||
logger.info(f"WebRTC pipeline 已取消: task={task.get_name()}")
|
||||
return
|
||||
if error is not None:
|
||||
logger.opt(exception=error).error(
|
||||
f"WebRTC pipeline 异常结束: task={task.get_name()}"
|
||||
)
|
||||
|
||||
|
||||
def _start_pipeline_task(
|
||||
connection: object,
|
||||
coroutine: Coroutine[Any, Any, None],
|
||||
) -> asyncio.Task[None]:
|
||||
"""Start one strongly referenced pipeline task for a WebRTC connection."""
|
||||
connection_id = str(getattr(connection, "pc_id", "unknown"))
|
||||
task = asyncio.create_task(
|
||||
) -> object:
|
||||
"""Compatibility proxy for existing RTVI callers and tests."""
|
||||
return realtime_lifecycle.start_pipeline_task(
|
||||
connection,
|
||||
coroutine,
|
||||
name=f"webrtc-pipeline:{connection_id}",
|
||||
protocol="webrtc",
|
||||
)
|
||||
_active_connections.add(connection)
|
||||
_pipeline_tasks.add(task)
|
||||
_connection_tasks[connection] = task
|
||||
task.add_done_callback(
|
||||
lambda completed, connection=connection: _consume_pipeline_result(
|
||||
completed,
|
||||
connection,
|
||||
)
|
||||
)
|
||||
return task
|
||||
|
||||
|
||||
async def _wait_for_pipeline_close(
|
||||
task: asyncio.Task[None] | None,
|
||||
task,
|
||||
*,
|
||||
connection_id: str,
|
||||
) -> None:
|
||||
"""Let transport disconnect finish normally, then cancel a stuck pipeline."""
|
||||
if task is None:
|
||||
return
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(task),
|
||||
timeout=PIPELINE_CLOSE_GRACE_SECONDS,
|
||||
)
|
||||
return
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
f"WebRTC pipeline 关闭超过 {PIPELINE_CLOSE_GRACE_SECONDS:g} 秒，"
|
||||
f"执行取消: pc_id={connection_id}"
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
# The done callback owns exception reporting and retrieval.
|
||||
return
|
||||
|
||||
task.cancel()
|
||||
done, _pending = await asyncio.wait(
|
||||
{task},
|
||||
await realtime_lifecycle.wait_for_pipeline_close(
|
||||
task,
|
||||
connection_id=connection_id,
|
||||
timeout=PIPELINE_CLOSE_GRACE_SECONDS,
|
||||
)
|
||||
if not done:
|
||||
logger.error(f"WebRTC pipeline 取消后仍未退出: pc_id={connection_id}")
|
||||
|
||||
|
||||
async def shutdown_active_sessions() -> None:
|
||||
"""Close active peers and drain every managed pipeline during app shutdown."""
|
||||
connections = list(_active_connections)
|
||||
if connections:
|
||||
await asyncio.gather(
|
||||
*(connection.disconnect() for connection in connections),
|
||||
return_exceptions=True,
|
||||
)
|
||||
_active_connections.difference_update(connections)
|
||||
_http_peers.clear()
|
||||
|
||||
tasks = list(_pipeline_tasks)
|
||||
if not tasks:
|
||||
return
|
||||
done, pending = await asyncio.wait(
|
||||
tasks,
|
||||
"""Compatibility proxy; now drains RTVI and OpenAI Realtime sessions."""
|
||||
_http_peers.clear()
|
||||
await realtime_lifecycle.shutdown_active_sessions(
|
||||
timeout=PIPELINE_CLOSE_GRACE_SECONDS,
|
||||
)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
if pending:
|
||||
cancelled, stuck = await asyncio.wait(
|
||||
pending,
|
||||
timeout=PIPELINE_CLOSE_GRACE_SECONDS,
|
||||
)
|
||||
if stuck:
|
||||
logger.error(f"应用关闭时仍有 {len(stuck)} 个 WebRTC pipeline 未退出")
|
||||
else:
|
||||
cancelled = set()
|
||||
logger.info(
|
||||
f"WebRTC 会话清理完成: normal={len(done)} cancelled={len(cancelled)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/webrtc/ice-servers", dependencies=[Depends(require_admin)])
|
||||
@@ -229,12 +157,9 @@ async def voice_signaling(websocket: WebSocket):
|
||||
async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
|
||||
"""优先用 assistant_id 从 DB 解析(含真 key);否则用调试内联配置。"""
|
||||
if offer.assistant_id:
|
||||
async with SessionLocal() as session:
|
||||
cfg = await resolve_runtime_config(session, offer.assistant_id)
|
||||
return prepare_dynamic_config(
|
||||
cfg,
|
||||
offer.dynamic_variables,
|
||||
assistant_id=offer.assistant_id,
|
||||
return await resolve_assistant_config(
|
||||
offer.assistant_id,
|
||||
dynamic_variables=offer.dynamic_variables,
|
||||
)
|
||||
if offer.inline_config:
|
||||
return prepare_dynamic_config(
|
||||
@@ -280,24 +205,7 @@ async def _handle_offer_payload(payload, peers):
|
||||
else:
|
||||
cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连
|
||||
# 服务端助手配置是视觉理解的唯一授权来源；客户端 offer 只负责携带媒体轨。
|
||||
if cfg.type == "workflow":
|
||||
from services.workflow_engine import WorkflowEngine
|
||||
|
||||
vision_enabled = WorkflowEngine(cfg.graph).uses_vision()
|
||||
else:
|
||||
vision_enabled = cfg.vision_enabled
|
||||
if vision_enabled and cfg.type != "workflow":
|
||||
has_native_vision = (
|
||||
not cfg.vision_model_resource_id and cfg.llm_support_image_input
|
||||
)
|
||||
has_aux_vision_model = (
|
||||
bool(cfg.vision_model_resource_id)
|
||||
and cfg.vision_llm_support_image_input
|
||||
)
|
||||
if not (has_native_vision or has_aux_vision_model):
|
||||
raise ValueError(
|
||||
"当前模型不支持图片输入,请在模型资源中选择支持图片输入的视觉模型"
|
||||
)
|
||||
vision_enabled = validate_visual_runtime(cfg)
|
||||
pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
|
||||
if pc_id:
|
||||
pc._pc_id = pc_id
|
||||
|
||||
Reference in New Issue
Block a user