feat: implement OpenAI-compatible Realtime API with authentication and management features
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
This commit is contained in:
Xin Wang
@@ -0,0 +1,195 @@
|
||||
"""Resolve and validate an assistant before creating a media connection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from db.session import SessionLocal
|
||||
from models import AssistantConfig
|
||||
from services.config_resolver import resolve_runtime_config
|
||||
from services.node_specs import graph_references
|
||||
from services.runtime_variables import prepare_dynamic_config
|
||||
from services.workflow_engine import WorkflowEngine
|
||||
|
||||
|
||||
async def resolve_assistant_config(
|
||||
assistant_id: str,
|
||||
*,
|
||||
dynamic_variables: dict[str, Any] | None = None,
|
||||
) -> AssistantConfig:
|
||||
async with SessionLocal() as session:
|
||||
config = await resolve_runtime_config(session, assistant_id)
|
||||
return prepare_dynamic_config(
|
||||
config,
|
||||
dynamic_variables or {},
|
||||
assistant_id=assistant_id,
|
||||
)
|
||||
|
||||
|
||||
def validate_visual_runtime(config: AssistantConfig) -> bool:
|
||||
"""Return the authoritative video-input permission or fail before connect."""
|
||||
|
||||
if config.type == "workflow":
|
||||
return WorkflowEngine(config.graph).uses_vision()
|
||||
|
||||
vision_enabled = config.vision_enabled
|
||||
if not vision_enabled:
|
||||
return False
|
||||
has_native_vision = (
|
||||
not config.vision_model_resource_id and config.llm_support_image_input
|
||||
)
|
||||
has_aux_vision_model = (
|
||||
bool(config.vision_model_resource_id)
|
||||
and config.vision_llm_support_image_input
|
||||
)
|
||||
if not (has_native_vision or has_aux_vision_model):
|
||||
raise ValueError(
|
||||
"当前模型不支持图片输入,请在模型资源中选择支持图片输入的视觉模型"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _require_values(labels: list[tuple[str, Any]]) -> None:
|
||||
missing = [label for label, value in labels if not value]
|
||||
if missing:
|
||||
raise ValueError(f"助手运行配置不完整: {', '.join(missing)}")
|
||||
|
||||
|
||||
def _validate_voice_resource(
|
||||
capability: str,
|
||||
*,
|
||||
interface_type: str,
|
||||
values: dict[str, Any],
|
||||
secrets: dict[str, Any],
|
||||
) -> None:
|
||||
labels: list[tuple[str, Any]] = []
|
||||
if capability == "ASR":
|
||||
if interface_type not in {"openai-asr", "dashscope-asr", "xfyun-asr"}:
|
||||
raise ValueError(f"不支持的 ASR 接口类型: {interface_type}")
|
||||
if interface_type == "xfyun-asr":
|
||||
labels.extend(
|
||||
(f"ASR {key}", secrets.get(key))
|
||||
for key in ("appId", "apiKey", "apiSecret")
|
||||
)
|
||||
else:
|
||||
labels.extend(
|
||||
[
|
||||
("ASR modelId", values.get("modelId")),
|
||||
("ASR apiUrl", values.get("apiUrl")),
|
||||
("ASR apiKey", secrets.get("apiKey")),
|
||||
]
|
||||
)
|
||||
elif capability == "TTS":
|
||||
if interface_type not in {
|
||||
"openai-tts",
|
||||
"dashscope-tts",
|
||||
"xfyun-tts",
|
||||
"xfyun-super-tts",
|
||||
}:
|
||||
raise ValueError(f"不支持的 TTS 接口类型: {interface_type}")
|
||||
labels.append(("TTS voice", values.get("voice")))
|
||||
if interface_type in {"xfyun-tts", "xfyun-super-tts"}:
|
||||
labels.extend(
|
||||
(f"TTS {key}", secrets.get(key))
|
||||
for key in ("appId", "apiKey", "apiSecret")
|
||||
)
|
||||
else:
|
||||
labels.extend(
|
||||
[
|
||||
("TTS modelId", values.get("modelId")),
|
||||
("TTS apiUrl", values.get("apiUrl")),
|
||||
("TTS apiKey", secrets.get("apiKey")),
|
||||
]
|
||||
)
|
||||
elif capability == "LLM":
|
||||
if interface_type not in {"openai-llm", "dashscope-llm"}:
|
||||
raise ValueError(f"不支持的 LLM 接口类型: {interface_type}")
|
||||
labels.extend(
|
||||
[
|
||||
("LLM modelId", values.get("modelId")),
|
||||
("LLM apiUrl", values.get("apiUrl")),
|
||||
("LLM apiKey", secrets.get("apiKey")),
|
||||
]
|
||||
)
|
||||
_require_values(labels)
|
||||
|
||||
|
||||
def validate_runtime_requirements(config: AssistantConfig) -> None:
|
||||
"""Reject structurally incomplete assistants before media negotiation."""
|
||||
|
||||
if config.type not in {"prompt", "workflow", "dify", "fastgpt"}:
|
||||
raise ValueError(f"当前助手类型不支持 Realtime API: {config.type}")
|
||||
if config.runtimeMode == "realtime":
|
||||
if config.type not in {"prompt", "workflow"}:
|
||||
raise ValueError(f"助手类型 {config.type} 不支持 realtime 运行模式")
|
||||
if config.realtime_interface_type not in {
|
||||
"qwen-audio-realtime",
|
||||
"stepfun-realtime",
|
||||
}:
|
||||
raise ValueError(
|
||||
f"不支持的 Realtime 接口类型: {config.realtime_interface_type}"
|
||||
)
|
||||
_require_values(
|
||||
[
|
||||
("Realtime interfaceType", config.realtime_interface_type),
|
||||
("Realtime modelId", config.realtimeModel),
|
||||
("Realtime apiUrl", config.realtime_base_url),
|
||||
("Realtime apiKey", config.realtime_api_key),
|
||||
],
|
||||
)
|
||||
return
|
||||
|
||||
_validate_voice_resource(
|
||||
"ASR",
|
||||
interface_type=config.stt_interface_type,
|
||||
values=config.stt_values,
|
||||
secrets=config.stt_secrets,
|
||||
)
|
||||
_validate_voice_resource(
|
||||
"TTS",
|
||||
interface_type=config.tts_interface_type,
|
||||
values=config.tts_values,
|
||||
secrets=config.tts_secrets,
|
||||
)
|
||||
if config.type == "prompt":
|
||||
_validate_voice_resource(
|
||||
"LLM",
|
||||
interface_type=config.llm_interface_type,
|
||||
values=config.llm_values,
|
||||
secrets=config.llm_secrets,
|
||||
)
|
||||
elif config.type == "dify":
|
||||
_require_values(
|
||||
[
|
||||
("Dify apiUrl", config.dify_api_url),
|
||||
("Dify apiKey", config.dify_api_key),
|
||||
],
|
||||
)
|
||||
elif config.type == "fastgpt":
|
||||
_require_values(
|
||||
[
|
||||
("FastGPT apiUrl", config.fastgpt_api_url),
|
||||
("FastGPT apiKey", config.fastgpt_api_key),
|
||||
],
|
||||
)
|
||||
|
||||
if config.type != "workflow":
|
||||
return
|
||||
references = graph_references(config.graph)
|
||||
missing_models = references["model_resources"] - set(
|
||||
config.workflow_model_resources
|
||||
)
|
||||
missing_knowledge = references["knowledge_bases"] - set(
|
||||
config.workflow_knowledge_bases
|
||||
)
|
||||
if missing_models or missing_knowledge:
|
||||
missing = sorted([*missing_models, *missing_knowledge])
|
||||
raise ValueError(f"Workflow 引用了不可用资源: {', '.join(missing)}")
|
||||
for resource in config.workflow_model_resources.values():
|
||||
if resource.capability in {"ASR", "TTS", "LLM"}:
|
||||
_validate_voice_resource(
|
||||
resource.capability,
|
||||
interface_type=resource.interface_type,
|
||||
values=resource.values,
|
||||
secrets=resource.secrets,
|
||||
)
|
||||
Reference in New Issue
Block a user