feat: implement OpenAI-compatible Realtime API with authentication and management features

- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections.
- Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys.
- Implemented authentication mechanisms for API keys and client secrets.
- Enhanced environment configuration with new secrets for Realtime API.
- Created OpenAIRealtime session management and event processing for real-time interactions.
- Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.

backend/services/openai_realtime/session.py

@@ -0,0 +1,141 @@
+"""Connection-local state for the OpenAI-compatible Realtime wire protocol."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+from uuid import uuid4
+
+from models import AssistantConfig
+from services.openai_realtime.events import normalize_turn_detection
+
+
+def _id(prefix: str) -> str:
+    return f"{prefix}_{uuid4().hex}"
+
+
+@dataclass
+class OpenAIRealtimeSession:
+    assistant_id: str
+    config: AssistantConfig
+    vision_enabled: bool
+    safety_identifier_hash: str | None = None
+    id: str = field(default_factory=lambda: _id("sess"))
+    output_modalities: list[str] = field(default_factory=lambda: ["audio"])
+    turn_detection: dict[str, Any] | None = field(
+        default_factory=lambda: {
+            "type": "server_vad",
+            "threshold": 0.7,
+            "prefix_padding_ms": 200,
+            "silence_duration_ms": 600,
+            "create_response": True,
+            "interrupt_response": True,
+        }
+    )
+    capabilities: set[str] = field(default_factory=set)
+    pending_item: dict[str, Any] | None = None
+    audio_chunks: list[tuple[bytes, int, int]] = field(default_factory=list)
+    buffered_audio_bytes: int = 0
+    audio_committed: bool = False
+    active_response_id: str | None = None
+    active_output_item_id: str | None = None
+    active_input_item_id: str | None = None
+    response_text: str = ""
+
+    @property
+    def model(self) -> str:
+        return f"assistant:{self.assistant_id}"
+
+    @property
+    def external_turn_control(self) -> bool:
+        return self.turn_detection is None
+
+    @property
+    def output_is_audio(self) -> bool:
+        return self.output_modalities == ["audio"]
+
+    def client_tools(self) -> list[dict[str, Any]]:
+        result = []
+        for tool in self.config.tools:
+            if tool.type != "client":
+                continue
+            definition = tool.definition or {}
+            parameters = (definition.get("config") or {}).get("parameters") or []
+            properties: dict[str, Any] = {}
+            required: list[str] = []
+            for parameter in parameters:
+                name = str(parameter.get("name") or "")
+                if not name:
+                    continue
+                properties[name] = {
+                    "type": parameter.get("type") or "string",
+                    **(
+                        {"description": parameter["description"]}
+                        if parameter.get("description")
+                        else {}
+                    ),
+                }
+                if parameter.get("required"):
+                    required.append(name)
+            result.append(
+                {
+                    "type": "function",
+                    "name": tool.function_name,
+                    "description": tool.description,
+                    "parameters": {
+                        "type": "object",
+                        "properties": properties,
+                        "required": required,
+                    },
+                }
+            )
+        return result
+
+    def public_value(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "type": "realtime",
+            "model": self.model,
+            "output_modalities": self.output_modalities,
+            "audio": {
+                "input": {
+                    "format": {"type": "audio/pcm", "rate": 24000},
+                    "turn_detection": self.turn_detection,
+                },
+                "output": {
+                    "format": {"type": "audio/pcm", "rate": 24000},
+                },
+            },
+            "tools": self.client_tools(),
+            "tool_choice": "auto",
+        }
+
+    def apply_initial_options(self, value: dict[str, Any]) -> None:
+        modalities = value.get("output_modalities")
+        if modalities in (["audio"], ["text"]):
+            self.output_modalities = list(modalities)
+        audio = value.get("audio")
+        if isinstance(audio, dict) and isinstance(audio.get("input"), dict):
+            turn_detection = audio["input"].get("turn_detection", self.turn_detection)
+        else:
+            turn_detection = value.get("turn_detection", self.turn_detection)
+        self.turn_detection = normalize_turn_detection(turn_detection)
+
+    def begin_response(self) -> tuple[str, str]:
+        if not self.active_response_id:
+            self.active_response_id = _id("resp")
+        if not self.active_output_item_id:
+            self.active_output_item_id = _id("item")
+            self.response_text = ""
+        return self.active_response_id, str(self.active_output_item_id)
+
+    def finish_response(self) -> tuple[str | None, str | None, str]:
+        result = (
+            self.active_response_id,
+            self.active_output_item_id,
+            self.response_text,
+        )
+        self.active_response_id = None
+        self.active_output_item_id = None
+        self.response_text = ""
+        return result