feat: implement OpenAI-compatible Realtime API with authentication and management features

- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections.
- Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys.
- Implemented authentication mechanisms for API keys and client secrets.
- Enhanced environment configuration with new secrets for Realtime API.
- Created OpenAIRealtime session management and event processing for real-time interactions.
- Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.

backend/services/pipecat/stepfun_realtime.py

@@ -35,6 +35,7 @@ from services.pipecat.realtime_tools import (
     RealtimeToolDispatcher,
     RealtimeToolSession,
 )
+from services.realtime.protocol import RealtimeProviderControlFrame
 
 DEFAULT_STEPFUN_REALTIME_URL = "wss://api.stepfun.com/v1/realtime"
 SpeechStartedHandler = Callable[[], Awaitable[None]]
@@ -69,6 +70,7 @@ class StepFunRealtimeService(AIService):
         self._prefix_padding_ms = prefix_padding_ms
         self._silence_duration_ms = silence_duration_ms
         self._energy_awakeness_threshold = energy_awakeness_threshold
+        self._turn_detection_enabled = True
         self._warned_input_sample_rate = False
         self._websocket = None
         self._receive_task: asyncio.Task | None = None
@@ -115,6 +117,15 @@ class StepFunRealtimeService(AIService):
     async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
         await super().process_frame(frame, direction)
 
+        if isinstance(frame, RealtimeProviderControlFrame):
+            if frame.action == "commit_audio":
+                await self.commit_audio_buffer()
+            elif frame.action == "clear_audio":
+                await self.clear_audio_buffer()
+            else:
+                await self.request_response()
+            return
+
         if isinstance(frame, InputAudioRawFrame):
             if (
                 frame.sample_rate != self._input_sample_rate
@@ -183,6 +194,23 @@ class StepFunRealtimeService(AIService):
             )
         await self._send_event({"type": "response.create"})
 
+    async def commit_audio_buffer(self) -> None:
+        await self._send_event({"type": "input_audio_buffer.commit"})
+
+    async def clear_audio_buffer(self) -> None:
+        await self._send_event({"type": "input_audio_buffer.clear"})
+
+    async def update_turn_detection(self, config: dict[str, Any] | None) -> None:
+        self._turn_detection_enabled = config is not None
+        if self._session_ready.is_set():
+            await self._send_event(
+                {
+                    "type": "session.update",
+                    "session": {"turn_detection": self._turn_detection_config()},
+                },
+                wait_until_ready=False,
+            )
+
     async def wait_for_response_boundary(self) -> None:
         """Wait until StepFun has finished or cancelled the current response."""
         await self._response_done.wait()
@@ -386,12 +414,7 @@ class StepFunRealtimeService(AIService):
                     "voice": self._voice,
                     "input_audio_format": "pcm16",
                     "output_audio_format": "pcm16",
-                    "turn_detection": {
-                        "type": "server_vad",
-                        "prefix_padding_ms": self._prefix_padding_ms,
-                        "silence_duration_ms": self._silence_duration_ms,
-                        "energy_awakeness_threshold": self._energy_awakeness_threshold,
-                    },
+                    "turn_detection": self._turn_detection_config(),
                     "tools": [tool.provider_schema() for tool in self._tools],
                     "tool_choice": "auto",
                 },
@@ -399,6 +422,16 @@ class StepFunRealtimeService(AIService):
             wait_until_ready=False,
         )
 
+    def _turn_detection_config(self) -> dict[str, Any] | None:
+        if not self._turn_detection_enabled:
+            return None
+        return {
+            "type": "server_vad",
+            "prefix_padding_ms": self._prefix_padding_ms,
+            "silence_duration_ms": self._silence_duration_ms,
+            "energy_awakeness_threshold": self._energy_awakeness_threshold,
+        }
+
     async def update_instructions(self, instructions: str) -> None:
         """Refresh model instructions without rebuilding the realtime session."""
         self._instructions = instructions