wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement OpenAI-compatible Realtime API with authentication and management features

Browse Source 
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections.
- Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys.
- Implemented authentication mechanisms for API keys and client secrets.
- Enhanced environment configuration with new secrets for Realtime API.
- Created OpenAIRealtime session management and event processing for real-time interactions.
- Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
This commit is contained in:
Xin Wang
2026-08-11 10:05:55 +08:00
parent cf32ea605d
commit 86639692ba
28 changed files with 3286 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/pipecat/pipeline.py
+115
View File
@@ -47,6 +47,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.flows import FlowsFunctionSchema
from pipecat.frames.frames import (
EndFrame,
InterruptionFrame,
LLMMessagesAppendFrame,
OutputTransportMessageUrgentFrame,
UserImageRawFrame,
@@ -94,6 +95,11 @@ from services.pipecat.pipeline_events import (
bind_cascade_pipeline_events,
bind_realtime_pipeline_events,
)
from services.realtime.protocol import (
PipelineProtocolAdapter,
PipelineProtocolRuntime,
RealtimeProviderControlFrame,
)
from pipecat.workers.runner import WorkerRunner
@@ -201,6 +207,7 @@ async def run_pipeline(
vision_enabled: bool = False,
assistant_id: str | None = None,
channel: str = "webrtc",
protocol_adapter: PipelineProtocolAdapter | None = None,
) -> None:
"""在给定 transport 上构建并运行管线,直到连接结束。
@@ -234,6 +241,7 @@ async def run_pipeline(
vision_enabled=vision_enabled,
assistant_id=assistant_id,
channel=channel,
protocol_adapter=protocol_adapter,
)
return
@@ -626,15 +634,22 @@ async def run_pipeline(
analysis_plan=cfg.analysis_config,
extra=(workflow_engine.session_metadata() if workflow_engine else None),
)
protocol_inputs = protocol_adapter.input_processors() if protocol_adapter else []
protocol_inference = (
protocol_adapter.inference_processors() if protocol_adapter else []
)
protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
pipeline = Pipeline(
[
transport.input(),
*protocol_inputs,
client_tools,
session_update,
vision_capture,
user_input,
stt_processor,
user_aggregator,
*protocol_inference,
user_turn_router,
knowledge_retrieval,
llm,
@@ -645,6 +660,7 @@ async def run_pipeline(
tts_processor,
EndCallAfterSpeechProcessor(call_end),
ConversationHistoryProcessor(recorder),
*protocol_outputs,
transport.output(),
]
)
@@ -657,6 +673,47 @@ async def run_pipeline(
enable_rtvi=False,
)
worker_holder["worker"] = worker
protocol_turn_state: dict[str, bool | None] = {
"external": False,
"interrupt_response": None,
}
if protocol_adapter:
async def set_external_turn_control(enabled: bool) -> None:
protocol_turn_state["external"] = enabled
await user_aggregator.apply_external_turn_control(enabled)
async def request_response() -> None:
# Cascade inference is triggered when the adapter submits user-input.
return None
async def commit_audio() -> None:
return None
async def clear_audio() -> None:
return None
async def set_response_interruption(enabled: bool) -> None:
protocol_turn_state["interrupt_response"] = enabled
await user_aggregator.apply_turn_strategies(
cfg.turnConfig,
enable_interruptions=enabled,
)
async def cancel_response() -> None:
await worker.queue_frame(InterruptionFrame())
await protocol_adapter.bind(
PipelineProtocolRuntime(
queue_frame=worker.queue_frame,
set_external_turn_control=set_external_turn_control,
set_response_interruption=set_response_interruption,
commit_audio=commit_audio,
clear_audio=clear_audio,
request_response=request_response,
cancel_response=cancel_response,
)
)
service_controller = WorkflowServiceController(
worker=worker,
llm_services=llm_services,
@@ -685,6 +742,15 @@ async def run_pipeline(
normalized,
enable_interruptions=enable_interrupt,
)
if protocol_turn_state["external"]:
await user_aggregator.apply_external_turn_control(True)
elif protocol_turn_state["interrupt_response"] is not None:
await user_aggregator.apply_turn_strategies(
normalized,
enable_interruptions=bool(
protocol_turn_state["interrupt_response"]
),
)
await worker.queue_frame(
VADParamsUpdateFrame(params=create_vad_params(normalized))
)
@@ -872,6 +938,7 @@ async def run_realtime_pipeline(
vision_enabled: bool = False,
assistant_id: str | None = None,
channel: str = "webrtc",
protocol_adapter: PipelineProtocolAdapter | None = None,
) -> None:
"""Run a speech-to-speech model that owns ASR, reasoning, and synthesis."""
realtime = create_realtime_service(
@@ -960,18 +1027,26 @@ async def run_realtime_pipeline(
else None
),
)
protocol_inputs = protocol_adapter.input_processors() if protocol_adapter else []
protocol_inference = (
protocol_adapter.inference_processors() if protocol_adapter else []
)
protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
pipeline = Pipeline(
[
transport.input(),
*protocol_inputs,
vision_capture,
client_tools,
session_update,
user_input,
input_gate,
*protocol_inference,
realtime,
dynamic_variables,
EndCallAfterSpeechProcessor(call_end),
ConversationHistoryProcessor(recorder),
*protocol_outputs,
transport.output(),
]
)
@@ -985,6 +1060,46 @@ async def run_realtime_pipeline(
enable_rtvi=False,
)
worker_holder["worker"] = worker
if protocol_adapter:
async def set_external_turn_control(enabled: bool) -> None:
update = getattr(realtime, "update_turn_detection", None)
if callable(update):
await update(None if enabled else cfg.turnConfig)
async def request_response() -> None:
await worker.queue_frame(
RealtimeProviderControlFrame(action="request_response")
)
async def commit_audio() -> None:
await worker.queue_frame(
RealtimeProviderControlFrame(action="commit_audio")
)
async def clear_audio() -> None:
await worker.queue_frame(
RealtimeProviderControlFrame(action="clear_audio")
)
async def cancel_response() -> None:
await realtime.interrupt()
async def set_response_interruption(_enabled: bool) -> None:
# Provider-side settings vary; explicit response.cancel remains portable.
return None
await protocol_adapter.bind(
PipelineProtocolRuntime(
queue_frame=worker.queue_frame,
set_external_turn_control=set_external_turn_control,
set_response_interruption=set_response_interruption,
commit_audio=commit_audio,
clear_audio=clear_audio,
request_response=request_response,
cancel_response=cancel_response,
)
)
def set_input_enabled(enabled: bool) -> None:
input_state["enabled"] = enabled
backend/services/pipecat/qwen_audio_realtime.py
+28 -2
View File
@@ -41,13 +41,14 @@ from services.pipecat.realtime_tools import (
RealtimeToolDispatcher,
RealtimeToolSession,
)
from services.realtime.protocol import RealtimeProviderControlFrame
DEFAULT_QWEN_AUDIO_REALTIME_MODEL = "qwen-audio-3.0-realtime-flash"
DEFAULT_QWEN_AUDIO_REALTIME_VOICE = "longanqian"
QWEN_INPUT_SAMPLE_RATE = 16_000
QWEN_OUTPUT_SAMPLE_RATE = 24_000
SUPPORTED_TURN_DETECTION_MODES = frozenset({"server_vad", "smart_turn"})
SUPPORTED_TURN_DETECTION_MODES = frozenset({"none", "server_vad", "smart_turn"})
ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None]
SpeechStartedHandler = Callable[[], Awaitable[None]]
@@ -161,6 +162,15 @@ class QwenAudioRealtimeService(AIService):
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
await super().process_frame(frame, direction)
if isinstance(frame, RealtimeProviderControlFrame):
if frame.action == "commit_audio":
await self.commit_audio_buffer()
elif frame.action == "clear_audio":
await self.clear_audio_buffer()
else:
await self.request_response()
return
if isinstance(frame, InputAudioRawFrame):
if (
frame.sample_rate != self._input_sample_rate
@@ -234,6 +244,20 @@ class QwenAudioRealtimeService(AIService):
)
await self._send_event({"type": "response.create"})
async def commit_audio_buffer(self) -> None:
await self._send_event({"type": "input_audio_buffer.commit"})
async def clear_audio_buffer(self) -> None:
await self._send_event({"type": "input_audio_buffer.clear"})
async def update_turn_detection(self, config: dict[str, Any] | None) -> None:
"""Apply OpenAI PTT before provider startup; reject unsafe live changes."""
mode = "none" if config is None else "server_vad"
if self._session_ready.is_set():
raise ValueError("Qwen Realtime 不支持会话建立后切换轮次检测")
self._turn_detection_mode = mode
async def wait_for_response_boundary(self) -> None:
"""Wait until Qwen has finished or cancelled the current response."""
await self._response_done.wait()
@@ -355,7 +379,9 @@ class QwenAudioRealtimeService(AIService):
"tool_choice": "auto",
}
def _turn_detection_config(self) -> dict[str, Any]:
def _turn_detection_config(self) -> dict[str, Any] | None:
if self._turn_detection_mode == "none":
return None
if self._turn_detection_mode == "smart_turn":
return {"type": "smart_turn"}
return {
backend/services/pipecat/stepfun_realtime.py
+39 -6
View File
@@ -35,6 +35,7 @@ from services.pipecat.realtime_tools import (
RealtimeToolDispatcher,
RealtimeToolSession,
)
from services.realtime.protocol import RealtimeProviderControlFrame
DEFAULT_STEPFUN_REALTIME_URL = "wss://api.stepfun.com/v1/realtime"
SpeechStartedHandler = Callable[[], Awaitable[None]]
@@ -69,6 +70,7 @@ class StepFunRealtimeService(AIService):
self._prefix_padding_ms = prefix_padding_ms
self._silence_duration_ms = silence_duration_ms
self._energy_awakeness_threshold = energy_awakeness_threshold
self._turn_detection_enabled = True
self._warned_input_sample_rate = False
self._websocket = None
self._receive_task: asyncio.Task | None = None
@@ -115,6 +117,15 @@ class StepFunRealtimeService(AIService):
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
await super().process_frame(frame, direction)
if isinstance(frame, RealtimeProviderControlFrame):
if frame.action == "commit_audio":
await self.commit_audio_buffer()
elif frame.action == "clear_audio":
await self.clear_audio_buffer()
else:
await self.request_response()
return
if isinstance(frame, InputAudioRawFrame):
if (
frame.sample_rate != self._input_sample_rate
@@ -183,6 +194,23 @@ class StepFunRealtimeService(AIService):
)
await self._send_event({"type": "response.create"})
async def commit_audio_buffer(self) -> None:
await self._send_event({"type": "input_audio_buffer.commit"})
async def clear_audio_buffer(self) -> None:
await self._send_event({"type": "input_audio_buffer.clear"})
async def update_turn_detection(self, config: dict[str, Any] | None) -> None:
self._turn_detection_enabled = config is not None
if self._session_ready.is_set():
await self._send_event(
{
"type": "session.update",
"session": {"turn_detection": self._turn_detection_config()},
},
wait_until_ready=False,
)
async def wait_for_response_boundary(self) -> None:
"""Wait until StepFun has finished or cancelled the current response."""
await self._response_done.wait()
@@ -386,12 +414,7 @@ class StepFunRealtimeService(AIService):
"voice": self._voice,
"input_audio_format": "pcm16",
"output_audio_format": "pcm16",
"turn_detection": {
"type": "server_vad",
"prefix_padding_ms": self._prefix_padding_ms,
"silence_duration_ms": self._silence_duration_ms,
"energy_awakeness_threshold": self._energy_awakeness_threshold,
},
"turn_detection": self._turn_detection_config(),
"tools": [tool.provider_schema() for tool in self._tools],
"tool_choice": "auto",
},
@@ -399,6 +422,16 @@ class StepFunRealtimeService(AIService):
wait_until_ready=False,
)
def _turn_detection_config(self) -> dict[str, Any] | None:
if not self._turn_detection_enabled:
return None
return {
"type": "server_vad",
"prefix_padding_ms": self._prefix_padding_ms,
"silence_duration_ms": self._silence_duration_ms,
"energy_awakeness_threshold": self._energy_awakeness_threshold,
}
async def update_instructions(self, instructions: str) -> None:
"""Refresh model instructions without rebuilding the realtime session."""
self._instructions = instructions
backend/services/pipecat/transports.py
+25 -1
View File
@@ -25,6 +25,7 @@ from pipecat.transports.websocket.fastapi import (
FastAPIWebsocketTransport,
FastAPIWebsocketParams,
)
from pipecat.serializers.base_serializer import FrameSerializer
from pipecat.serializers.protobuf import ProtobufFrameSerializer
from services.pipecat.call_lifecycle import FixedSpeechPlaybackMarkerFrame
@@ -103,10 +104,33 @@ def build_ws_transport(websocket: WebSocket) -> FastAPIWebsocketTransport:
"""裸 WS 输出。序列化用 protobuf(自定义客户端用同款解码);
若对接电话商,把 serializer 换成对应的 TwilioFrameSerializer 等即可。
"""
return build_serialized_ws_transport(
websocket,
serializer=ProtobufFrameSerializer(),
)
def build_serialized_ws_transport(
websocket: WebSocket,
*,
serializer: FrameSerializer,
sample_rate: int | None = None,
) -> FastAPIWebsocketTransport:
"""Build a text/binary WS transport without coupling it to one protocol."""
sample_rates = (
{
"audio_in_sample_rate": sample_rate,
"audio_out_sample_rate": sample_rate,
}
if sample_rate
else {}
)
return _WebsocketTransport(
websocket=websocket,
params=FastAPIWebsocketParams(
serializer=ProtobufFrameSerializer(),
serializer=serializer,
**_base_params(),
**sample_rates,
),
)
backend/services/pipecat/turn_config.py
+14 -1
View File
@@ -20,7 +20,10 @@ from pipecat.turns.user_stop import (
SpeechTimeoutUserTurnStopStrategy,
TurnAnalyzerUserTurnStopStrategy,
)
from pipecat.turns.user_turn_strategies import UserTurnStrategies
from pipecat.turns.user_turn_strategies import (
ExternalUserTurnStrategies,
UserTurnStrategies,
)
DEFAULT_VAD = {
@@ -126,3 +129,13 @@ class ConfigurableLLMUserAggregator(LLMUserAggregator):
enable_interruptions=enable_interruptions,
)
await self._user_turn_controller.update_strategies(strategies)
async def apply_external_turn_control(self, enabled: bool) -> None:
"""Switch between client-controlled PTT and the configured VAD."""
strategies = (
ExternalUserTurnStrategies()
if enabled
else self._params.user_turn_strategies
)
await self._user_turn_controller.update_strategies(strategies)