wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: organize tools and enable qwen realtime calls

Browse Source
This commit is contained in:
Xin Wang
2026-08-14 00:15:00 +08:00
parent eb111a3890
commit 5be21f5ccc
71 changed files with 5771 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/pipecat/qwen_audio_realtime.py
+53 -20
View File
@@ -52,6 +52,7 @@ SUPPORTED_TURN_DETECTION_MODES = frozenset({"none", "server_vad", "smart_turn"})
ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None]
SpeechStartedHandler = Callable[[], Awaitable[None]]
ResponseDoneHandler = Callable[[str, bool], Awaitable[None]]
class QwenAudioRealtimeService(AIService):
@@ -117,6 +118,7 @@ class QwenAudioRealtimeService(AIService):
self._warned_input_sample_rate = False
self._response_active = False
self._active_response_id = ""
self._audio_suppressed = False
self._assistant_turn_id: str | None = None
self._assistant_text = ""
@@ -130,12 +132,14 @@ class QwenAudioRealtimeService(AIService):
self._tools: list[RealtimeTool] = []
self._tool_session = RealtimeToolSession(
self._send_tool_event,
wait_for_response_boundary=self._response_done.wait,
suppress_response=self._cancel_active_response,
)
self._fixed_speech_completion: asyncio.Future[None] | None = None
self._suppress_response_transcript = False
self._speech_started_handler: SpeechStartedHandler | None = None
self._response_done_handler: ResponseDoneHandler | None = None
self._function_names: dict[str, str] = {}
self._responses_with_tool_calls: set[str] = set()
async def start(self, frame: StartFrame) -> None:
await super().start(frame)
@@ -201,6 +205,7 @@ class QwenAudioRealtimeService(AIService):
return
if isinstance(frame, InterruptionFrame):
self._tool_session.invalidate_pending_follow_ups()
await self._cancel_active_response()
await self.push_frame(frame, direction)
@@ -209,6 +214,7 @@ class QwenAudioRealtimeService(AIService):
"""Append text to Qwen's conversation and optionally start a response."""
if not text:
return
self._tool_session.invalidate_pending_follow_ups()
await self._send_event(
{
"type": "conversation.item.create",
@@ -224,6 +230,7 @@ class QwenAudioRealtimeService(AIService):
async def interrupt(self) -> None:
"""Cancel current model output and notify the rest of the pipeline."""
self._tool_session.invalidate_pending_follow_ups()
await self._cancel_active_response()
await self.broadcast_interruption()
@@ -268,6 +275,12 @@ class QwenAudioRealtimeService(AIService):
) -> None:
self._speech_started_handler = handler
def set_response_done_handler(
self,
handler: ResponseDoneHandler | None,
) -> None:
self._response_done_handler = handler
async def speak(self, text: str) -> None:
"""Ask Qwen to speak a fixed greeting, then remove the hidden request.
@@ -345,7 +358,7 @@ class QwenAudioRealtimeService(AIService):
"type": "session.update",
"session": {
"instructions": instructions,
"tools": [tool.provider_schema() for tool in tools],
"tools": [tool.qwen_schema() for tool in tools],
"tool_choice": "auto",
},
},
@@ -375,7 +388,7 @@ class QwenAudioRealtimeService(AIService):
"output_audio_format": "pcm",
"turn_detection": self._turn_detection_config(),
"max_history_turns": self._max_history_turns,
"tools": [tool.provider_schema() for tool in self._tools],
"tools": [tool.qwen_schema() for tool in self._tools],
"tool_choice": "auto",
}
@@ -427,12 +440,14 @@ class QwenAudioRealtimeService(AIService):
self._response_done.set()
self._pending_events.clear()
self._response_active = False
self._active_response_id = ""
self._user_transcript_pending = False
self._user_transcript_item_id = ""
self._user_transcript_timestamp = ""
self._deferred_assistant_messages.clear()
self._tool_session.clear()
await self._tool_session.shutdown()
self._function_names.clear()
self._responses_with_tool_calls.clear()
self._entry_trigger_item_id = None
self._resolve_fixed_speech()
if websocket and websocket.state is State.OPEN:
@@ -458,6 +473,8 @@ class QwenAudioRealtimeService(AIService):
self._websocket = None
self._session_ready.clear()
self._response_done.set()
await self._tool_session.shutdown()
self._responses_with_tool_calls.clear()
self._greeting_request_item_id = None
self._entry_trigger_item_id = None
self._resolve_fixed_speech()
@@ -481,6 +498,10 @@ class QwenAudioRealtimeService(AIService):
await self._send_event(payload, wait_until_ready=False)
elif event_type == "response.created":
self._response_active = True
response = event.get("response")
self._active_response_id = str(
response.get("id") if isinstance(response, dict) else ""
)
self._response_done.clear()
self._audio_suppressed = False
elif event_type == "response.audio.delta":
@@ -513,6 +534,7 @@ class QwenAudioRealtimeService(AIService):
await self._release_deferred_assistant_messages()
elif event_type == "input_audio_buffer.speech_started":
user_turn_timestamp = time_now_iso8601()
self._tool_session.invalidate_pending_follow_ups()
await self._cancel_active_response()
await self.broadcast_interruption()
if self._speech_started_handler is not None:
@@ -526,7 +548,13 @@ class QwenAudioRealtimeService(AIService):
elif event_type == "response.done":
response = event.get("response")
status = response.get("status") if isinstance(response, dict) else None
response_id = str(
(response.get("id") if isinstance(response, dict) else "")
or event.get("response_id")
or self._active_response_id
)
interrupted = status in {"cancelled", "incomplete", "interrupted", "failed"}
response_content = self._assistant_text
self._response_active = False
await self.push_frame(TTSStoppedFrame())
await self._finish_assistant_text(interrupted=interrupted)
@@ -534,16 +562,22 @@ class QwenAudioRealtimeService(AIService):
await self._delete_entry_trigger()
self._resolve_fixed_speech()
self._response_done.set()
self._tool_session.finish_response(
response_id,
cancelled=interrupted,
)
had_tool_calls = response_id in self._responses_with_tool_calls
self._responses_with_tool_calls.discard(response_id)
if self._response_done_handler is not None and not had_tool_calls:
await self._response_done_handler(response_content, interrupted)
self._active_response_id = ""
elif event_type == "response.output_item.added":
self._remember_function_call(event)
elif event_type in {
"response.function_call_arguments.done",
"response.output_item.done",
}:
self.create_task(
self._handle_function_call_event(dict(event)),
name="qwen_audio_realtime_tool_call",
)
await self._handle_function_call_event(dict(event))
elif event_type == "error":
error = event.get("error")
message = error.get("message") if isinstance(error, dict) else str(error)
@@ -591,18 +625,17 @@ class QwenAudioRealtimeService(AIService):
)
if not name:
return
try:
await self._tool_session.handle_call(
name=name,
call_id=call_id,
arguments=source.get("arguments", event.get("arguments")),
)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - background provider task
logger.exception(f"Qwen-Audio Realtime 工具调用处理失败:{exc}")
finally:
self._function_names.pop(call_id, None)
response_id = str(event.get("response_id") or self._active_response_id)
if response_id:
self._responses_with_tool_calls.add(response_id)
self._tool_session.submit_call(
response_id=response_id,
name=name,
call_id=call_id,
arguments=source.get("arguments", event.get("arguments")),
output_index=int(event.get("output_index") or 0),
)
self._function_names.pop(call_id, None)
def _remember_function_call(self, event: dict[str, Any]) -> None:
item = event.get("item")