feat: organize tools and enable qwen realtime calls

backend/services/pipecat/stepfun_realtime.py

@@ -79,12 +79,12 @@ class StepFunRealtimeService(AIService):
         self._response_done.set()
         self._pending_events: list[dict[str, Any]] = []
         self._assistant_turn_id: str | None = None
+        self._active_response_id = ""
         self._assistant_text = ""
         self._assistant_timestamp = ""
         self._tools: list[RealtimeTool] = []
         self._tool_session = RealtimeToolSession(
             self._send_tool_event,
-            wait_for_response_boundary=self._response_done.wait,
         )
         self._fixed_speech_completion: asyncio.Future[None] | None = None
         self._fixed_speech_request_item_id: str | None = None
@@ -151,6 +151,7 @@ class StepFunRealtimeService(AIService):
                     await self.send_text(text, run_immediately=frame.run_llm is not False)
             return
         if isinstance(frame, InterruptionFrame):
+            self._tool_session.invalidate_pending_follow_ups()
             await self._send_event({"type": "response.cancel"}, wait_until_ready=False)
             await self._finish_assistant_text(interrupted=True)
             self._resolve_fixed_speech()
@@ -158,6 +159,7 @@ class StepFunRealtimeService(AIService):
         await self.push_frame(frame, direction)
 
     async def send_text(self, text: str, *, run_immediately: bool = True) -> None:
+        self._tool_session.invalidate_pending_follow_ups()
         await self._send_event(
             {
                 "type": "conversation.item.create",
@@ -172,6 +174,7 @@ class StepFunRealtimeService(AIService):
             await self._send_event({"type": "response.create"})
 
     async def interrupt(self) -> None:
+        self._tool_session.invalidate_pending_follow_ups()
         await self._send_event({"type": "response.cancel"}, wait_until_ready=False)
         await self._finish_assistant_text(interrupted=True)
         self._resolve_fixed_speech()
@@ -291,7 +294,7 @@ class StepFunRealtimeService(AIService):
         self._websocket = None
         self._session_ready.clear()
         self._response_done.set()
-        self._tool_session.clear()
+        await self._tool_session.shutdown()
         self._function_names.clear()
         self._fixed_speech_request_item_id = None
         self._entry_trigger_item_id = None
@@ -328,6 +331,7 @@ class StepFunRealtimeService(AIService):
                 self._websocket = None
                 self._session_ready.clear()
             self._response_done.set()
+            await self._tool_session.shutdown()
             self._fixed_speech_request_item_id = None
             self._entry_trigger_item_id = None
             self._resolve_fixed_speech()
@@ -344,6 +348,10 @@ class StepFunRealtimeService(AIService):
             for payload in pending:
                 await self._send_event(payload, wait_until_ready=False)
         elif event_type == "response.created":
+            response = event.get("response")
+            self._active_response_id = str(
+                response.get("id") if isinstance(response, dict) else ""
+            )
             self._response_done.clear()
         elif event_type == "response.audio.delta":
             audio = event.get("delta")
@@ -369,6 +377,7 @@ class StepFunRealtimeService(AIService):
         elif event_type == "conversation.item.input_audio_transcription.completed":
             await self._send_transcript("user", str(event.get("transcript") or ""))
         elif event_type == "input_audio_buffer.speech_started":
+            self._tool_session.invalidate_pending_follow_ups()
             await self._send_event({"type": "response.cancel"}, wait_until_ready=False)
             await self.broadcast_interruption()
             self._resolve_fixed_speech()
@@ -387,16 +396,23 @@ class StepFunRealtimeService(AIService):
             await self._delete_entry_trigger()
             self._resolve_fixed_speech()
             self._response_done.set()
+            response_id = str(
+                (response.get("id") if isinstance(response, dict) else "")
+                or event.get("response_id")
+                or self._active_response_id
+            )
+            self._tool_session.finish_response(
+                response_id,
+                cancelled=interrupted,
+            )
+            self._active_response_id = ""
         elif event_type == "response.output_item.added":
             self._remember_function_call(event)
         elif event_type in {
             "response.function_call_arguments.done",
             "response.output_item.done",
         }:
-            self.create_task(
-                self._handle_function_call_event(dict(event)),
-                name="stepfun_realtime_tool_call",
-            )
+            await self._handle_function_call_event(dict(event))
         elif event_type == "error":
             error = event.get("error")
             message = error.get("message") if isinstance(error, dict) else str(error)
@@ -415,7 +431,7 @@ class StepFunRealtimeService(AIService):
                     "input_audio_format": "pcm16",
                     "output_audio_format": "pcm16",
                     "turn_detection": self._turn_detection_config(),
-                    "tools": [tool.provider_schema() for tool in self._tools],
+                    "tools": [tool.openai_schema() for tool in self._tools],
                     "tool_choice": "auto",
                 },
             },
@@ -458,7 +474,7 @@ class StepFunRealtimeService(AIService):
                     "type": "session.update",
                     "session": {
                         "instructions": instructions,
-                        "tools": [tool.provider_schema() for tool in tools],
+                        "tools": [tool.openai_schema() for tool in tools],
                         "tool_choice": "auto",
                     },
                 },
@@ -493,18 +509,14 @@ class StepFunRealtimeService(AIService):
         )
         if not name:
             return
-        try:
-            await self._tool_session.handle_call(
-                name=name,
-                call_id=call_id,
-                arguments=source.get("arguments", event.get("arguments")),
-            )
-        except asyncio.CancelledError:
-            raise
-        except Exception as exc:  # noqa: BLE001 - background provider task
-            logger.exception(f"StepFun Realtime 工具调用处理失败:{exc}")
-        finally:
-            self._function_names.pop(call_id, None)
+        self._tool_session.submit_call(
+            response_id=str(event.get("response_id") or self._active_response_id),
+            name=name,
+            call_id=call_id,
+            arguments=source.get("arguments", event.get("arguments")),
+            output_index=int(event.get("output_index") or 0),
+        )
+        self._function_names.pop(call_id, None)
 
     def _remember_function_call(self, event: dict[str, Any]) -> None:
         item = event.get("item")