feat: show client tools in debug preview

frontend/src/hooks/use-photo-capture-tool.ts

@@ -5,32 +5,47 @@ import { useCallback, useEffect, useState } from "react";
 import type { VoicePreview } from "@/hooks/use-voice-preview";
 
 const PHOTO_BUTTON_TOOL = "set_photo_button_visible";
+const PHOTO_BUTTON_DEFINITION = {
+  name: PHOTO_BUTTON_TOOL,
+  label: "控制拍照按钮",
+  description: "显示或隐藏客户端的拍照按钮。",
+  parameters: [
+    {
+      name: "visible",
+      type: "boolean",
+      required: true,
+      description: "是否显示拍照按钮。",
+    },
+  ],
+} as const;
 
-export function usePhotoCaptureTool(preview: VoicePreview) {
+export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
   const { registerClientTool, sendUserInput, status } = preview;
   const [visible, setVisible] = useState(false);
   const [capturing, setCapturing] = useState(false);
   const [error, setError] = useState<string | null>(null);
 
-  useEffect(
-    () =>
-      registerClientTool(PHOTO_BUTTON_TOOL, ({ visible: nextValue }) => {
+  useEffect(() => {
+    if (!enabled) return;
+    return registerClientTool(
+      PHOTO_BUTTON_DEFINITION,
+      ({ visible: nextValue }) => {
         if (typeof nextValue !== "boolean") {
           throw new Error("visible 参数必须是布尔值");
         }
         setVisible(nextValue);
         setError(null);
         return { visible: nextValue };
-      }),
-    [registerClientTool],
-  );
+      },
+    );
+  }, [enabled, registerClientTool]);
 
   useEffect(() => {
-    if (status === "connected") return;
+    if (enabled && status === "connected") return;
     // eslint-disable-next-line react-hooks/set-state-in-effect
     setVisible(false);
     setCapturing(false);
-  }, [status]);
+  }, [enabled, status]);
 
   const capture = useCallback(async () => {
     if (capturing || status !== "connected") return;

frontend/src/hooks/use-voice-preview.ts

@@ -71,6 +71,23 @@ export type SessionUpdateResult = {
 export type ClientToolHandler = (
   argumentsValue: Record<string, unknown>,
 ) => unknown | Promise<unknown>;
+export type ClientToolDefinition = {
+  name: string;
+  label: string;
+  description: string;
+  parameters: readonly ClientToolParameter[];
+};
+export type ClientToolParameter = {
+  name: string;
+  type: string;
+  required: boolean;
+  description: string;
+};
+
+type RegisteredClientTool = {
+  definition: ClientToolDefinition;
+  handler: ClientToolHandler;
+};
 
 type PendingUserInput = {
   resolve: (result: UserInputResult) => void;
@@ -276,6 +293,7 @@ export function useVoicePreview(
   const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
   const [selectedDeviceId, setSelectedDeviceId] = useState("");
   const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState("");
+  const [clientTools, setClientTools] = useState<ClientToolDefinition[]>([]);
 
   const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
   const cleanupPromiseRef = useRef<Promise<void>>(Promise.resolve());
@@ -290,7 +308,9 @@ export function useVoicePreview(
   const pendingSessionUpdatesRef = useRef(
     new Map<string, PendingSessionUpdate>(),
   );
-  const clientToolHandlersRef = useRef(new Map<string, ClientToolHandler>());
+  const clientToolHandlersRef = useRef(
+    new Map<string, RegisteredClientTool>(),
+  );
   const endedByServerRef = useRef(false);
   const selectedDeviceIdRef = useRef("");
   const selectedOutputDeviceIdRef = useRef("");
@@ -396,10 +416,10 @@ export function useVoicePreview(
     const waitForResponse = msg.wait_for_response !== false;
     if (!toolCallId || !functionName) return;
 
-    const handler = clientToolHandlersRef.current.get(functionName);
+    const registeredTool = clientToolHandlersRef.current.get(functionName);
     const transport = transportRef.current;
     if (!transport || transport.state !== "connected") return;
-    if (!handler) {
+    if (!registeredTool) {
       if (waitForResponse) {
         transport.sendAppMessage({
           type: "client-tool-result",
@@ -418,7 +438,7 @@ export function useVoicePreview(
         !Array.isArray(msg.arguments)
           ? (msg.arguments as Record<string, unknown>)
           : {};
-      const data = await handler(argumentsValue);
+      const data = await registeredTool.handler(argumentsValue);
       if (transportRef.current !== transport) return;
       if (!waitForResponse) return;
       transport.sendAppMessage({
@@ -889,11 +909,27 @@ export function useVoicePreview(
   );
 
   const registerClientTool = useCallback(
-    (functionName: string, handler: ClientToolHandler): (() => void) => {
-      clientToolHandlersRef.current.set(functionName, handler);
+    (
+      definition: ClientToolDefinition,
+      handler: ClientToolHandler,
+    ): (() => void) => {
+      const registeredTool = { definition, handler };
+      clientToolHandlersRef.current.set(definition.name, registeredTool);
+      setClientTools(
+        [...clientToolHandlersRef.current.values()].map(
+          (item) => item.definition,
+        ),
+      );
       return () => {
-        if (clientToolHandlersRef.current.get(functionName) === handler) {
-          clientToolHandlersRef.current.delete(functionName);
+        if (
+          clientToolHandlersRef.current.get(definition.name) === registeredTool
+        ) {
+          clientToolHandlersRef.current.delete(definition.name);
+          setClientTools(
+            [...clientToolHandlersRef.current.values()].map(
+              (item) => item.definition,
+            ),
+          );
         }
       };
     },
@@ -911,6 +947,7 @@ export function useVoicePreview(
     remoteStream,
     messages,
     sessionVariables,
+    clientTools,
     callEnded,
     networkQuality,
     audioInputs,