From abfcd4cd19f0bec62690dc5ac89a06ec9fd7dc2b Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 14 Aug 2026 18:28:52 +0800 Subject: [PATCH] fix: support photo client tool in debug preview --- backend/routes/voice_webrtc.py | 62 +++++++++++++++- backend/tests/test_voice_webrtc_lifecycle.py | 73 ++++++++++++++++++- .../assistant-editor/debug-preview.tsx | 14 ++-- frontend/src/hooks/use-voice-preview.ts | 9 ++- 4 files changed, 147 insertions(+), 11 deletions(-) diff --git a/backend/routes/voice_webrtc.py b/backend/routes/voice_webrtc.py index a21f9c0..37fd55b 100644 --- a/backend/routes/voice_webrtc.py +++ b/backend/routes/voice_webrtc.py @@ -13,7 +13,7 @@ import json from collections.abc import Coroutine from typing import Any -from fastapi import APIRouter, Body, Depends, Request, WebSocket +from fastapi import APIRouter, Body, Depends, HTTPException, Request, WebSocket from loguru import logger from models import AssistantConfig, SignalingOffer from services.auth import require_admin, require_admin_websocket @@ -35,6 +35,7 @@ _active_connections = realtime_lifecycle.active_connections _pipeline_tasks = realtime_lifecycle.pipeline_tasks _connection_tasks = realtime_lifecycle.connection_tasks PIPELINE_CLOSE_GRACE_SECONDS = 10.0 +MAX_CLIENT_TOOL_NAME_LENGTH = 64 def _start_pipeline_task( @@ -69,6 +70,53 @@ async def shutdown_active_sessions() -> None: ) +def _decode_client_tool_names(encoded: str | None) -> list[str] | None: + """Decode client capabilities carried outside the WebRTC SDP body. + + Pipecat's browser transport replaces the body of a ``Request`` endpoint + with its SDP offer and drops ``APIRequest.requestData``. Request headers are + preserved, so they are the authoritative metadata channel for this path. + ``None`` means the header was absent and callers should use the legacy body. + """ + + if encoded is None: + return None + try: + padding = "=" * (-len(encoded) % 4) + value = json.loads( + base64.urlsafe_b64decode(encoded + padding).decode("utf-8") + ) + except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HTTPException( + status_code=400, + detail="客户端工具能力请求头格式无效", + ) from exc + if not isinstance(value, list): + raise HTTPException( + status_code=400, + detail="客户端工具能力必须是函数名列表", + ) + + names: list[str] = [] + seen: set[str] = set() + for item in value: + if not isinstance(item, str): + raise HTTPException( + status_code=400, + detail="客户端工具函数名必须是字符串", + ) + name = item.strip() + if not name or len(name) > MAX_CLIENT_TOOL_NAME_LENGTH: + raise HTTPException( + status_code=400, + detail="客户端工具函数名为空或过长", + ) + if name not in seen: + names.append(name) + seen.add(name) + return names + + @router.get("/api/webrtc/ice-servers", dependencies=[Depends(require_admin)]) async def ice_servers(): """Browser fetches STUN/TURN config (with ephemeral TURN creds when configured).""" @@ -86,12 +134,22 @@ async def http_offer(request: Request, payload: dict = Body(...)): header_variables = json.loads( base64.urlsafe_b64decode(encoded_variables + padding).decode("utf-8") ) + header_client_tools = _decode_client_tool_names( + request.headers.get("x-pipecat-client-tools") + ) + client_tool_names = ( + header_client_tools + if header_client_tools is not None + else request_data.get("client_tool_names") or [] + ) + logger.debug(f"WebRTC 客户端工具上报: {client_tool_names or ''}") offer_payload = { **payload, "assistant_id": request.headers.get("x-pipecat-assistant-id"), "dynamic_variables": header_variables, - "client_tool_names": request_data.get("client_tool_names") or [], **request_data, + # Keep header metadata authoritative when both paths are present. + "client_tool_names": client_tool_names, } answer = await _handle_offer_payload(offer_payload, _http_peers) return answer diff --git a/backend/tests/test_voice_webrtc_lifecycle.py b/backend/tests/test_voice_webrtc_lifecycle.py index ca76a67..3fc01c3 100644 --- a/backend/tests/test_voice_webrtc_lifecycle.py +++ b/backend/tests/test_voice_webrtc_lifecycle.py @@ -1,16 +1,87 @@ from __future__ import annotations import asyncio +import base64 +import json import unittest -from unittest.mock import patch +from unittest.mock import AsyncMock, patch +from fastapi import HTTPException from routes import voice_webrtc +from starlette.requests import Request class FakeConnection: pc_id = "pc_test" +def encoded_header_json(value: object) -> str: + payload = json.dumps(value).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def offer_request(*, client_tools: object | None = None) -> Request: + headers = [] + if client_tools is not None: + headers.append( + ( + b"x-pipecat-client-tools", + encoded_header_json(client_tools).encode("ascii"), + ) + ) + return Request({"type": "http", "method": "POST", "headers": headers}) + + +class WebRTCOfferMetadataTest(unittest.IsolatedAsyncioTestCase): + async def test_client_tool_header_survives_request_endpoint_body_replacement(self): + request = offer_request( + client_tools=["set_photo_button_visible", "show_message"] + ) + payload = {"sdp": "offer", "type": "offer"} + + with patch.object( + voice_webrtc, + "_handle_offer_payload", + new=AsyncMock(return_value={"type": "answer"}), + ) as handle_offer: + await voice_webrtc.http_offer(request, payload) + + forwarded = handle_offer.await_args.args[0] + self.assertEqual( + forwarded["client_tool_names"], + ["set_photo_button_visible", "show_message"], + ) + + async def test_client_tool_request_data_remains_a_compatibility_fallback(self): + request = offer_request() + payload = { + "sdp": "offer", + "type": "offer", + "requestData": {"client_tool_names": ["show_message"]}, + } + + with patch.object( + voice_webrtc, + "_handle_offer_payload", + new=AsyncMock(return_value={"type": "answer"}), + ) as handle_offer: + await voice_webrtc.http_offer(request, payload) + + forwarded = handle_offer.await_args.args[0] + self.assertEqual(forwarded["client_tool_names"], ["show_message"]) + + async def test_invalid_client_tool_header_is_rejected(self): + request = offer_request(client_tools={"name": "show_message"}) + + with self.assertRaises(HTTPException) as raised: + await voice_webrtc.http_offer( + request, + {"sdp": "offer", "type": "offer"}, + ) + + self.assertEqual(raised.exception.status_code, 400) + + class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase): async def asyncTearDown(self): tasks = list(voice_webrtc._pipeline_tasks) diff --git a/frontend/src/components/assistant-editor/debug-preview.tsx b/frontend/src/components/assistant-editor/debug-preview.tsx index a58168f..ca37939 100644 --- a/frontend/src/components/assistant-editor/debug-preview.tsx +++ b/frontend/src/components/assistant-editor/debug-preview.tsx @@ -5,6 +5,7 @@ import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react import { AudioLines, Braces, + Camera, Check, Copy, ImageIcon, @@ -13,7 +14,6 @@ import { Mic, Orbit, PhoneOff, - ScanLine, Send, Smartphone, Sparkles, @@ -1674,7 +1674,10 @@ function DebugPhotoCaptureButton({ onCapture: () => Promise; }) { return ( -
+
+ + 拍照按钮已开启 + {error && (
{error} @@ -1682,18 +1685,19 @@ function DebugPhotoCaptureButton({ )}
); diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts index b58982e..70b4db9 100644 --- a/frontend/src/hooks/use-voice-preview.ts +++ b/frontend/src/hooks/use-voice-preview.ts @@ -826,6 +826,10 @@ export function useVoicePreview( setVideoStream(localVideo ? new MediaStream([localVideo]) : null); void refreshDevices(); + // SmallWebRTCTransport discards APIRequest.requestData when endpoint is a + // Request object. Keep capabilities in a header as the reliable path; + // requestData remains below for compatibility with other transports. + const clientToolNames = Array.from(clientToolHandlersRef.current.keys()); const request = new Request(`${API_BASE}/api/webrtc/offer`, { method: "POST", headers: { @@ -834,6 +838,7 @@ export function useVoicePreview( "X-Pipecat-Dynamic-Variables": encodeHeaderJson( options.dynamicVariables ?? {}, ), + "X-Pipecat-Client-Tools": encodeHeaderJson(clientToolNames), }, credentials: "include", }); @@ -844,9 +849,7 @@ export function useVoicePreview( assistant_id: assistantId, vision_enabled: Boolean(options.visionEnabled), dynamic_variables: options.dynamicVariables ?? {}, - client_tool_names: Array.from( - clientToolHandlersRef.current.keys(), - ), + client_tool_names: clientToolNames, }, }, });