feat: add configurable client tools and photo input

This commit is contained in:
Xin Wang
2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions
@@ -12,6 +12,7 @@ import {
Mic,
Orbit,
PhoneOff,
ScanLine,
Send,
Smartphone,
Sparkles,
@@ -43,6 +44,7 @@ import { Textarea } from "@/components/ui/textarea";
import { WaveVisualizer } from "@/components/ui/wave-visualizer";
import { WaveformTimelinePanel } from "@/components/ui/waveform-timeline";
import { useCameraPreview, type CameraPreview } from "@/hooks/use-camera-preview";
import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
import {
useVoicePreview,
type ChatMessage,
@@ -626,6 +628,7 @@ function DebugVoicePanel({
dynamicVariables: Record<string, string | number | boolean>;
dynamicVariablesError: string;
}) {
const photoCapture = usePhotoCaptureTool(preview);
const {
status,
error,
@@ -689,6 +692,10 @@ function DebugVoicePanel({
recording={recording}
camera={camera}
videoStream={preview.videoStream}
photoCaptureVisible={photoCapture.visible}
photoCapturing={photoCapture.capturing}
photoError={photoCapture.error}
onPhotoCapture={photoCapture.capture}
/>
) : view === "video" ? (
showIdleHub ? (
@@ -966,6 +973,10 @@ function DebugVisionWorkspace({
recording,
camera,
videoStream,
photoCaptureVisible,
photoCapturing,
photoError,
onPhotoCapture,
}: {
view: DebugView;
onViewChange: (view: DebugView) => void;
@@ -973,6 +984,10 @@ function DebugVisionWorkspace({
recording: boolean;
camera: CameraPreview;
videoStream: MediaStream | null;
photoCaptureVisible: boolean;
photoCapturing: boolean;
photoError: string | null;
onPhotoCapture: () => Promise<void>;
}) {
const containerRef = useRef<HTMLDivElement>(null);
const [position, setPosition] = useState({ x: 16, y: 16 });
@@ -1038,6 +1053,14 @@ function DebugVisionWorkspace({
</span>
</span>
</button>
{photoCaptureVisible && (
<DebugPhotoCaptureButton
capturing={photoCapturing}
enabled={Boolean(videoStream)}
error={photoError}
onCapture={onPhotoCapture}
/>
)}
</div>
);
}
@@ -1074,6 +1097,51 @@ function DebugVisionWorkspace({
>
<DebugVideoPanel camera={camera} streamOverride={videoStream} compact />
</button>
{photoCaptureVisible && (
<DebugPhotoCaptureButton
capturing={photoCapturing}
enabled={Boolean(videoStream)}
error={photoError}
onCapture={onPhotoCapture}
/>
)}
</div>
);
}
function DebugPhotoCaptureButton({
capturing,
enabled,
error,
onCapture,
}: {
capturing: boolean;
enabled: boolean;
error: string | null;
onCapture: () => Promise<void>;
}) {
return (
<div className="absolute bottom-4 left-1/2 z-20 flex -translate-x-1/2 flex-col items-center gap-2">
{error && (
<div className="max-w-xs rounded-full border border-destructive/20 bg-background/90 px-3 py-1 text-xs text-destructive shadow-sm backdrop-blur">
{error}
</div>
)}
<Button
type="button"
size="icon"
onClick={() => void onCapture()}
disabled={capturing || !enabled}
aria-label={capturing ? "正在提交照片" : "拍照并发送"}
title={capturing ? "正在提交照片" : "拍照并发送"}
className="size-12 rounded-full shadow-lg"
>
{capturing ? (
<Loader2 className="size-5 animate-spin" />
) : (
<ScanLine className="size-5" />
)}
</Button>
</div>
);
}
@@ -1250,4 +1318,3 @@ function DebugTranscriptPanel({
</div>
);
}
@@ -546,7 +546,9 @@ export function ToolPicker({
? "End Call"
: tool.type === "mcp"
? "MCP"
: "HTTP"}
: tool.type === "client"
? "Client"
: "HTTP"}
</Badge>
</div>
<div className="mt-0.5 truncate font-mono text-xs text-muted-foreground">