wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add configurable vision focus prompts

Browse Source
This commit is contained in:
Xin Wang
2026-08-12 17:29:30 +08:00
parent 7560f9a525
commit 09b07751b6
26 changed files with 477 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/components/assistant-editor/prompt-editor.tsx
+6
View File
@@ -648,12 +648,18 @@ export function PromptEditor({
hint="开启后，助手会获得读取当前视频画面的工具。选择「模型自己」时，大语言模型必须支持图片输入。"
enabled={form.visionEnabled}
modelResourceId={form.visionModelResourceId}
focusPrompt={form.visionFocusPrompt}
sourcePrompt={form.prompt}
generatorModelResourceId={form.model}
mainModelResourceId={form.model}
modelOptions={visionModelOptions}
onEnabledChange={handlePromptVisionEnabledChange}
onModelResourceIdChange={(value) =>
updateForm("visionModelResourceId", value)
}
onFocusPromptChange={(value) =>
updateForm("visionFocusPrompt", value)
}
/>
)}
frontend/src/components/assistant-editor/types.ts
+1
View File
@@ -26,5 +26,6 @@ export type AssistantForm = {
startup: StartupConfig;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
toolIds: string[];
};
frontend/src/components/editor/vision-config-section.tsx
+84 -1
View File
@@ -1,12 +1,16 @@
"use client";
import { Camera } from "lucide-react";
import { useState } from "react";
import { Camera, Loader2, Sparkles } from "lucide-react";
import {
ResourceSelectField,
ToggleRow,
} from "@/components/assistant-editor/editor-controls";
import { SectionCard } from "@/components/editor/section-card";
import { Button } from "@/components/ui/button";
import { Textarea } from "@/components/ui/textarea";
import { assistantsApi } from "@/lib/api";
type VisionModelOption = {
value: string;
@@ -18,22 +22,32 @@ export function VisionConfigSection({
hint,
enabled,
modelResourceId,
focusPrompt,
sourcePrompt,
generatorModelResourceId,
mainModelResourceId,
mainModelSupportsImageInput,
modelOptions,
onEnabledChange,
onModelResourceIdChange,
onFocusPromptChange,
}: {
description: string;
hint: string;
enabled: boolean;
modelResourceId: string;
focusPrompt: string;
sourcePrompt: string;
generatorModelResourceId: string;
mainModelResourceId: string;
mainModelSupportsImageInput?: boolean;
modelOptions: VisionModelOption[];
onEnabledChange: (enabled: boolean) => void;
onModelResourceIdChange: (modelResourceId: string) => void;
onFocusPromptChange: (focusPrompt: string) => void;
}) {
const [generating, setGenerating] = useState(false);
const [generationError, setGenerationError] = useState("");
const mainModelSupportsVision =
mainModelSupportsImageInput ??
modelOptions.some((option) => option.value === mainModelResourceId);
@@ -41,6 +55,25 @@ export function VisionConfigSection({
(option) => option.value !== mainModelResourceId,
);
async function generateFocusPrompt() {
if (!sourcePrompt.trim() || !generatorModelResourceId || generating) return;
setGenerating(true);
setGenerationError("");
try {
const result = await assistantsApi.generateVisionFocus({
sourcePrompt,
modelResourceId: generatorModelResourceId,
});
onFocusPromptChange(result.focusPrompt);
} catch (error) {
setGenerationError(
error instanceof Error ? error.message : "生成视觉关注目标失败",
);
} finally {
setGenerating(false);
}
}
return (
<SectionCard
icon={<Camera size={15} />}
@@ -67,6 +100,56 @@ export function VisionConfigSection({
当前模型未标记支持图片输入，请选择独立视觉模型。
</p>
)}
<div className="space-y-1.5">
<div className="flex items-center justify-between gap-3">
<label className="text-sm font-medium text-foreground">
视觉关注目标
</label>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 gap-1.5 rounded-full px-2.5 text-xs"
disabled={
generating || !sourcePrompt.trim() || !generatorModelResourceId
}
onClick={() => void generateFocusPrompt()}
title={
!sourcePrompt.trim()
? "请先填写助手提示词"
: !generatorModelResourceId
? "请先选择大语言模型"
: "根据当前助手提示词生成可编辑草稿"
}
>
{generating ? (
<Loader2 size={12} className="animate-spin" />
) : (
<Sparkles size={12} />
)}
{generating ? "生成中" : "根据提示词生成"}
</Button>
</div>
<Textarea
value={focusPrompt}
maxLength={4000}
rows={5}
onChange={(event) => {
onFocusPromptChange(event.target.value);
if (generationError) setGenerationError("");
}}
placeholder="例如：优先识别车辆数量、碰撞位置、明显损伤和人员状态；只报告画面中可直接确认的信息。"
className="field-sizing-fixed min-h-28 resize-y border-hairline-strong bg-background text-sm leading-6 text-foreground placeholder:text-muted-soft"
/>
<p className="text-[11px] leading-5 text-muted-soft">
可选。系统仍保留通用视觉安全规则；这里用于让每次看图稳定聚焦业务目标。
</p>
{generationError && (
<p className="text-xs leading-5 text-destructive">
{generationError}
</p>
)}
</div>
</>
)}
</SectionCard>
frontend/src/components/pages/AssistantPage.tsx
+5
View File
@@ -180,6 +180,7 @@ function blankPromptForm(name: string): AssistantForm {
},
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
toolIds: [],
};
}
@@ -435,6 +436,7 @@ export function AssistantPage(props: AssistantPageProps) {
},
visionEnabled: a.visionEnabled,
visionModelResourceId: a.visionModelResourceId ?? "",
visionFocusPrompt: a.visionFocusPrompt ?? "",
toolIds: a.toolIds ?? [],
};
setForm(next);
@@ -485,6 +487,7 @@ export function AssistantPage(props: AssistantPageProps) {
},
visionEnabled: false,
visionModelResourceId: null,
visionFocusPrompt: "",
modelResourceIds: {},
knowledgeBaseId: null,
knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
@@ -549,6 +552,7 @@ export function AssistantPage(props: AssistantPageProps) {
startup: form.startup,
visionEnabled: form.visionEnabled,
visionModelResourceId: form.visionModelResourceId || null,
visionFocusPrompt: form.visionFocusPrompt,
modelResourceIds: {
...(form.model ? { LLM: form.model } : {}),
...(form.asr ? { ASR: form.asr } : {}),
@@ -676,6 +680,7 @@ export function AssistantPage(props: AssistantPageProps) {
graph.settings?.visionModelResourceId ??
assistant.visionModelResourceId ??
"",
visionFocusPrompt: graph.settings?.visionFocusPrompt ?? "",
toolIds: graph.settings?.toolIds ?? [],
knowledgeBaseId:
graph.settings?.knowledgeBaseId || assistant.knowledgeBaseId || "",
frontend/src/components/pages/CreateAssistantPage.tsx
+1
View File
@@ -108,6 +108,7 @@ function baseUpsert(over: Partial<AssistantUpsert>): AssistantUpsert {
},
visionEnabled: false,
visionModelResourceId: null,
visionFocusPrompt: "",
modelResourceIds: {},
knowledgeBaseId: null,
knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
frontend/src/components/pages/TemplateLibraryPage.tsx
+1
View File
@@ -96,6 +96,7 @@ function baseUpsertFromTemplate(
},
visionEnabled: false,
visionModelResourceId: null,
visionFocusPrompt: "",
modelResourceIds: {},
knowledgeBaseId: null,
knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
frontend/src/components/workflow/WorkflowCanvas.tsx
+1
View File
@@ -129,6 +129,7 @@ function fromFlow(nodes: Node[], edges: Edge[]): WorkflowGraph {
defaultTtsResourceId: "",
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
toolIds: [],
knowledgeBaseId: "",
knowledgeMode: "automatic",
frontend/src/components/workflow/graph-settings.ts
+2
View File
@@ -15,6 +15,7 @@ export function settingsFromWorkflowGraph(
tts: graph.settings.defaultTtsResourceId,
visionEnabled: graph.settings.visionEnabled,
visionModelResourceId: graph.settings.visionModelResourceId,
visionFocusPrompt: graph.settings.visionFocusPrompt,
toolIds: graph.settings.toolIds,
knowledgeBaseId: graph.settings.knowledgeBaseId,
knowledgeRetrievalConfig: {
@@ -44,6 +45,7 @@ export function workflowGraphWithSettings(
defaultTtsResourceId: settings.tts ?? "",
visionEnabled: settings.visionEnabled,
visionModelResourceId: settings.visionModelResourceId,
visionFocusPrompt: settings.visionFocusPrompt,
toolIds: settings.toolIds,
knowledgeBaseId: settings.knowledgeBaseId,
knowledgeMode: settings.knowledgeRetrievalConfig.mode,
frontend/src/components/workflow/panels/AgentNodePanel.tsx
+12
View File
@@ -78,6 +78,8 @@ export function AgentNodePanel({
visionModelResourceId:
(draft.visionModelResourceId as string) ||
workflowSettings.visionModelResourceId,
visionFocusPrompt:
draft.visionFocusPrompt ?? workflowSettings.visionFocusPrompt,
...(!isRealtime
? {
llmResourceId:
@@ -306,6 +308,13 @@ export function AgentNodePanel({
}
enabled={Boolean(draft.visionEnabled)}
modelResourceId={draft.visionModelResourceId ?? ""}
focusPrompt={draft.visionFocusPrompt ?? ""}
sourcePrompt={draft.prompt ?? ""}
generatorModelResourceId={
isRealtime
? draft.visionModelResourceId ?? ""
: (draft.llmResourceId as string) || ""
}
mainModelResourceId={
isRealtime
? workflowSettings.realtime ?? ""
@@ -330,6 +339,9 @@ export function AgentNodePanel({
onModelResourceIdChange={(visionModelResourceId) =>
set("visionModelResourceId", visionModelResourceId)
}
onFocusPromptChange={(visionFocusPrompt) =>
set("visionFocusPrompt", visionFocusPrompt)
}
/>
<SectionCard
frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+10
View File
@@ -184,6 +184,13 @@ export function GlobalSettingsPanel({
}
enabled={settings.visionEnabled}
modelResourceId={settings.visionModelResourceId}
focusPrompt={settings.visionFocusPrompt}
sourcePrompt={settings.globalPrompt}
generatorModelResourceId={
settings.runtimeMode === "realtime"
? settings.visionModelResourceId
: settings.llm ?? ""
}
mainModelResourceId={
settings.runtimeMode === "realtime"
? settings.realtime ?? ""
@@ -209,6 +216,9 @@ export function GlobalSettingsPanel({
onModelResourceIdChange={(visionModelResourceId) =>
onSettingsChange({ ...settings, visionModelResourceId })
}
onFocusPromptChange={(visionFocusPrompt) =>
onSettingsChange({ ...settings, visionFocusPrompt })
}
/>
<SectionCard
frontend/src/components/workflow/specs.ts
+3
View File
@@ -54,6 +54,7 @@ export type WorkflowNodeData = {
ttsResourceId?: string;
visionEnabled?: boolean;
visionModelResourceId?: string;
visionFocusPrompt?: string;
enableInterrupt?: boolean;
turnConfig?: TurnConfig;
toolId?: string;
@@ -213,6 +214,7 @@ export type WorkflowGraph = {
defaultTtsResourceId: string;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
toolIds: string[];
knowledgeBaseId: string;
knowledgeMode: "automatic" | "on_demand";
@@ -250,6 +252,7 @@ export function defaultGraph(): WorkflowGraph {
defaultTtsResourceId: "",
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
toolIds: [],
knowledgeBaseId: "",
knowledgeMode: "automatic",
frontend/src/components/workflow/types.ts
+1
View File
@@ -18,6 +18,7 @@ export type WorkflowSettings = {
tts?: string;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
toolIds: string[];
knowledgeBaseId: string;
knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
frontend/src/lib/api.ts
+9
View File
@@ -327,6 +327,7 @@ export type Assistant = {
startup: StartupConfig;
visionEnabled: boolean;
visionModelResourceId: string | null;
visionFocusPrompt: string;
modelResourceIds: Partial<Record<ModelType, string>>;
knowledgeBaseId: string | null;
knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
@@ -371,6 +372,14 @@ export const assistantsApi = {
// 服务端整行复制(含真 key,密钥不经浏览器)
duplicate: (id: string) =>
request<Assistant>(`/api/assistants/${id}/duplicate`, { method: "POST" }),
generateVisionFocus: (body: {
sourcePrompt: string;
modelResourceId: string;
}) =>
request<{ focusPrompt: string }>("/api/assistants/vision-focus/generate", {
method: "POST",
body: JSON.stringify(body),
}),
remove: (id: string) =>
request<{ ok: boolean }>(`/api/assistants/${id}`, { method: "DELETE" }),
};