fix: support vision prompt modes

2026-08-12 19:13:04 +08:00
parent 09b07751b6
commit 94a02d7e2b
29 changed files with 676 additions and 439 deletions
@@ -648,17 +648,19 @@ export function PromptEditor({
                    hint="开启后，助手会获得读取当前视频画面的工具。选择「模型自己」时，大语言模型必须支持图片输入。"
                    enabled={form.visionEnabled}
                    modelResourceId={form.visionModelResourceId}
-                    focusPrompt={form.visionFocusPrompt}
-                    sourcePrompt={form.prompt}
-                    generatorModelResourceId={form.model}
+                    promptMode={form.visionPromptMode}
+                    prompt={form.visionPrompt}
                    mainModelResourceId={form.model}
                    modelOptions={visionModelOptions}
                    onEnabledChange={handlePromptVisionEnabledChange}
                    onModelResourceIdChange={(value) =>
                      updateForm("visionModelResourceId", value)
                    }
-                    onFocusPromptChange={(value) =>
-                      updateForm("visionFocusPrompt", value)
+                    onPromptModeChange={(value) =>
+                      updateForm("visionPromptMode", value)
+                    }
+                    onPromptChange={(value) =>
+                      updateForm("visionPrompt", value)
                    }
                  />
                )}
@@ -4,6 +4,7 @@ import type {
  KnowledgeRetrievalConfig,
  StartupConfig,
  TurnConfig,
+  VisionPromptMode,
 } from "@/lib/api";

 export type RuntimeMode = "pipeline" | "realtime";
@@ -26,6 +27,7 @@ export type AssistantForm = {
  startup: StartupConfig;
  visionEnabled: boolean;
  visionModelResourceId: string;
-  visionFocusPrompt: string;
+  visionPromptMode: VisionPromptMode;
+  visionPrompt: string;
  toolIds: string[];
 };
@@ -1,53 +1,71 @@
 "use client";

-import { useState } from "react";
-import { Camera, Loader2, Sparkles } from "lucide-react";
+import { Camera, Check } from "lucide-react";

 import {
  ResourceSelectField,
  ToggleRow,
 } from "@/components/assistant-editor/editor-controls";
 import { SectionCard } from "@/components/editor/section-card";
-import { Button } from "@/components/ui/button";
 import { Textarea } from "@/components/ui/textarea";
-import { assistantsApi } from "@/lib/api";
+import type { VisionPromptMode } from "@/lib/api";

 type VisionModelOption = {
  value: string;
  label: string;
 };

+const promptModes: Array<{
+  value: VisionPromptMode;
+  title: string;
+  description: string;
+}> = [
+  {
+    value: "fixed",
+    title: "固定提示词",
+    description: "每次获取画面都发送同一个问题。",
+  },
+  {
+    value: "runtime",
+    title: "运行时自动生成",
+    description: "模型结合当前对话，临时生成看图问题。",
+  },
+  {
+    value: "guided",
+    title: "固定提示词引导",
+    description: "模型动态生成问题，同时遵守固定引导。",
+  },
+];
+
 export function VisionConfigSection({
  description,
  hint,
  enabled,
  modelResourceId,
-  focusPrompt,
-  sourcePrompt,
-  generatorModelResourceId,
+  promptMode,
+  prompt,
  mainModelResourceId,
  mainModelSupportsImageInput,
  modelOptions,
  onEnabledChange,
  onModelResourceIdChange,
-  onFocusPromptChange,
+  onPromptModeChange,
+  onPromptChange,
 }: {
  description: string;
  hint: string;
  enabled: boolean;
  modelResourceId: string;
-  focusPrompt: string;
-  sourcePrompt: string;
-  generatorModelResourceId: string;
+  promptMode: VisionPromptMode;
+  prompt: string;
  mainModelResourceId: string;
  mainModelSupportsImageInput?: boolean;
  modelOptions: VisionModelOption[];
  onEnabledChange: (enabled: boolean) => void;
  onModelResourceIdChange: (modelResourceId: string) => void;
-  onFocusPromptChange: (focusPrompt: string) => void;
+  onPromptModeChange: (promptMode: VisionPromptMode) => void;
+  onPromptChange: (prompt: string) => void;
 }) {
-  const [generating, setGenerating] = useState(false);
-  const [generationError, setGenerationError] = useState("");
  const mainModelSupportsVision =
    mainModelSupportsImageInput ??
    modelOptions.some((option) => option.value === mainModelResourceId);
@@ -55,25 +73,6 @@ export function VisionConfigSection({
    (option) => option.value !== mainModelResourceId,
  );

-  async function generateFocusPrompt() {
-    if (!sourcePrompt.trim() || !generatorModelResourceId || generating) return;
-    setGenerating(true);
-    setGenerationError("");
-    try {
-      const result = await assistantsApi.generateVisionFocus({
-        sourcePrompt,
-        modelResourceId: generatorModelResourceId,
-      });
-      onFocusPromptChange(result.focusPrompt);
-    } catch (error) {
-      setGenerationError(
-        error instanceof Error ? error.message : "生成视觉关注目标失败",
-      );
-    } finally {
-      setGenerating(false);
-    }
-  }
-
  return (
    <SectionCard
      icon={<Camera size={15} />}
@@ -100,56 +99,79 @@ export function VisionConfigSection({
              当前模型未标记支持图片输入，请选择独立视觉模型。
            </p>
          )}
-          <div className="space-y-1.5">
-            <div className="flex items-center justify-between gap-3">
-              <label className="text-sm font-medium text-foreground">
-                视觉关注目标
-              </label>
-              <Button
-                type="button"
-                variant="ghost"
-                size="sm"
-                className="h-7 gap-1.5 rounded-full px-2.5 text-xs"
-                disabled={
-                  generating || !sourcePrompt.trim() || !generatorModelResourceId
-                }
-                onClick={() => void generateFocusPrompt()}
-                title={
-                  !sourcePrompt.trim()
-                    ? "请先填写助手提示词"
-                    : !generatorModelResourceId
-                      ? "请先选择大语言模型"
-                      : "根据当前助手提示词生成可编辑草稿"
-                }
-              >
-                {generating ? (
-                  <Loader2 size={12} className="animate-spin" />
-                ) : (
-                  <Sparkles size={12} />
-                )}
-                {generating ? "生成中" : "根据提示词生成"}
-              </Button>
+          <div className="space-y-2">
+            <label className="text-sm font-medium text-foreground">
+              看图问题生成方式
+            </label>
+            <div role="radiogroup" aria-label="看图问题生成方式" className="space-y-2">
+              {promptModes.map((option) => {
+                const selected = promptMode === option.value;
+                return (
+                  <button
+                    key={option.value}
+                    type="button"
+                    role="radio"
+                    aria-checked={selected}
+                    onClick={() => onPromptModeChange(option.value)}
+                    className={[
+                      "flex w-full items-start gap-3 rounded-xl border px-3.5 py-3 text-left transition-colors",
+                      selected
+                        ? "border-primary bg-surface-strong/70 ring-1 ring-primary"
+                        : "border-hairline bg-background hover:border-hairline-strong",
+                    ].join(" ")}
+                  >
+                    <span
+                      className={[
+                        "mt-0.5 flex h-5 w-5 shrink-0 items-center justify-center rounded-full border",
+                        selected
+                          ? "border-primary bg-primary text-primary-foreground"
+                          : "border-hairline-strong text-transparent",
+                      ].join(" ")}
+                    >
+                      <Check size={12} />
+                    </span>
+                    <span className="min-w-0">
+                      <span className="block text-sm font-medium text-foreground">
+                        {option.title}
+                      </span>
+                      <span className="mt-0.5 block text-xs leading-5 text-muted-foreground">
+                        {option.description}
+                      </span>
+                    </span>
+                  </button>
+                );
+              })}
            </div>
-            <Textarea
-              value={focusPrompt}
-              maxLength={4000}
-              rows={5}
-              onChange={(event) => {
-                onFocusPromptChange(event.target.value);
-                if (generationError) setGenerationError("");
-              }}
-              placeholder="例如：优先识别车辆数量、碰撞位置、明显损伤和人员状态；只报告画面中可直接确认的信息。"
-              className="field-sizing-fixed min-h-28 resize-y border-hairline-strong bg-background text-sm leading-6 text-foreground placeholder:text-muted-soft"
-            />
-            <p className="text-[11px] leading-5 text-muted-soft">
-              可选。系统仍保留通用视觉安全规则；这里用于让每次看图稳定聚焦业务目标。
-            </p>
-            {generationError && (
-              <p className="text-xs leading-5 text-destructive">
-                {generationError}
-              </p>
-            )}
          </div>
+          {promptMode !== "runtime" && (
+            <div className="space-y-1.5">
+              <label className="text-sm font-medium text-foreground">
+                {promptMode === "fixed" ? "固定视觉提示词" : "运行时提示词引导"}
+              </label>
+              <Textarea
+                value={prompt}
+                maxLength={4000}
+                rows={5}
+                onChange={(event) => onPromptChange(event.target.value)}
+                placeholder={
+                  promptMode === "fixed"
+                    ? "例如：请识别画面中的车辆数量、碰撞位置、明显损伤和人员状态。"
+                    : "例如：问题应聚焦车辆数量、碰撞位置和人员状态；只询问画面中可直接判断的信息。"
+                }
+                className="field-sizing-fixed min-h-28 resize-y border-hairline-strong bg-background text-sm leading-6 text-foreground placeholder:text-muted-soft"
+              />
+              <p className="text-[11px] leading-5 text-muted-soft">
+                {promptMode === "fixed"
+                  ? "工具调用时不会让模型改写这段内容。"
+                  : "模型会结合当前对话生成具体问题，并受到这段内容约束。"}
+              </p>
+              {!prompt.trim() && (
+                <p className="text-xs leading-5 text-destructive">
+                  当前模式需要填写提示词后才能保存。
+                </p>
+              )}
+            </div>
+          )}
        </>
      )}
    </SectionCard>
@@ -180,7 +180,8 @@ function blankPromptForm(name: string): AssistantForm {
    },
    visionEnabled: false,
    visionModelResourceId: "",
-    visionFocusPrompt: "",
+    visionPromptMode: "runtime",
+    visionPrompt: "",
    toolIds: [],
  };
 }
@@ -436,7 +437,8 @@ export function AssistantPage(props: AssistantPageProps) {
      },
      visionEnabled: a.visionEnabled,
      visionModelResourceId: a.visionModelResourceId ?? "",
-      visionFocusPrompt: a.visionFocusPrompt ?? "",
+      visionPromptMode: a.visionPromptMode ?? "runtime",
+      visionPrompt: a.visionPrompt ?? "",
      toolIds: a.toolIds ?? [],
    };
    setForm(next);
@@ -487,7 +489,8 @@ export function AssistantPage(props: AssistantPageProps) {
      },
      visionEnabled: false,
      visionModelResourceId: null,
-      visionFocusPrompt: "",
+      visionPromptMode: "runtime",
+      visionPrompt: "",
      modelResourceIds: {},
      knowledgeBaseId: null,
      knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
@@ -552,7 +555,8 @@ export function AssistantPage(props: AssistantPageProps) {
        startup: form.startup,
        visionEnabled: form.visionEnabled,
        visionModelResourceId: form.visionModelResourceId || null,
-        visionFocusPrompt: form.visionFocusPrompt,
+        visionPromptMode: form.visionPromptMode,
+        visionPrompt: form.visionPrompt,
        modelResourceIds: {
          ...(form.model ? { LLM: form.model } : {}),
          ...(form.asr ? { ASR: form.asr } : {}),
@@ -680,7 +684,8 @@ export function AssistantPage(props: AssistantPageProps) {
              graph.settings?.visionModelResourceId ??
              assistant.visionModelResourceId ??
              "",
-            visionFocusPrompt: graph.settings?.visionFocusPrompt ?? "",
+            visionPromptMode: graph.settings?.visionPromptMode ?? "runtime",
+            visionPrompt: graph.settings?.visionPrompt ?? "",
            toolIds: graph.settings?.toolIds ?? [],
            knowledgeBaseId:
              graph.settings?.knowledgeBaseId || assistant.knowledgeBaseId || "",
@@ -108,7 +108,8 @@ function baseUpsert(over: Partial<AssistantUpsert>): AssistantUpsert {
    },
    visionEnabled: false,
    visionModelResourceId: null,
-    visionFocusPrompt: "",
+    visionPromptMode: "runtime",
+    visionPrompt: "",
    modelResourceIds: {},
    knowledgeBaseId: null,
    knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
@@ -96,7 +96,8 @@ function baseUpsertFromTemplate(
    },
    visionEnabled: false,
    visionModelResourceId: null,
-    visionFocusPrompt: "",
+    visionPromptMode: "runtime",
+    visionPrompt: "",
    modelResourceIds: {},
    knowledgeBaseId: null,
    knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
@@ -129,7 +129,8 @@ function fromFlow(nodes: Node[], edges: Edge[]): WorkflowGraph {
      defaultTtsResourceId: "",
      visionEnabled: false,
      visionModelResourceId: "",
-      visionFocusPrompt: "",
+      visionPromptMode: "runtime",
+      visionPrompt: "",
      toolIds: [],
      knowledgeBaseId: "",
      knowledgeMode: "automatic",
@@ -15,7 +15,8 @@ export function settingsFromWorkflowGraph(
    tts: graph.settings.defaultTtsResourceId,
    visionEnabled: graph.settings.visionEnabled,
    visionModelResourceId: graph.settings.visionModelResourceId,
-    visionFocusPrompt: graph.settings.visionFocusPrompt,
+    visionPromptMode: graph.settings.visionPromptMode,
+    visionPrompt: graph.settings.visionPrompt,
    toolIds: graph.settings.toolIds,
    knowledgeBaseId: graph.settings.knowledgeBaseId,
    knowledgeRetrievalConfig: {
@@ -45,7 +46,8 @@ export function workflowGraphWithSettings(
      defaultTtsResourceId: settings.tts ?? "",
      visionEnabled: settings.visionEnabled,
      visionModelResourceId: settings.visionModelResourceId,
-      visionFocusPrompt: settings.visionFocusPrompt,
+      visionPromptMode: settings.visionPromptMode,
+      visionPrompt: settings.visionPrompt,
      toolIds: settings.toolIds,
      knowledgeBaseId: settings.knowledgeBaseId,
      knowledgeMode: settings.knowledgeRetrievalConfig.mode,
@@ -78,8 +78,10 @@ export function AgentNodePanel({
      visionModelResourceId:
        (draft.visionModelResourceId as string) ||
        workflowSettings.visionModelResourceId,
-      visionFocusPrompt:
-        draft.visionFocusPrompt ?? workflowSettings.visionFocusPrompt,
+      visionPromptMode:
+        draft.visionPromptMode ?? workflowSettings.visionPromptMode,
+      visionPrompt:
+        draft.visionPrompt ?? workflowSettings.visionPrompt,
      ...(!isRealtime
        ? {
            llmResourceId:
@@ -308,13 +310,8 @@ export function AgentNodePanel({
              }
              enabled={Boolean(draft.visionEnabled)}
              modelResourceId={draft.visionModelResourceId ?? ""}
-              focusPrompt={draft.visionFocusPrompt ?? ""}
-              sourcePrompt={draft.prompt ?? ""}
-              generatorModelResourceId={
-                isRealtime
-                  ? draft.visionModelResourceId ?? ""
-                  : (draft.llmResourceId as string) || ""
-              }
+              promptMode={draft.visionPromptMode ?? "runtime"}
+              prompt={draft.visionPrompt ?? ""}
              mainModelResourceId={
                isRealtime
                  ? workflowSettings.realtime ?? ""
@@ -339,8 +336,11 @@ export function AgentNodePanel({
              onModelResourceIdChange={(visionModelResourceId) =>
                set("visionModelResourceId", visionModelResourceId)
              }
-              onFocusPromptChange={(visionFocusPrompt) =>
-                set("visionFocusPrompt", visionFocusPrompt)
+              onPromptModeChange={(visionPromptMode) =>
+                set("visionPromptMode", visionPromptMode)
+              }
+              onPromptChange={(visionPrompt) =>
+                set("visionPrompt", visionPrompt)
              }
            />

@@ -184,13 +184,8 @@ export function GlobalSettingsPanel({
          }
          enabled={settings.visionEnabled}
          modelResourceId={settings.visionModelResourceId}
-          focusPrompt={settings.visionFocusPrompt}
-          sourcePrompt={settings.globalPrompt}
-          generatorModelResourceId={
-            settings.runtimeMode === "realtime"
-              ? settings.visionModelResourceId
-              : settings.llm ?? ""
-          }
+          promptMode={settings.visionPromptMode}
+          prompt={settings.visionPrompt}
          mainModelResourceId={
            settings.runtimeMode === "realtime"
              ? settings.realtime ?? ""
@@ -216,8 +211,11 @@ export function GlobalSettingsPanel({
          onModelResourceIdChange={(visionModelResourceId) =>
            onSettingsChange({ ...settings, visionModelResourceId })
          }
-          onFocusPromptChange={(visionFocusPrompt) =>
-            onSettingsChange({ ...settings, visionFocusPrompt })
+          onPromptModeChange={(visionPromptMode) =>
+            onSettingsChange({ ...settings, visionPromptMode })
+          }
+          onPromptChange={(visionPrompt) =>
+            onSettingsChange({ ...settings, visionPrompt })
          }
        />

@@ -3,7 +3,7 @@
 import * as LucideIcons from "lucide-react";
 import { Circle, type LucideIcon } from "lucide-react";

-import type { NodeSpecDto, TurnConfig } from "@/lib/api";
+import type { NodeSpecDto, TurnConfig, VisionPromptMode } from "@/lib/api";
 import { defaultTurnConfig } from "@/lib/turn-config";

 export type WorkflowNodeType =
@@ -54,7 +54,8 @@ export type WorkflowNodeData = {
  ttsResourceId?: string;
  visionEnabled?: boolean;
  visionModelResourceId?: string;
-  visionFocusPrompt?: string;
+  visionPromptMode?: VisionPromptMode;
+  visionPrompt?: string;
  enableInterrupt?: boolean;
  turnConfig?: TurnConfig;
  toolId?: string;
@@ -214,7 +215,8 @@ export type WorkflowGraph = {
    defaultTtsResourceId: string;
    visionEnabled: boolean;
    visionModelResourceId: string;
-    visionFocusPrompt: string;
+    visionPromptMode: VisionPromptMode;
+    visionPrompt: string;
    toolIds: string[];
    knowledgeBaseId: string;
    knowledgeMode: "automatic" | "on_demand";
@@ -252,7 +254,8 @@ export function defaultGraph(): WorkflowGraph {
      defaultTtsResourceId: "",
      visionEnabled: false,
      visionModelResourceId: "",
-      visionFocusPrompt: "",
+      visionPromptMode: "runtime",
+      visionPrompt: "",
      toolIds: [],
      knowledgeBaseId: "",
      knowledgeMode: "automatic",
@@ -5,6 +5,7 @@ import type {
  SystemToolKind,
  Tool,
  TurnConfig,
+  VisionPromptMode,
 } from "@/lib/api";

 import type { WorkflowGraph } from "./specs";
@@ -18,7 +19,8 @@ export type WorkflowSettings = {
  tts?: string;
  visionEnabled: boolean;
  visionModelResourceId: string;
-  visionFocusPrompt: string;
+  visionPromptMode: VisionPromptMode;
+  visionPrompt: string;
  toolIds: string[];
  knowledgeBaseId: string;
  knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
@@ -315,6 +315,8 @@ export type AnalysisConfig = {
  fields: AnalysisField[];
 };

+export type VisionPromptMode = "fixed" | "runtime" | "guided";
+
 /** 后端 AssistantOut(宽表 STI:瘦字段平铺,workflow 用 graph)。apiKey 读时打码 */
 export type Assistant = {
  id: string;
@@ -327,7 +329,8 @@ export type Assistant = {
  startup: StartupConfig;
  visionEnabled: boolean;
  visionModelResourceId: string | null;
-  visionFocusPrompt: string;
+  visionPromptMode: VisionPromptMode;
+  visionPrompt: string;
  modelResourceIds: Partial<Record<ModelType, string>>;
  knowledgeBaseId: string | null;
  knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
@@ -372,14 +375,6 @@ export const assistantsApi = {
  // 服务端整行复制(含真 key,密钥不经浏览器)
  duplicate: (id: string) =>
    request<Assistant>(`/api/assistants/${id}/duplicate`, { method: "POST" }),
-  generateVisionFocus: (body: {
-    sourcePrompt: string;
-    modelResourceId: string;
-  }) =>
-    request<{ focusPrompt: string }>("/api/assistants/vision-focus/generate", {
-      method: "POST",
-      body: JSON.stringify(body),
-    }),
  remove: (id: string) =>
    request<{ ok: boolean }>(`/api/assistants/${id}`, { method: "DELETE" }),
 };