feat: add configurable vision focus prompts

This commit is contained in:
Xin Wang
2026-08-12 17:29:30 +08:00
parent 7560f9a525
commit 09b07751b6
26 changed files with 477 additions and 20 deletions
@@ -129,6 +129,7 @@ function fromFlow(nodes: Node[], edges: Edge[]): WorkflowGraph {
defaultTtsResourceId: "",
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
toolIds: [],
knowledgeBaseId: "",
knowledgeMode: "automatic",
@@ -15,6 +15,7 @@ export function settingsFromWorkflowGraph(
tts: graph.settings.defaultTtsResourceId,
visionEnabled: graph.settings.visionEnabled,
visionModelResourceId: graph.settings.visionModelResourceId,
visionFocusPrompt: graph.settings.visionFocusPrompt,
toolIds: graph.settings.toolIds,
knowledgeBaseId: graph.settings.knowledgeBaseId,
knowledgeRetrievalConfig: {
@@ -44,6 +45,7 @@ export function workflowGraphWithSettings(
defaultTtsResourceId: settings.tts ?? "",
visionEnabled: settings.visionEnabled,
visionModelResourceId: settings.visionModelResourceId,
visionFocusPrompt: settings.visionFocusPrompt,
toolIds: settings.toolIds,
knowledgeBaseId: settings.knowledgeBaseId,
knowledgeMode: settings.knowledgeRetrievalConfig.mode,
@@ -78,6 +78,8 @@ export function AgentNodePanel({
visionModelResourceId:
(draft.visionModelResourceId as string) ||
workflowSettings.visionModelResourceId,
visionFocusPrompt:
draft.visionFocusPrompt ?? workflowSettings.visionFocusPrompt,
...(!isRealtime
? {
llmResourceId:
@@ -306,6 +308,13 @@ export function AgentNodePanel({
}
enabled={Boolean(draft.visionEnabled)}
modelResourceId={draft.visionModelResourceId ?? ""}
focusPrompt={draft.visionFocusPrompt ?? ""}
sourcePrompt={draft.prompt ?? ""}
generatorModelResourceId={
isRealtime
? draft.visionModelResourceId ?? ""
: (draft.llmResourceId as string) || ""
}
mainModelResourceId={
isRealtime
? workflowSettings.realtime ?? ""
@@ -330,6 +339,9 @@ export function AgentNodePanel({
onModelResourceIdChange={(visionModelResourceId) =>
set("visionModelResourceId", visionModelResourceId)
}
onFocusPromptChange={(visionFocusPrompt) =>
set("visionFocusPrompt", visionFocusPrompt)
}
/>
<SectionCard
@@ -184,6 +184,13 @@ export function GlobalSettingsPanel({
}
enabled={settings.visionEnabled}
modelResourceId={settings.visionModelResourceId}
focusPrompt={settings.visionFocusPrompt}
sourcePrompt={settings.globalPrompt}
generatorModelResourceId={
settings.runtimeMode === "realtime"
? settings.visionModelResourceId
: settings.llm ?? ""
}
mainModelResourceId={
settings.runtimeMode === "realtime"
? settings.realtime ?? ""
@@ -209,6 +216,9 @@ export function GlobalSettingsPanel({
onModelResourceIdChange={(visionModelResourceId) =>
onSettingsChange({ ...settings, visionModelResourceId })
}
onFocusPromptChange={(visionFocusPrompt) =>
onSettingsChange({ ...settings, visionFocusPrompt })
}
/>
<SectionCard
@@ -54,6 +54,7 @@ export type WorkflowNodeData = {
ttsResourceId?: string;
visionEnabled?: boolean;
visionModelResourceId?: string;
visionFocusPrompt?: string;
enableInterrupt?: boolean;
turnConfig?: TurnConfig;
toolId?: string;
@@ -213,6 +214,7 @@ export type WorkflowGraph = {
defaultTtsResourceId: string;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
toolIds: string[];
knowledgeBaseId: string;
knowledgeMode: "automatic" | "on_demand";
@@ -250,6 +252,7 @@ export function defaultGraph(): WorkflowGraph {
defaultTtsResourceId: "",
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
toolIds: [],
knowledgeBaseId: "",
knowledgeMode: "automatic",
@@ -18,6 +18,7 @@ export type WorkflowSettings = {
tts?: string;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
toolIds: string[];
knowledgeBaseId: string;
knowledgeRetrievalConfig: KnowledgeRetrievalConfig;