diff --git a/backend/routes/assistants.py b/backend/routes/assistants.py index 331fb0d..3f05b1f 100644 --- a/backend/routes/assistants.py +++ b/backend/routes/assistants.py @@ -85,6 +85,7 @@ async def _validate_workflow_references( graph = body.graph engine = WorkflowEngine(graph) settings = graph.get("settings") or {} + runtime_mode = str(settings.get("runtimeMode") or "pipeline") resource_expectations: dict[str, str] = {} vision_resource_ids: set[str] = set() for key, capability in ( @@ -118,16 +119,22 @@ async def _validate_workflow_references( stage = engine.agent_stage_config(node_id) if not stage.vision_enabled: continue - resource_id = ( - stage.vision_model_resource_id or stage.llm_resource_id - ) + if stage.vision_model_resource_id: + resource_id = stage.vision_model_resource_id + capability = "LLM" + elif runtime_mode == "realtime": + resource_id = str(settings.get("defaultRealtimeResourceId") or "") + capability = "Realtime" + else: + resource_id = stage.llm_resource_id + capability = "LLM" if not resource_id: raise HTTPException( 400, f"Agent 节点 {node_id} 开启视觉理解时必须选择" - "支持图片输入的大语言模型或视觉模型", + "支持图片输入的当前模型或独立视觉模型", ) - resource_expectations[resource_id] = "LLM" + resource_expectations[resource_id] = capability vision_resource_ids.add(resource_id) for resource_id, capability in resource_expectations.items(): resource = await session.get(ModelResource, resource_id) diff --git a/backend/routes/model_registry.py b/backend/routes/model_registry.py index c463ad6..1333930 100644 --- a/backend/routes/model_registry.py +++ b/backend/routes/model_registry.py @@ -30,6 +30,19 @@ router = APIRouter( tags=["model-registry"], dependencies=[Depends(require_admin)], ) +NATIVE_IMAGE_UNSUPPORTED_REALTIME_INTERFACES = frozenset( + {"qwen-audio-realtime"} +) + + +def _supports_configurable_image_input(definition: InterfaceDefinition) -> bool: + if definition.capability == "LLM": + return True + return ( + definition.capability == "Realtime" + and definition.interface_type + not in NATIVE_IMAGE_UNSUPPORTED_REALTIME_INTERFACES + ) def _definition_dict(row: InterfaceDefinition) -> dict: @@ -156,9 +169,11 @@ async def create_model_resource( interface_type=definition.interface_type, values=body.values, secrets=secrets, - support_image_input=body.support_image_input - if definition.capability == "LLM" - else False, + support_image_input=( + body.support_image_input + if _supports_configurable_image_input(definition) + else False + ), enabled=body.enabled, is_default=body.is_default, ) @@ -244,7 +259,9 @@ async def update_model_resource( row.values = body.values row.secrets = secrets row.support_image_input = ( - body.support_image_input if definition.capability == "LLM" else False + body.support_image_input + if _supports_configurable_image_input(definition) + else False ) row.enabled = body.enabled row.is_default = body.is_default diff --git a/backend/services/brains/base.py b/backend/services/brains/base.py index 08e2e03..1af2e10 100644 --- a/backend/services/brains/base.py +++ b/backend/services/brains/base.py @@ -111,6 +111,7 @@ class RealtimeBrainRuntime: session_id: str = "" client_tools: ClientToolPort | None = None set_input_enabled: Callable[[bool], None] | None = None + capture_image: Callable[[str], Awaitable[Any | None]] | None = None class BaseBrain: diff --git a/backend/services/node_specs.py b/backend/services/node_specs.py index 93d09c5..3278c4c 100644 --- a/backend/services/node_specs.py +++ b/backend/services/node_specs.py @@ -438,8 +438,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: "defaultRealtimeResourceId" ): errors.append("Realtime 工作流必须选择 Realtime 模型") - if runtime_mode == "realtime" and settings.get("visionEnabled"): - errors.append("Realtime 工作流暂不支持视觉能力") if ( runtime_mode == "realtime" and settings.get("knowledgeBaseId") @@ -488,10 +486,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: f"Realtime Agent 节点 {node_id} 不支持节点级模型、语音或交互策略覆盖" ) source = settings if data.get("inheritGlobalConfig", True) else data - if source.get("visionEnabled"): - errors.append( - f"Realtime Agent 节点 {node_id} 暂不支持视觉能力" - ) if ( source.get("knowledgeBaseId") and source.get("knowledgeMode") != "on_demand" diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index 79b90c5..5597043 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -7,14 +7,10 @@ """ import asyncio -import base64 -from io import BytesIO from typing import Any from loguru import logger from models import AssistantConfig -from openai import AsyncOpenAI -from PIL import Image from services.brains import Brain, BrainRuntime, build_brain from services.brains.base import RealtimeBrainRuntime from services.conversation_history import ConversationRecorder @@ -34,11 +30,12 @@ from services.knowledge import search as search_knowledge from services.client_tools import ClientToolBroker from services.tool_policy import policy_for_tool from services.vision import ( - VISION_ANALYSIS_SYSTEM_PROMPT, VISION_SYSTEM_HINT, VISION_TOOL_NAME, + analyze_image_with_vision_model, config_with_main_llm_as_vision, config_with_vision_resource, + image_data_uri, ) from services.workflow_engine import WorkflowEngine @@ -164,19 +161,6 @@ def _workflow_vision_uses_main_llm( return True -def _image_data_uri(frame: UserImageRawFrame) -> str: - if not frame.format: - raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") - buffer = BytesIO() - Image.frombytes(frame.format, frame.size, frame.image).save( - buffer, - format="JPEG", - quality=85, - ) - encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{encoded}" - - def _multimodal_user_input_frame( image_frame: UserImageRawFrame, prompt_text: str, @@ -197,7 +181,7 @@ def _multimodal_user_input_frame( {"type": "text", "text": prompt_text}, { "type": "image_url", - "image_url": {"url": _image_data_uri(image_frame)}, + "image_url": {"url": image_data_uri(image_frame)}, }, ], } @@ -206,45 +190,6 @@ def _multimodal_user_input_frame( ) -async def _analyze_image_with_vision_model( - cfg: AssistantConfig, - frame: UserImageRawFrame, - question: str, -) -> str: - if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}: - raise ValueError(f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}") - - data_uri = await asyncio.to_thread(_image_data_uri, frame) - extra_body = cfg.vision_llm_values.get("extraBody") - extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} - client = AsyncOpenAI( - api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"), - base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"), - ) - try: - response = await client.chat.completions.create( - model=_require(cfg.vision_model, "Vision LLM modelId"), - messages=[ - {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": data_uri}}, - ], - }, - ], - **extra, - ) - finally: - await client.close() - - content = response.choices[0].message.content if response.choices else "" - if isinstance(content, str): - return content.strip() - return str(content or "").strip() - - async def run_pipeline( transport, cfg: AssistantConfig, @@ -278,12 +223,11 @@ async def run_pipeline( raise ValueError(f"类型 {cfg.type} 不支持 realtime 运行模式") if cfg.runtimeMode == "realtime": - if vision_enabled: - logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话") await run_realtime_pipeline( transport, cfg, brain=brain, + vision_enabled=vision_enabled, assistant_id=assistant_id, channel=channel, ) @@ -524,7 +468,7 @@ async def run_pipeline( ) try: frame = await vision_capture.request_image(params.llm, request) - observation = await _analyze_image_with_vision_model(cfg, frame, question) + observation = await analyze_image_with_vision_model(cfg, frame, question) except asyncio.TimeoutError: await params.result_callback( { @@ -614,7 +558,7 @@ async def run_pipeline( ) try: frame = await vision_capture.request_image(llm, request) - observation = await _analyze_image_with_vision_model( + observation = await analyze_image_with_vision_model( active_vision_config(), frame, question, @@ -830,7 +774,7 @@ async def run_pipeline( try: assert analysis_cfg is not None - observation = await _analyze_image_with_vision_model( + observation = await analyze_image_with_vision_model( analysis_cfg, image_frame, value.prompt_text, @@ -883,6 +827,7 @@ async def run_realtime_pipeline( cfg: AssistantConfig, *, brain: Brain, + vision_enabled: bool = False, assistant_id: str | None = None, channel: str = "webrtc", ) -> None: @@ -894,6 +839,8 @@ async def run_realtime_pipeline( input_sample_rate, output_sample_rate = realtime_audio_sample_rates(cfg) worker_holder: dict[str, PipelineWorker] = {} input_state = {"enabled": True} + vision_state: dict[str, str | None] = {"client_id": None} + vision_capture = VisionCaptureProcessor() async def queue_call_end(reason: str) -> None: worker = worker_holder.get("worker") @@ -946,6 +893,18 @@ async def run_realtime_pipeline( ) greeting = await brain.greeting(cfg) + async def capture_image(question: str): + user_id = vision_state["client_id"] + if not user_id: + return None + request = UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=False, + function_name=VISION_TOOL_NAME, + ) + return await vision_capture.request_image(realtime, request) + recorder = await ConversationRecorder.start( assistant_id=assistant_id, assistant_name=cfg.name, @@ -961,6 +920,7 @@ async def run_realtime_pipeline( pipeline = Pipeline( [ transport.input(), + vision_capture, client_tools, session_update, user_input, @@ -995,6 +955,7 @@ async def run_realtime_pipeline( session_id=cfg.conversation_id or "", client_tools=client_tools, set_input_enabled=set_input_enabled, + capture_image=capture_image if vision_enabled else None, ), ) @@ -1005,6 +966,8 @@ async def run_realtime_pipeline( brain=brain, text_input=user_input, greeting=greeting, + vision_enabled=vision_enabled, + vision_state=vision_state, ) runner = WorkerRunner(handle_sigint=False) run_status = "completed" diff --git a/backend/services/pipecat/pipeline_events.py b/backend/services/pipecat/pipeline_events.py index 277c1b9..ef8c782 100644 --- a/backend/services/pipecat/pipeline_events.py +++ b/backend/services/pipecat/pipeline_events.py @@ -230,6 +230,8 @@ def bind_realtime_pipeline_events( brain, text_input, greeting: str, + vision_enabled: bool, + vision_state: dict[str, str | None], ) -> None: """Connect text and lifecycle events for a realtime model pipeline.""" @@ -274,6 +276,19 @@ def bind_realtime_pipeline_events( @transport.event_handler("on_client_connected") async def on_client_connected(_transport, _client): + if vision_enabled: + try: + vision_state["client_id"] = get_transport_client_id( + _transport, + _client, + ) + await maybe_capture_participant_camera(_transport, _client) + logger.info( + f"Realtime 视觉理解已接入视频客户端: " + f"{vision_state['client_id']}" + ) + except Exception as exc: # noqa: BLE001 - video remains optional + logger.warning(f"Realtime 视觉理解摄像头捕获初始化失败: {exc}") await brain.on_connected(greeting_pending=bool(greeting)) await brain.on_client_ready() if greeting: diff --git a/backend/services/vision.py b/backend/services/vision.py index eabe9c4..0bec588 100644 --- a/backend/services/vision.py +++ b/backend/services/vision.py @@ -1,8 +1,15 @@ -"""Shared vision capability metadata and Workflow model resolution.""" +"""Shared camera-frame analysis for Pipeline and Realtime runtimes.""" from __future__ import annotations +import asyncio +import base64 +from io import BytesIO + from models import AssistantConfig, RuntimeModelResource +from openai import AsyncOpenAI +from PIL import Image +from pipecat.frames.frames import UserImageRawFrame VISION_TOOL_NAME = "fetch_user_image" @@ -17,6 +24,68 @@ VISION_ANALYSIS_SYSTEM_PROMPT = ( ) +def _require(value: str, label: str) -> str: + if value: + return value + raise ValueError(f"缺少模型资源配置: {label}") + + +def image_data_uri(frame: UserImageRawFrame) -> str: + """Encode one Pipecat camera frame for a vision chat-completion request.""" + if not frame.format: + raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") + buffer = BytesIO() + Image.frombytes(frame.format, frame.size, frame.image).save( + buffer, + format="JPEG", + quality=85, + ) + encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") + return f"data:image/jpeg;base64,{encoded}" + + +async def analyze_image_with_vision_model( + cfg: AssistantConfig, + frame: UserImageRawFrame, + question: str, +) -> str: + """Analyze one frame with the configured independent vision LLM.""" + if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}: + raise ValueError( + f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}" + ) + + data_uri = await asyncio.to_thread(image_data_uri, frame) + extra_body = cfg.vision_llm_values.get("extraBody") + extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} + client = AsyncOpenAI( + api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"), + base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"), + ) + try: + response = await client.chat.completions.create( + model=_require(cfg.vision_model, "Vision LLM modelId"), + messages=[ + {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, + { + "role": "user", + "content": [ + {"type": "text", "text": question}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ], + }, + ], + **extra, + ) + finally: + await client.close() + + content = response.choices[0].message.content if response.choices else "" + if isinstance(content, str): + return content.strip() + return str(content or "").strip() + + def config_with_vision_resource( cfg: AssistantConfig, resource: RuntimeModelResource, diff --git a/backend/services/workflow/realtime.py b/backend/services/workflow/realtime.py index f786641..6e73e66 100644 --- a/backend/services/workflow/realtime.py +++ b/backend/services/workflow/realtime.py @@ -28,6 +28,12 @@ from services.runtime_variables import DynamicVariableError, DynamicVariableStor from services.system_tools import state_update_properties, system_tool_kind from services.tool_executor import ToolExecutionError, ToolExecutor from services.tool_policy import policy_for_tool +from services.vision import ( + VISION_SYSTEM_HINT, + VISION_TOOL_NAME, + analyze_image_with_vision_model, + config_with_vision_resource, +) from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION from services.workflow.models import WorkflowRuntimeState, WorkflowStatus from services.workflow.output import WorkflowOutput @@ -385,10 +391,13 @@ class WorkflowRealtimeController: return RealtimeActivation(continue_response=generate) def _agent_prompt(self, node_id: str) -> str: - return ( + prompt = ( f"{self._engine.prompt_for(node_id, self._store)}\n\n" f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}" ) + if self._engine.agent_stage_config(node_id).vision_enabled: + prompt = f"{prompt}\n{VISION_SYSTEM_HINT}" + return prompt def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]: stage = self._engine.agent_stage_config(node_id) @@ -428,6 +437,9 @@ class WorkflowRealtimeController: knowledge = self._knowledge_tool(node_id, transition_id) if knowledge: add(*knowledge) + vision = self._vision_tool(node_id, transition_id) + if vision: + add(*vision) for edge in self._engine.edge_tool_edges(node_id): add(*self._transition_tool(edge, node_id, transition_id)) self._handlers = handlers @@ -519,6 +531,105 @@ class WorkflowRealtimeController: handler, ) + def _vision_tool( + self, + node_id: str, + transition_id: int, + ) -> tuple[RealtimeTool, ToolHandler] | None: + """Build the on-demand camera tool for the active Realtime Agent.""" + stage = self._engine.agent_stage_config(node_id) + if not stage.vision_enabled: + return None + + vision_cfg: AssistantConfig | None = None + if stage.vision_model_resource_id: + resource = self._cfg.workflow_model_resources.get( + stage.vision_model_resource_id + ) + if resource is not None: + vision_cfg = config_with_vision_resource(self._cfg, resource) + + async def handler(arguments: dict[str, Any]) -> RealtimeToolResult: + if not self._is_current(node_id, transition_id): + return RealtimeToolResult( + {"status": "stale", "message": "当前 Agent 已经切换。"}, + continue_response=False, + ) + question = str(arguments.get("question") or "").strip() + if not question: + return RealtimeToolResult( + {"status": "error", "message": "视觉问题为空"} + ) + if self._runtime.capture_image is None: + return RealtimeToolResult( + {"status": "no_video", "message": "当前会话未启用视频输入"} + ) + try: + frame = await self._runtime.capture_image(question) + except TimeoutError: + return RealtimeToolResult( + {"status": "timeout", "message": "等待摄像头画面超时"} + ) + except Exception as exc: # noqa: BLE001 - expose a stable tool error + logger.warning(f"Realtime Workflow 获取摄像头画面失败:{exc}") + return RealtimeToolResult( + {"status": "error", "message": "暂时无法获取摄像头画面"} + ) + if frame is None: + return RealtimeToolResult( + {"status": "no_video", "message": "当前没有可用的视频客户端"} + ) + if not self._is_current(node_id, transition_id): + return RealtimeToolResult( + {"status": "stale", "message": "获取画面时 Agent 已经切换。"}, + continue_response=False, + ) + + try: + if vision_cfg is not None: + observation = await analyze_image_with_vision_model( + vision_cfg, + frame, + question, + ) + else: + analyze = getattr(self._runtime.realtime, "analyze_image", None) + if not callable(analyze): + raise ValueError("当前 Realtime 适配器未实现原生视觉理解") + observation = await analyze(frame, question) + except Exception as exc: # noqa: BLE001 - provider errors become tool output + logger.warning(f"Realtime Workflow 视觉模型调用失败:{exc}") + return RealtimeToolResult( + {"status": "error", "message": "视觉模型暂时不可用"} + ) + if not self._is_current(node_id, transition_id): + return RealtimeToolResult( + {"status": "stale", "message": "分析画面时 Agent 已经切换。"}, + continue_response=False, + ) + return RealtimeToolResult( + { + "status": "ok", + "question": question, + "observation": observation, + } + ) + + return ( + RealtimeTool( + name=VISION_TOOL_NAME, + description="获取用户摄像头的当前画面,并回答一个视觉问题。", + properties={ + "question": { + "type": "string", + "description": "需要根据当前画面判断的具体问题", + } + }, + required=("question",), + ), + handler, + ) + def _system_tool( self, tool: RuntimeTool, diff --git a/frontend/src/components/editor/vision-config-section.tsx b/frontend/src/components/editor/vision-config-section.tsx index 81007d5..40ae4c0 100644 --- a/frontend/src/components/editor/vision-config-section.tsx +++ b/frontend/src/components/editor/vision-config-section.tsx @@ -19,6 +19,7 @@ export function VisionConfigSection({ enabled, modelResourceId, mainModelResourceId, + mainModelSupportsImageInput, modelOptions, onEnabledChange, onModelResourceIdChange, @@ -28,13 +29,14 @@ export function VisionConfigSection({ enabled: boolean; modelResourceId: string; mainModelResourceId: string; + mainModelSupportsImageInput?: boolean; modelOptions: VisionModelOption[]; onEnabledChange: (enabled: boolean) => void; onModelResourceIdChange: (modelResourceId: string) => void; }) { - const mainModelSupportsVision = modelOptions.some( - (option) => option.value === mainModelResourceId, - ); + const mainModelSupportsVision = + mainModelSupportsImageInput ?? + modelOptions.some((option) => option.value === mainModelResourceId); const independentModelOptions = modelOptions.filter( (option) => option.value !== mainModelResourceId, ); @@ -62,7 +64,7 @@ export function VisionConfigSection({ /> {!modelResourceId && !mainModelSupportsVision && (

- 当前大语言模型未标记支持图片输入,请选择独立视觉模型。 + 当前模型未标记支持图片输入,请选择独立视觉模型。

)} diff --git a/frontend/src/components/pages/AssistantPage.tsx b/frontend/src/components/pages/AssistantPage.tsx index 6fa38cf..94e2de8 100644 --- a/frontend/src/components/pages/AssistantPage.tsx +++ b/frontend/src/components/pages/AssistantPage.tsx @@ -411,7 +411,11 @@ export function AssistantPage(props: AssistantPageProps) { const credOptions = (type: ModelResource["capability"]) => modelResources .filter((c) => c.capability === type) - .map((c) => ({ value: c.id, label: c.name })); + .map((c) => ({ + value: c.id, + label: c.name, + supportImageInput: c.supportImageInput, + })); const agentOptions = (interfaceType: "dify" | "fastgpt" | "opencode") => modelResources .filter( diff --git a/frontend/src/components/pages/ComponentsModelsPage.tsx b/frontend/src/components/pages/ComponentsModelsPage.tsx index dfa6ece..4ee3218 100644 --- a/frontend/src/components/pages/ComponentsModelsPage.tsx +++ b/frontend/src/components/pages/ComponentsModelsPage.tsx @@ -66,6 +66,20 @@ const capabilities: ModelType[] = [ ]; const capabilityFilters = ["全部", ...capabilities] as const; +const nativeImageUnsupportedRealtimeInterfaces = new Set([ + "qwen-audio-realtime", +]); + +function canConfigureImageInput( + capability: ModelType, + interfaceType: string, +): boolean { + return ( + capability === "LLM" || + (capability === "Realtime" && + !nativeImageUnsupportedRealtimeInterfaces.has(interfaceType)) + ); +} type ResourceDraft = { name: string; @@ -294,6 +308,12 @@ export function ComponentsModelsPage() { interfaceType, values: definition ? defaults(definition) : {}, secrets: {}, + supportImageInput: canConfigureImageInput( + previous.capability, + interfaceType, + ) + ? previous.supportImageInput + : false, })); setStoredSecretMasks({}); setExtraBodyDraft(""); @@ -310,8 +330,7 @@ export function ComponentsModelsPage() { interfaceType: first?.interfaceType ?? "", values: first ? defaults(first) : {}, secrets: {}, - supportImageInput: - capability === "LLM" ? previous.supportImageInput : false, + supportImageInput: false, })); setStoredSecretMasks({}); setExtraBodyDraft(""); @@ -392,7 +411,9 @@ export function ComponentsModelsPage() { ? { ...storedSecretMasks, ...draft.secrets } : draft.secrets, supportImageInput: - draft.capability === "LLM" ? draft.supportImageInput : false, + canConfigureImageInput(draft.capability, draft.interfaceType) + ? draft.supportImageInput + : false, enabled: editingId ? resources.find((resource) => resource.id === editingId)?.enabled ?? true : true, @@ -747,11 +768,25 @@ export function ComponentsModelsPage() { - {draft.capability === "LLM" && ( + {(draft.capability === "LLM" || + draft.capability === "Realtime") && (
- 支持图片输入 +
+

原生支持图片输入

+ {draft.capability === "Realtime" && ( +

+ Qwen Audio Realtime 请保持关闭,并为助手选择独立视觉模型。 +

+ )} +
setDraft((previous) => ({ ...previous, diff --git a/frontend/src/components/workflow/WorkflowCanvas.tsx b/frontend/src/components/workflow/WorkflowCanvas.tsx index ecebf15..02ac41f 100644 --- a/frontend/src/components/workflow/WorkflowCanvas.tsx +++ b/frontend/src/components/workflow/WorkflowCanvas.tsx @@ -798,6 +798,7 @@ export function WorkflowCanvas({ llmOptions={modelOptions.llm} asrOptions={modelOptions.asr} ttsOptions={modelOptions.tts} + realtimeOptions={modelOptions.realtime} visionOptions={modelOptions.vision} dynamicVariableOptions={dynamicVariableOptions} workflowSettings={settings} diff --git a/frontend/src/components/workflow/panels/AgentNodePanel.tsx b/frontend/src/components/workflow/panels/AgentNodePanel.tsx index aafa34a..66cdd3f 100644 --- a/frontend/src/components/workflow/panels/AgentNodePanel.tsx +++ b/frontend/src/components/workflow/panels/AgentNodePanel.tsx @@ -37,6 +37,7 @@ export function AgentNodePanel({ llmOptions, asrOptions, ttsOptions, + realtimeOptions, visionOptions, dynamicVariableOptions, }: { @@ -49,6 +50,7 @@ export function AgentNodePanel({ llmOptions: ModelOption[]; asrOptions: ModelOption[]; ttsOptions: ModelOption[]; + realtimeOptions: ModelOption[]; visionOptions: ModelOption[]; dynamicVariableOptions: ModelOption[]; }) { @@ -71,6 +73,11 @@ export function AgentNodePanel({ } setPatch({ inheritGlobalConfig: false, + visionEnabled: + draft.visionEnabled ?? workflowSettings.visionEnabled, + visionModelResourceId: + (draft.visionModelResourceId as string) || + workflowSettings.visionModelResourceId, ...(!isRealtime ? { llmResourceId: @@ -79,11 +86,6 @@ export function AgentNodePanel({ (draft.asrResourceId as string) || workflowSettings.asr || "", ttsResourceId: (draft.ttsResourceId as string) || workflowSettings.tts || "", - visionEnabled: - draft.visionEnabled ?? workflowSettings.visionEnabled, - visionModelResourceId: - (draft.visionModelResourceId as string) || - workflowSettings.visionModelResourceId, enableInterrupt: draft.enableInterrupt ?? workflowSettings.allowInterrupt, turnConfig: agentTurnConfig, @@ -295,12 +297,29 @@ export function AgentNodePanel({ : null} - {!isRealtime ? option.value === workflowSettings.realtime, + )?.supportImageInput, + ) + : undefined + } modelOptions={visionOptions} onEnabledChange={(visionEnabled) => setPatch({ @@ -311,7 +330,7 @@ export function AgentNodePanel({ onModelResourceIdChange={(visionModelResourceId) => set("visionModelResourceId", visionModelResourceId) } - /> : null} + /> } diff --git a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx index 8dd47ab..4f74c95 100644 --- a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx +++ b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx @@ -175,12 +175,29 @@ export function GlobalSettingsPanel({ - {settings.runtimeMode === "pipeline" ? option.value === settings.realtime, + )?.supportImageInput, + ) + : undefined + } modelOptions={modelOptions.vision} onEnabledChange={(visionEnabled) => onSettingsChange({ @@ -192,7 +209,7 @@ export function GlobalSettingsPanel({ onModelResourceIdChange={(visionModelResourceId) => onSettingsChange({ ...settings, visionModelResourceId }) } - /> : null} + /> } diff --git a/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx b/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx index f21a39d..3af3a71 100644 --- a/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx +++ b/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx @@ -31,6 +31,7 @@ export function NodeSettingsPanel({ llmOptions, asrOptions, ttsOptions, + realtimeOptions, visionOptions, dynamicVariableOptions, workflowSettings, @@ -44,6 +45,7 @@ export function NodeSettingsPanel({ llmOptions: ModelOption[]; asrOptions: ModelOption[]; ttsOptions: ModelOption[]; + realtimeOptions: ModelOption[]; visionOptions: ModelOption[]; dynamicVariableOptions: ModelOption[]; workflowSettings: WorkflowSettings; @@ -136,6 +138,7 @@ export function NodeSettingsPanel({ llmOptions={llmOptions} asrOptions={asrOptions} ttsOptions={ttsOptions} + realtimeOptions={realtimeOptions} visionOptions={visionOptions} dynamicVariableOptions={dynamicVariableOptions} /> diff --git a/frontend/src/components/workflow/types.ts b/frontend/src/components/workflow/types.ts index ae62768..3b839ec 100644 --- a/frontend/src/components/workflow/types.ts +++ b/frontend/src/components/workflow/types.ts @@ -30,6 +30,7 @@ export type ModelOption = { value: string; label: string; disabled?: boolean; + supportImageInput?: boolean; toolType?: Tool["type"]; systemKind?: SystemToolKind; };