feat: add realtime workflow vision tools

This commit is contained in:
Xin Wang
2026-08-05 13:35:00 +08:00
parent 11374238f3
commit 10bea48d72
16 changed files with 362 additions and 103 deletions
@@ -230,6 +230,8 @@ def bind_realtime_pipeline_events(
brain,
text_input,
greeting: str,
vision_enabled: bool,
vision_state: dict[str, str | None],
) -> None:
"""Connect text and lifecycle events for a realtime model pipeline."""
@@ -274,6 +276,19 @@ def bind_realtime_pipeline_events(
@transport.event_handler("on_client_connected")
async def on_client_connected(_transport, _client):
if vision_enabled:
try:
vision_state["client_id"] = get_transport_client_id(
_transport,
_client,
)
await maybe_capture_participant_camera(_transport, _client)
logger.info(
f"Realtime 视觉理解已接入视频客户端: "
f"{vision_state['client_id']}"
)
except Exception as exc: # noqa: BLE001 - video remains optional
logger.warning(f"Realtime 视觉理解摄像头捕获初始化失败: {exc}")
await brain.on_connected(greeting_pending=bool(greeting))
await brain.on_client_ready()
if greeting: