wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support image inputs in assistant tests

Browse Source
This commit is contained in:
Xin Wang
2026-08-23 23:46:18 +08:00
parent 0966f382b6
commit 9e4dfbe596
26 changed files with 1010 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/test_runs/evaluator.py
+2
View File
@@ -161,6 +161,7 @@ class EvaluationEngine:
id=expected_turn.id,
index=index,
user_input=expected_turn.user_input,
image=expected_turn.image,
assistant_reply="",
tool_calls=[],
runtime_records=[],
@@ -174,6 +175,7 @@ class EvaluationEngine:
id=raw.id,
index=index,
user_input=raw.user_input,
image=raw.image,
assistant_reply=raw.assistant_reply,
tool_calls=[item.as_result() for item in raw.tool_calls],
runtime_records=list(raw.runtime_records),
backend/services/test_runs/orchestrator.py
+20
View File
@@ -14,6 +14,7 @@ from db.session import SessionLocal
from models import RuntimeModelResource
from services.config_resolver import resolve_runtime_config
from services.runtime_variables import prepare_dynamic_config
from services.test_case_assets import load_test_asset_bytes
from services.test_runs.errors import TestExecutionError
from services.test_runs.evaluator import EvaluationEngine
from services.test_runs.text_runner import (
@@ -178,6 +179,19 @@ class TestRunOrchestrator:
snapshot = dict(row.case_snapshot or {})
definition = TestCaseDefinition.model_validate(snapshot)
max_attempts = row.max_attempts
try:
image_assets = await load_test_asset_bytes(session, definition)
except Exception as exc:
await self._mark_error(
run_case_id,
TestExecutionError(
code="TEST_IMAGE_READ_FAILED",
message=str(exc) or "测试图片无法读取",
stage="pipeline",
retryable=True,
),
)
return
for attempt in range(1, max_attempts + 1):
if control.stop_event.is_set():
@@ -212,6 +226,7 @@ class TestRunOrchestrator:
attempt_cfg,
definition,
timeout_seconds=timeout_seconds,
image_assets=image_assets,
on_turn_complete=save_turn_progress,
on_startup_complete=save_startup_progress,
)
@@ -346,6 +361,11 @@ class TestRunOrchestrator:
"id": turn.id,
"index": turn.index,
"userInput": turn.user_input,
"image": (
turn.image.model_dump(mode="json", by_alias=True)
if turn.image is not None
else None
),
"assistantReply": turn.assistant_reply,
"toolCalls": [item.as_result() for item in turn.tool_calls],
"runtimeRecords": list(turn.runtime_records),
backend/services/test_runs/text_runner.py
+83 -15
View File
@@ -55,7 +55,8 @@ from services.pipecat.service_factory import config_with_resource
from services.pipecat.workflow_services import build_workflow_llm_switcher
from services.test_runs.errors import TestExecutionError
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition
from services.vision import build_image_user_message
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef
IDLE_SETTLE_SECONDS = 0.2
@@ -104,6 +105,7 @@ class RawTurnResult:
user_input: str
assistant_reply: str
tool_calls: list[RawToolCall]
image: TestImageRef | None = None
runtime_records: list[dict[str, Any]] = field(default_factory=list)
@@ -359,6 +361,31 @@ def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
return messages
def _transcript_content(message: dict[str, Any]) -> str:
"""Keep evaluator transcripts readable without persisting image data URIs."""
content = message.get("content")
if isinstance(content, str):
return content.strip()
if not isinstance(content, list):
return ""
text_parts = [
str(part.get("text") or "").strip()
for part in content
if isinstance(part, dict)
and part.get("type") in {"text", "input_text"}
and str(part.get("text") or "").strip()
]
has_image = any(
isinstance(part, dict)
and part.get("type") in {"image_url", "input_image"}
for part in content
)
if has_image:
text_parts.append("[用户提交了一张测试图片]")
return "\n".join(text_parts).strip()
async def _wait_for_quiescence(
capture: _TextCaptureProcessor,
runner_task: asyncio.Task[None],
@@ -423,6 +450,7 @@ class TextPipelineRunner:
definition: TestCaseDefinition,
*,
timeout_seconds: int,
image_assets: dict[str, bytes] | None = None,
on_turn_complete: TurnCompleteCallback | None = None,
on_startup_complete: StartupCompleteCallback | None = None,
) -> TextCaseResult:
@@ -431,6 +459,7 @@ class TextPipelineRunner:
return await self._run(
cfg,
definition,
image_assets=image_assets or {},
on_turn_complete=on_turn_complete,
on_startup_complete=on_startup_complete,
)
@@ -447,6 +476,7 @@ class TextPipelineRunner:
cfg: AssistantConfig,
definition: TestCaseDefinition,
*,
image_assets: dict[str, bytes],
on_turn_complete: TurnCompleteCallback | None,
on_startup_complete: StartupCompleteCallback | None,
) -> TextCaseResult:
@@ -545,6 +575,7 @@ class TextPipelineRunner:
executor_holder: dict[str, MockToolExecutor] = {}
text_client_tools = TextClientToolPort()
workflow_vision_scope: dict[str, Any] = {"enabled": False}
def create_test_executor(store) -> MockToolExecutor:
executor = MockToolExecutor(store)
@@ -699,7 +730,7 @@ class TextPipelineRunner:
transport=None,
switch_services=switch_services,
set_knowledge_scope=knowledge.set_scope,
set_vision_scope=lambda _scope: None,
set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
vision_function=None,
set_input_enabled=lambda _enabled: None,
apply_turn_config=lambda _enabled, _config: asyncio.sleep(0),
@@ -809,26 +840,32 @@ class TextPipelineRunner:
retryable=False,
)
result = await self._run_turn(
cfg,
worker,
runner_task,
capture,
executor,
turn,
index,
image_assets=image_assets,
workflow_vision_scope=workflow_vision_scope,
)
results.append(result)
if on_turn_complete is not None:
await on_turn_complete(result)
transcript = [
{
"role": str(message.get("role") or ""),
"content": str(message.get("content") or ""),
}
for message in context.get_messages()
if message.get("role") in {"user", "assistant"}
and str(message.get("content") or "").strip()
]
transcript = []
for message in context.get_messages():
if message.get("role") not in {"user", "assistant"}:
continue
content = _transcript_content(message)
if content:
transcript.append(
{
"role": str(message.get("role") or ""),
"content": content,
}
)
return TextCaseResult(
turns=results,
transcript=transcript,
@@ -853,12 +890,16 @@ class TextPipelineRunner:
async def _run_turn(
self,
cfg: AssistantConfig,
worker: PipelineWorker,
runner_task: asyncio.Task[None],
capture: _TextCaptureProcessor,
executor: MockToolExecutor,
turn: FixedInputTurn,
index: int,
*,
image_assets: dict[str, bytes],
workflow_vision_scope: dict[str, Any],
) -> RawTurnResult:
tool_behaviors = [
behavior for behavior in turn.behaviors if behavior.type == "tool_call"
@@ -868,11 +909,37 @@ class TextPipelineRunner:
output_marker = len(capture.outputs)
tool_marker = len(capture.tool_calls)
runtime_record_marker = len(capture.runtime_records)
user_message: dict[str, Any] = {
"role": "user",
"content": turn.user_input,
}
if turn.image is not None:
image_bytes = image_assets.get(turn.image.asset_id)
if image_bytes is None:
raise TestExecutionError(
code="TEST_IMAGE_MISSING",
message=f"测试图片不存在: {turn.image.asset_id}",
stage="pipeline",
retryable=False,
)
try:
user_message = await build_image_user_message(
cfg,
image_bytes,
turn.user_input,
workflow_scope=(
workflow_vision_scope if cfg.type == "workflow" else None
),
)
except ValueError as exc:
raise TestExecutionError(
code="TEST_IMAGE_UNSUPPORTED",
message=str(exc),
stage="model",
retryable=False,
) from exc
await worker.queue_frame(
LLMMessagesAppendFrame(
messages=[{"role": "user", "content": turn.user_input}],
run_llm=True,
)
LLMMessagesAppendFrame(messages=[user_message], run_llm=True)
)
await _wait_for_quiescence(capture, runner_task, activity_marker)
if executor.unmocked_calls:
@@ -886,6 +953,7 @@ class TextPipelineRunner:
id=turn.id,
index=index,
user_input=turn.user_input,
image=turn.image,
assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
tool_calls=list(capture.tool_calls[tool_marker:]),
runtime_records=list(