Xin Wang
7560f9a525
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types. - Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages. - Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs. - Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases. - Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
90 lines
3.1 KiB
Python
90 lines
3.1 KiB
Python
"""Test-only tool executor that never reaches external side effects."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from typing import Any
|
|
|
|
from models import RuntimeTool
|
|
from services.runtime_variables import DynamicVariableStore
|
|
from services.tool_executor import ToolExecutionError, ToolExecutor
|
|
from test_schemas import ToolCallExpectedBehavior
|
|
|
|
|
|
class MockToolExecutor(ToolExecutor):
|
|
"""Resolve the active turn's configured mocks and reject everything else."""
|
|
|
|
def __init__(self, store: DynamicVariableStore):
|
|
super().__init__(store)
|
|
self._mocks_by_tool_id: dict[str, ToolCallExpectedBehavior] = {}
|
|
self._mocks_by_function: dict[str, ToolCallExpectedBehavior] = {}
|
|
self.unmocked_calls: list[str] = []
|
|
|
|
def set_turn_behaviors(
|
|
self,
|
|
behaviors: list[ToolCallExpectedBehavior],
|
|
) -> None:
|
|
self._mocks_by_tool_id = {item.tool_id: item for item in behaviors}
|
|
self._mocks_by_function = {item.function_name: item for item in behaviors}
|
|
self.unmocked_calls = []
|
|
|
|
async def execute(
|
|
self,
|
|
tool: RuntimeTool,
|
|
arguments: dict[str, Any] | None = None,
|
|
*,
|
|
result_assignments: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
self.register_secrets(tool)
|
|
behavior = self._mocks_by_tool_id.get(tool.id) or self._mocks_by_function.get(
|
|
tool.function_name
|
|
)
|
|
if behavior is None:
|
|
self.unmocked_calls.append(tool.function_name)
|
|
raise ToolExecutionError(
|
|
f"UNMOCKED_TOOL_CALL: 工具 {tool.function_name} 未配置 Mock 返回值"
|
|
)
|
|
|
|
delay_seconds = behavior.mock_response.delay_ms / 1000
|
|
if delay_seconds:
|
|
await asyncio.sleep(delay_seconds)
|
|
|
|
payload = json.loads(behavior.mock_response.body)
|
|
status = "ok" if behavior.mock_response.outcome == "success" else "error"
|
|
if isinstance(payload, dict):
|
|
result: dict[str, Any] = dict(payload)
|
|
result["status"] = status
|
|
else:
|
|
result = {"status": status, "data": payload}
|
|
if result.get("status") != "ok":
|
|
return {**result, "updated_variables": []}
|
|
return self._apply_result_assignments(
|
|
tool,
|
|
result,
|
|
result_assignments=result_assignments,
|
|
)
|
|
|
|
|
|
class TextClientToolPort:
|
|
"""Automatically acknowledge built-in message stages in text tests."""
|
|
|
|
def __init__(self) -> None:
|
|
self.auto_confirmed_messages: list[dict[str, Any]] = []
|
|
|
|
async def call(
|
|
self,
|
|
function_name: str,
|
|
arguments: dict[str, Any],
|
|
**_kwargs: Any,
|
|
) -> dict[str, Any]:
|
|
if function_name == "show_message":
|
|
self.auto_confirmed_messages.append(dict(arguments))
|
|
return {
|
|
"status": "ok",
|
|
"data": {"action": "confirmed", "arguments": arguments},
|
|
}
|
|
raise ToolExecutionError(
|
|
f"UNMOCKED_CLIENT_TOOL: 客户端工具 {function_name} 不能脱离 Mock 执行"
|
|
)
|