feat(testing): add fixed tool simulations

2026-08-26 09:36:37 +08:00
parent 26582bfb87
commit 239b9027ca
16 changed files with 1525 additions and 237 deletions
@@ -77,7 +77,7 @@ class ToolParamAssertion(CamelModel):
        return self


-class ToolMockResponse(CamelModel):
+class ToolSimulationResponse(CamelModel):
    outcome: Literal["success", "error"] = "success"
    body: str = Field(default='{"status":"ok"}', max_length=100_000)
    delay_ms: int = Field(default=0, ge=0, le=60_000)
@@ -93,6 +93,63 @@ class ToolMockResponse(CamelModel):
        return self


+class AnyToolSimulationScope(CamelModel):
+    phase: Literal["any"] = "any"
+
+
+class StartupToolSimulationScope(CamelModel):
+    phase: Literal["startup"] = "startup"
+
+
+class TurnToolSimulationScope(CamelModel):
+    phase: Literal["turn"] = "turn"
+    turn_id: str = Field(min_length=1, max_length=128)
+
+
+ToolSimulationScope = Annotated[
+    Union[
+        AnyToolSimulationScope,
+        StartupToolSimulationScope,
+        TurnToolSimulationScope,
+    ],
+    Field(discriminator="phase"),
+]
+
+
+class ToolSimulationStep(ToolSimulationResponse):
+    id: str = Field(min_length=1, max_length=128)
+    scope: ToolSimulationScope = Field(default_factory=AnyToolSimulationScope)
+
+
+class ToolSimulationBase(CamelModel):
+    tool_id: str = Field(min_length=1, max_length=40)
+    function_name: str = Field(min_length=1, max_length=128)
+
+    @field_validator("tool_id", "function_name")
+    @classmethod
+    def strip_tool_identity(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("必须选择有效的工具")
+        return value
+
+
+class FixedToolSimulation(ToolSimulationBase):
+    mode: Literal["fixed"] = "fixed"
+    response: ToolSimulationResponse = Field(default_factory=ToolSimulationResponse)
+
+
+class SequenceToolSimulation(ToolSimulationBase):
+    mode: Literal["sequence"] = "sequence"
+    steps: list[ToolSimulationStep] = Field(min_length=1, max_length=10_000)
+
+
+ToolSimulation = Annotated[
+    Union[FixedToolSimulation, SequenceToolSimulation],
+    Field(discriminator="mode"),
+]
+
+
 class ReplyExpectedBehavior(CamelModel):
    id: str = Field(min_length=1, max_length=128)
    type: Literal["reply"] = "reply"
@@ -122,7 +179,6 @@ class ToolCallExpectedBehavior(CamelModel):
    min_calls: int = Field(default=1, ge=0, le=100)
    max_calls: int | None = Field(default=None, ge=0, le=100)
    param_assertions: list[ToolParamAssertion] = Field(default_factory=list, max_length=50)
-    mock_response: ToolMockResponse = Field(default_factory=ToolMockResponse)

    @model_validator(mode="after")
    def validate_call_expectation(self):
@@ -191,14 +247,176 @@ class OverallCriterion(CamelModel):
 class TestCaseDefinition(CamelModel):
    context_turns: list[ContextTurn] = Field(default_factory=list, max_length=100)
    turns: list[FixedInputTurn] = Field(min_length=1, max_length=100)
+    tool_simulations: list[ToolSimulation] = Field(default_factory=list, max_length=100)
    overall_criteria: list[OverallCriterion] = Field(default_factory=list, max_length=50)

+    @model_validator(mode="before")
+    @classmethod
+    def migrate_legacy_tool_mocks(cls, raw):
+        """Normalize the former per-assertion mockResponse contract on read."""
+
+        if not isinstance(raw, dict):
+            return raw
+        if "tool_simulations" in raw or "toolSimulations" in raw:
+            return raw
+
+        groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
+        for turn_index, turn in enumerate(raw.get("turns") or []):
+            if not isinstance(turn, dict):
+                continue
+            turn_id = str(turn.get("id") or "").strip()
+            for behavior_index, behavior in enumerate(turn.get("behaviors") or []):
+                if not isinstance(behavior, dict):
+                    continue
+                if behavior.get("type") != "tool_call":
+                    continue
+                if behavior.get("expectation") != "called":
+                    continue
+                mock = behavior.get("mock_response") or behavior.get("mockResponse")
+                if not isinstance(mock, dict):
+                    continue
+                tool_id = str(
+                    behavior.get("tool_id") or behavior.get("toolId") or ""
+                ).strip()
+                function_name = str(
+                    behavior.get("function_name")
+                    or behavior.get("functionName")
+                    or ""
+                ).strip()
+                if not tool_id or not function_name:
+                    continue
+                response = {
+                    "outcome": str(mock.get("outcome") or "success"),
+                    "body": str(mock.get("body") or '{"status":"ok"}'),
+                    "delayMs": int(
+                        mock.get("delay_ms")
+                        if mock.get("delay_ms") is not None
+                        else mock.get("delayMs") or 0
+                    ),
+                }
+                groups.setdefault((tool_id, function_name), []).append(
+                    {
+                        "response": response,
+                        "turnId": turn_id,
+                        "turnIndex": turn_index,
+                        "behaviorIndex": behavior_index,
+                        "minCalls": max(
+                            1,
+                            int(
+                                behavior.get("min_calls")
+                                if behavior.get("min_calls") is not None
+                                else behavior.get("minCalls") or 1
+                            ),
+                        ),
+                    }
+                )
+
+        migrated: list[dict[str, Any]] = []
+        for (tool_id, function_name), entries in groups.items():
+            response_signatures = {
+                json.dumps(entry["response"], ensure_ascii=False, sort_keys=True)
+                for entry in entries
+            }
+            if len(response_signatures) == 1:
+                migrated.append(
+                    {
+                        "toolId": tool_id,
+                        "functionName": function_name,
+                        "mode": "fixed",
+                        "response": entries[0]["response"],
+                    }
+                )
+                continue
+
+            steps: list[dict[str, Any]] = []
+            for entry in entries:
+                for repeat_index in range(entry["minCalls"]):
+                    steps.append(
+                        {
+                            "id": (
+                                f"legacy_step_{entry['turnIndex'] + 1}_"
+                                f"{entry['behaviorIndex'] + 1}_{repeat_index + 1}"
+                            ),
+                            "scope": {
+                                "phase": "turn",
+                                "turnId": entry["turnId"],
+                            },
+                            **entry["response"],
+                        }
+                    )
+            migrated.append(
+                {
+                    "toolId": tool_id,
+                    "functionName": function_name,
+                    "mode": "sequence",
+                    "steps": steps,
+                }
+            )
+
+        normalized = dict(raw)
+        normalized["toolSimulations"] = migrated
+        return normalized
+
    @model_validator(mode="after")
    def validate_has_expectation(self):
        if not any(turn.behaviors for turn in self.turns) and not self.overall_criteria:
            raise ValueError("至少配置一项有效的预期行为或整体评估标准")
        return self

+    @model_validator(mode="after")
+    def validate_tool_simulations(self):
+        tool_ids: set[str] = set()
+        step_ids: set[str] = set()
+        turn_ids = {turn.id for turn in self.turns}
+        simulations_by_tool: dict[str, ToolSimulation] = {}
+
+        for simulation in self.tool_simulations:
+            if simulation.tool_id in tool_ids:
+                raise ValueError(
+                    f"工具 {simulation.function_name} 只能配置一条工具模拟"
+                )
+            tool_ids.add(simulation.tool_id)
+            simulations_by_tool[simulation.tool_id] = simulation
+            if simulation.mode != "sequence":
+                continue
+            for step in simulation.steps:
+                if step.id in step_ids:
+                    raise ValueError(f"工具模拟步骤 ID 重复: {step.id}")
+                step_ids.add(step.id)
+                if step.scope.phase == "turn" and step.scope.turn_id not in turn_ids:
+                    raise ValueError(
+                        f"工具模拟步骤引用了不存在的轮次: {step.scope.turn_id}"
+                    )
+
+        for turn in self.turns:
+            for behavior in turn.behaviors:
+                if behavior.type != "tool_call" or behavior.expectation != "called":
+                    continue
+                simulation = simulations_by_tool.get(behavior.tool_id)
+                if simulation is None:
+                    raise ValueError(
+                        f"工具 {behavior.function_name} 配置了应调用断言，但缺少工具模拟"
+                    )
+                if simulation.function_name != behavior.function_name:
+                    raise ValueError(
+                        f"工具 {behavior.function_name} 的断言与模拟不匹配"
+                    )
+                if simulation.mode == "fixed":
+                    continue
+                covers_turn = any(
+                    step.scope.phase == "any"
+                    or (
+                        step.scope.phase == "turn"
+                        and step.scope.turn_id == turn.id
+                    )
+                    for step in simulation.steps
+                )
+                if not covers_turn:
+                    raise ValueError(
+                        f"工具 {behavior.function_name} 缺少覆盖本轮的模拟步骤"
+                    )
+        return self
+
    @model_validator(mode="after")
    def validate_tool_context_pairs(self):
        pending: list[tuple[str | None, str]] = []
@@ -275,6 +493,7 @@ class TestCaseWrite(CamelModel):
    input_mode: TestCaseInputMode = "fixed_script_text"
    context_turns: list[ContextTurn] = Field(default_factory=list, max_length=100)
    turns: list[FixedInputTurn] = Field(min_length=1, max_length=100)
+    tool_simulations: list[ToolSimulation] = Field(default_factory=list, max_length=100)
    overall_criteria: list[OverallCriterion] = Field(default_factory=list, max_length=50)

    @field_validator("name")
@@ -292,6 +511,7 @@ class TestCaseWrite(CamelModel):
        TestCaseDefinition(
            context_turns=self.context_turns,
            turns=self.turns,
+            tool_simulations=self.tool_simulations,
            overall_criteria=self.overall_criteria,
        )
        return self
@@ -300,6 +520,7 @@ class TestCaseWrite(CamelModel):
        return TestCaseDefinition(
            context_turns=self.context_turns,
            turns=self.turns,
+            tool_simulations=self.tool_simulations,
            overall_criteria=self.overall_criteria,
        )