112 lines
3.5 KiB
Python
112 lines
3.5 KiB
Python
|
import unittest
|
||
|
|
from unittest.mock import patch
|
||
|
|
|
||
|
|
from models import RuntimeModelResource
|
||
|
|
from services.test_runs.errors import TestExecutionError as ExecutionError
|
||
|
|
from services.test_runs.evaluator import LLMJudge
|
||
|
|
|
||
|
|
|
||
|
|
def evaluator_resource() -> RuntimeModelResource:
|
||
|
|
return RuntimeModelResource(
|
||
|
|
id="model_eval",
|
||
|
|
name="测试评估模型",
|
||
|
|
capability="LLM",
|
||
|
|
interface_type="openai-llm",
|
||
|
|
values={
|
||
|
|
"apiUrl": "https://model.example/v1",
|
||
|
|
"modelId": "judge-model",
|
||
|
|
},
|
||
|
|
secrets={"apiKey": "model-secret"},
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class FakeResponse:
|
||
|
|
def __init__(self, content: str):
|
||
|
|
self._content = content
|
||
|
|
|
||
|
|
def raise_for_status(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def json(self):
|
||
|
|
return {"choices": [{"message": {"content": self._content}}]}
|
||
|
|
|
||
|
|
|
||
|
|
class FakeClient:
|
||
|
|
def __init__(self, responses: list[FakeResponse]):
|
||
|
|
self.responses = list(responses)
|
||
|
|
self.requests: list[dict] = []
|
||
|
|
|
||
|
|
async def __aenter__(self):
|
||
|
|
return self
|
||
|
|
|
||
|
|
async def __aexit__(self, exc_type, exc, traceback):
|
||
|
|
return False
|
||
|
|
|
||
|
|
async def post(self, _url, *, headers, json):
|
||
|
|
self.requests.append({"headers": headers, "json": json})
|
||
|
|
return self.responses.pop(0)
|
||
|
|
|
||
|
|
|
||
|
|
class LLMJudgeJsonTests(unittest.TestCase):
|
||
|
|
def test_parser_uses_first_complete_object_when_model_repeats_json(self):
|
||
|
|
parsed = LLMJudge._parse_json(
|
||
|
|
'{"passed":false,"reason":"没有调用工具","actual":"调用 0 次"}'
|
||
|
|
'{"passed":true,"reason":"重复内容","actual":""}'
|
||
|
|
)
|
||
|
|
|
||
|
|
self.assertFalse(parsed["passed"])
|
||
|
|
self.assertEqual(parsed["reason"], "没有调用工具")
|
||
|
|
|
||
|
|
|
||
|
|
class LLMJudgeRequestTests(unittest.IsolatedAsyncioTestCase):
|
||
|
|
async def test_invalid_response_is_retried_once(self):
|
||
|
|
client = FakeClient(
|
||
|
|
[
|
||
|
|
FakeResponse("not json"),
|
||
|
|
FakeResponse(
|
||
|
|
'{"passed":false,"reason":"缺少调用","actual":"调用 0 次"}'
|
||
|
|
),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
judge = LLMJudge(evaluator_resource())
|
||
|
|
|
||
|
|
with patch(
|
||
|
|
"services.test_runs.evaluator.httpx.AsyncClient",
|
||
|
|
return_value=client,
|
||
|
|
):
|
||
|
|
passed, reason, actual = await judge.evaluate(
|
||
|
|
criteria="必须调用工具",
|
||
|
|
subject="当前轮助手回复",
|
||
|
|
context="（没有生成文字回复）",
|
||
|
|
)
|
||
|
|
|
||
|
|
self.assertFalse(passed)
|
||
|
|
self.assertEqual(reason, "缺少调用")
|
||
|
|
self.assertEqual(actual, "调用 0 次")
|
||
|
|
self.assertEqual(len(client.requests), 2)
|
||
|
|
retry_prompt = client.requests[1]["json"]["messages"][0]["content"]
|
||
|
|
self.assertIn("上一次响应无法解析", retry_prompt)
|
||
|
|
|
||
|
|
async def test_second_invalid_response_keeps_structured_error(self):
|
||
|
|
client = FakeClient([FakeResponse("bad"), FakeResponse("still bad")])
|
||
|
|
judge = LLMJudge(evaluator_resource())
|
||
|
|
|
||
|
|
with patch(
|
||
|
|
"services.test_runs.evaluator.httpx.AsyncClient",
|
||
|
|
return_value=client,
|
||
|
|
):
|
||
|
|
with self.assertRaises(ExecutionError) as raised:
|
||
|
|
await judge.evaluate(
|
||
|
|
criteria="必须调用工具",
|
||
|
|
subject="当前轮助手回复",
|
||
|
|
context="没有调用",
|
||
|
|
)
|
||
|
|
|
||
|
|
self.assertEqual(raised.exception.code, "EVALUATOR_REQUEST_FAILED")
|
||
|
|
self.assertTrue(raised.exception.retryable)
|
||
|
|
self.assertEqual(len(client.requests), 2)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|