fix(api): report checkpoint deletion failures and classify SSE errors

Delete checkpoints before removing business fields. Suppress only an
explicit CheckpointNotFoundError; propagate storage failures and
cancellation so the endpoint cannot report false success. Prefer async
checkpoint deletion and fail clearly when cleanup is unsupported.

Classify SSE error events as 401 for authentication, 429 for rate limits,
and 500 for other failures, using the same fixed messages as non-stream
responses. Errors terminate the stream without a success done event or
provider exception text in the response.

Validation: 286 offline tests passed (19 added), including failed-delete
HTTP retry with preserved business/checkpoint state, explicit missing
threads, cancellation, missing cleanup capability, and HTTP SSE errors
before and after text output.
This commit is contained in:
Eric Wang
2026-10-23 19:48:29 +08:00
parent 24e2cfbe10
commit 3caf17494f
9 changed files with 260 additions and 15 deletions
+40
View File
@@ -104,3 +104,43 @@ def test_assignment6_acceptance_flow(client):
"text": "新对话", "clientMode": "direct",
})
assert response.json()["code"] == "200"
def test_delete_checkpoint_failure_returns_error_and_retry_cleans_session(client, monkeypatch):
from unittest.mock import AsyncMock
from src.core import fastgpt_client as wiring
session = "delete-retry"
payload = {"sessionId": session, "timeStamp": TS}
assert client.post("/set_info", json={
**payload, "key": "hphm1", "value": "test-plate",
}).json()["code"] == "200"
assert client.post("/chat", json={
**payload, "text": "新对话",
}).json()["nextStageCode"] == "1001"
saver = wiring._langgraph_graph.checkpointer
config = {"configurable": {"thread_id": session}}
assert saver.get_tuple(config) is not None
original_delete = saver.adelete_thread
failing_delete = AsyncMock(side_effect=ConnectionError("private storage detail"))
monkeypatch.setattr(saver, "adelete_thread", failing_delete)
response = client.request("DELETE", "/delete_session", json=payload)
assert response.json()["code"] == "500"
assert response.json()["msg"] == "删除会话失败"
assert "private storage detail" not in response.text
failing_delete.assert_awaited_once_with(session)
assert saver.get_tuple(config) is not None
assert json.loads(client.post("/get_info", json={
**payload, "key": "hphm1",
}).json()["value"]) == "test-plate"
monkeypatch.setattr(saver, "adelete_thread", original_delete)
assert client.request("DELETE", "/delete_session", json=payload).json()["code"] == "200"
assert saver.get_tuple(config) is None
assert json.loads(client.post("/get_info", json={
**payload, "key": "hphm1",
}).json()["value"]) == ""
assert client.post("/chat", json={
**payload, "text": "新对话",
}).json()["nextStageCode"] == "1001"
+76
View File
@@ -0,0 +1,76 @@
import json
import httpx
import pytest
from fastapi import FastAPI
from src.agent.collect import validated_reply
from src.agent.llm import LLMAPIError, LLMAuthError, LLMRateLimitError
from src.api.endpoints import router
from src.backends.langgraph import LangGraphBackend
from src.core.config import Config
from src.core.fastgpt_client import get_chat_backend
class FailingLLM:
def __init__(self, error_type, partial):
self.error_type = error_type
self.partial = partial
async def complete(self, *args, **kwargs):
raise self.error_type('private provider detail')
async def stream_tokens(self, *args, **kwargs):
if self.partial:
yield '<state>1002</state>'
yield 'partial reply'
raise self.error_type('private provider detail')
class Graph:
async def ainvoke(self, data, *, config, context):
await validated_reply(context.llm, 'system', 'user', '1002')
raise AssertionError('failed turn must not complete')
def sse_events(body):
events = []
for block in body.strip().split('\n\n'):
lines = block.splitlines()
kind = next(line[7:] for line in lines if line.startswith('event: '))
data = json.loads(next(line[6:] for line in lines if line.startswith('data: ')))
events.append((kind, data))
return events
@pytest.mark.asyncio
@pytest.mark.parametrize('partial', [False, True], ids=['before-text', 'after-text'])
@pytest.mark.parametrize('error_type,code,message', [
(LLMAuthError, '401', '认证失败'),
(LLMRateLimitError, '429', '请求过于频繁,请稍后重试'),
(LLMAPIError, '500', '大模型服务器无响应'),
])
async def test_http_sse_errors_match_non_stream_codes_and_terminate(
monkeypatch, partial, error_type, code, message,
):
# 独立协议测试 app:实际 LangGraphBackend/LLM 错误映射,不启动真实 provider。
monkeypatch.setattr(Config, 'AGENT_BACKEND', 'fastgpt')
app = FastAPI()
app.include_router(router)
backend = LangGraphBackend(graph=Graph(), llm=FailingLLM(error_type, partial))
app.dependency_overrides[get_chat_backend] = lambda: backend
payload = {'sessionId': 'error-audit', 'timeStamp': '20261022120000', 'text': 'hello'}
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url='http://test') as client:
response = await client.post('/chat?stream=true', json=payload)
assert response.status_code == 200 # 错误分类在 SSE data.code,不改已发送的 HTTP 头
assert response.headers['content-type'].startswith('text/event-stream')
events = sse_events(response.text)
assert events[-1] == ('error', {'code': code, 'msg': message})
assert sum(kind == 'error' for kind, _ in events) == 1
assert all(kind != 'done' for kind, _ in events)
assert any(kind == 'text_delta' for kind, _ in events) is partial
assert 'private provider detail' not in response.text
result = await client.post('/chat', json=payload)
assert result.json()['code'] == code
assert result.json()['msg'] == message
+3 -1
View File
@@ -211,8 +211,10 @@ async def test_delete_clears_business_state_and_is_idempotent():
@pytest.mark.asyncio
async def test_delete_tolerates_missing_checkpoint():
from src.services.repository import CheckpointNotFoundError
async def missing(_session_id):
raise RuntimeError("no such thread")
raise CheckpointNotFoundError("no such thread")
service, _ = make_service()
await service.set_info("s1", "xm1", "王五")
@@ -0,0 +1,99 @@
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from src.core import fastgpt_client as wiring
from src.services.app_service import AccidentAppService
from src.services.repository import CheckpointNotFoundError, InMemorySessionRepository
@pytest.mark.asyncio
@pytest.mark.parametrize('error', [
ConnectionError('storage offline'), TimeoutError('storage timeout'),
RuntimeError('no such thread'), KeyError('storage schema'),
])
async def test_checkpoint_failure_keeps_business_fields_and_allows_retry(error):
repo = InMemorySessionRepository()
service = AccidentAppService(SimpleNamespace(), repo)
await service.set_info('s', 'hphm1', 'test-plate')
drop = AsyncMock(side_effect=error)
with pytest.raises(type(error)):
await service.delete_session('s', drop)
assert await service.get_info('s', 'hphm1') == '"test-plate"'
assert not service._locks._entries
drop.side_effect = None
await service.delete_session('s', drop)
assert await service.get_info('s', 'hphm1') == '""'
@pytest.mark.asyncio
async def test_explicit_missing_checkpoint_is_idempotent():
service = AccidentAppService(SimpleNamespace(), InMemorySessionRepository())
await service.set_info('s', 'hphm1', 'test-plate')
drop = AsyncMock(side_effect=CheckpointNotFoundError())
await service.delete_session('s', drop)
await service.delete_session('s', drop)
assert await service.get_info('s', 'hphm1') == '""'
assert drop.await_count == 2
@pytest.mark.asyncio
async def test_cancelled_checkpoint_cleanup_keeps_business_fields():
service = AccidentAppService(SimpleNamespace(), InMemorySessionRepository())
await service.set_info('s', 'hphm1', 'test-plate')
started = asyncio.Event()
async def drop(_):
started.set()
await asyncio.Future()
task = asyncio.create_task(service.delete_session('s', drop))
await asyncio.wait_for(started.wait(), 1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert await service.get_info('s', 'hphm1') == '"test-plate"'
assert not service._locks._entries
@pytest.mark.asyncio
async def test_business_delete_failure_is_not_reported_as_success():
class FailingRepository(InMemorySessionRepository):
def delete(self, _):
raise ConnectionError('business storage offline')
service = AccidentAppService(SimpleNamespace(), FailingRepository())
await service.set_info('s', 'hphm1', 'test-plate')
drop = AsyncMock()
with pytest.raises(ConnectionError):
await service.delete_session('s', drop)
drop.assert_awaited_once_with('s')
assert await service.get_info('s', 'hphm1') == '"test-plate"'
@pytest.mark.asyncio
async def test_checkpoint_cleanup_uses_async_api_and_propagates_errors(monkeypatch):
saver = SimpleNamespace(adelete_thread=AsyncMock(side_effect=ConnectionError()), delete_thread=Mock())
monkeypatch.setattr(wiring, '_langgraph_graph', SimpleNamespace(checkpointer=saver))
with pytest.raises(ConnectionError):
await wiring.delete_chat_session('s')
saver.adelete_thread.assert_awaited_once_with('s')
saver.delete_thread.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize('graph', [None, SimpleNamespace(), SimpleNamespace(checkpointer=object())])
async def test_missing_cleanup_capability_fails(monkeypatch, graph):
monkeypatch.setattr(wiring, '_langgraph_graph', graph)
with pytest.raises(RuntimeError):
await wiring.delete_chat_session('s')
@pytest.mark.asyncio
async def test_sync_checkpoint_cleanup_is_supported_and_errors_propagate(monkeypatch):
saver = SimpleNamespace(delete_thread=Mock())
monkeypatch.setattr(wiring, '_langgraph_graph', SimpleNamespace(checkpointer=saver))
await wiring.delete_chat_session('s')
saver.delete_thread.assert_called_once_with('s')
saver.delete_thread.side_effect = ConnectionError()
with pytest.raises(ConnectionError):
await wiring.delete_chat_session('s')