feat(streaming): real incremental SSE - model tokens flow through graph to client (assignment 8)

Replaces the 'complete answer sliced into chunks' stopgap with token-
level streaming. Graph nodes are untouched: they still consume the
batched LLMService.complete interface and validate full replies.

- agent/llm: OpenAILLMService.stream_tokens() iterates the provider
  stream with the same message construction, temperature and extra_body
  forwarding as complete(); identical error classification (401 ->
  LLMAuthError, 429 -> LLMRateLimitError, timeout via asyncio.timeout
  around the whole iteration); deliberately no retries (tokens already
  emitted cannot be recalled)
- backends/langgraph: _TokenPassThroughLLM wraps the injected LLM for
  stream turns - dialogue calls push tokens to a queue bridge while
  accumulating the full text for node validation; calls marked with the
  【字段提取】 marker take the silent inner.complete path so extraction
  JSON never leaks into the client stream; LLMs without stream_tokens
  fall back to single-chunk push (fake/test compatibility)
- stream(): graph runs in a background task; the generator yields
  TextDelta per token and FormUpdate after the turn; graph-task errors
  re-raise with the existing ChatBackend error mapping; generator close
  (client disconnect) cancels the graph task and the in-flight LLM call
- fixed-answer turns with no LLM call (welcome) fall back to emitting
  the full response text, preserving previous behavior
- endpoint untouched: its buffering parser already handles arbitrary
  chunk boundaries (state tags split across tokens, stage-first order)

Verified: 241 unit tests (11 new streaming tests: token incrementality,
extraction bypass, FormUpdate ordering, error mapping on both paths,
disconnect cancellation, fallback chunking, SSE protocol end-to-end
with per-token text_delta events)
This commit is contained in:
Eric Wang
2026-10-23 00:20:03 +08:00
parent 20e151b097
commit 952bffe6f0
4 changed files with 522 additions and 24 deletions
+118
View File
@@ -0,0 +1,118 @@
"""OpenAILLMService.stream_tokens 测试(模型侧流式)。
mock 客户端返回 token 事件流;验证消息构造与 complete 一致、
错误分类(401/429/超时/连接)与不重试语义。
"""
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import httpx
import pytest
from openai import AuthenticationError, RateLimitError
from src.agent.llm import (
LLMAPIError,
LLMAuthError,
LLMRateLimitError,
LLMTimeoutError,
OpenAILLMService,
)
def _event(content):
return SimpleNamespace(
choices=[SimpleNamespace(delta=SimpleNamespace(content=content))]
)
def _service_with_stream(events):
llm = OpenAILLMService(
model="test-model", api_key="test", base_url="http://llm.local/v1",
timeout_seconds=5, temperature=0, extra_body={"reasoning_effort": "none"},
)
class _Stream:
def __init__(self, items):
self._items = list(items)
def __aiter__(self):
return self._aiter()
async def _aiter(self):
for item in self._items:
yield item
create = AsyncMock(return_value=_Stream(events))
llm._client = SimpleNamespace(
chat=SimpleNamespace(completions=SimpleNamespace(create=create))
)
return llm, create
@pytest.mark.asyncio
async def test_stream_tokens_yields_incrementally_and_forwards_kwargs():
llm, create = _service_with_stream([_event("你"), _event(""), _event("好")])
tokens = [
token async for token in
llm.stream_tokens("系统", "输入", history=[{"role": "assistant", "content": "h"}])
]
assert tokens == ["你", "好"] # 空 delta 被跳过
kwargs = create.call_args.kwargs
assert kwargs["model"] == "test-model"
assert kwargs["stream"] is True
assert kwargs["temperature"] == 0
assert kwargs["reasoning_effort"] == "none" # extra_body 透传
assert kwargs["messages"][0] == {"role": "system", "content": "系统"}
assert kwargs["messages"][-1] == {"role": "user", "content": "输入"}
@pytest.mark.asyncio
async def test_stream_tokens_maps_authentication_error():
request = httpx.Request("POST", "http://llm.local/v1/chat/completions")
exc = AuthenticationError("bad key", response=httpx.Response(401, request=request), body=None)
llm, create = _service_with_stream([])
create.side_effect = exc
with pytest.raises(LLMAuthError):
async for _ in llm.stream_tokens("系统", "输入"):
pass
@pytest.mark.asyncio
async def test_stream_tokens_maps_rate_limit_error():
request = httpx.Request("POST", "http://llm.local/v1/chat/completions")
exc = RateLimitError("slow down", response=httpx.Response(429, request=request), body=None)
llm, create = _service_with_stream([])
create.side_effect = exc
with pytest.raises(LLMRateLimitError):
async for _ in llm.stream_tokens("系统", "输入"):
pass
@pytest.mark.asyncio
async def test_stream_tokens_times_out_on_stale_stream():
"""末 token 迟到同样算超时(asyncio.timeout 包整个迭代)。"""
class _HangingStream:
def __aiter__(self):
return self._gen()
async def _gen(self):
yield _event("你")
await asyncio.sleep(30) # 超过 timeout_seconds=5
yield _event("好")
llm = OpenAILLMService(model="m", api_key="k", timeout_seconds=0.2)
create = AsyncMock(return_value=_HangingStream())
llm._client = SimpleNamespace(
chat=SimpleNamespace(completions=SimpleNamespace(create=create))
)
tokens = []
with pytest.raises(LLMTimeoutError):
async for token in llm.stream_tokens("系统", "输入"):
tokens.append(token)
assert tokens == ["你"] # 超时前已流出的 token 保留