fix: harden realtime pipeline lifecycle

This commit is contained in:
Xin Wang
2026-09-07 08:27:52 +08:00
parent 43f2cc445e
commit 8025ed8aa0
12 changed files with 445 additions and 42 deletions
@@ -8,12 +8,16 @@ from unittest.mock import AsyncMock, patch
from fastapi import HTTPException
from routes import voice_webrtc
from services.realtime import lifecycle
from starlette.requests import Request
class FakeConnection:
pc_id = "pc_test"
def __init__(self):
self.disconnect = AsyncMock()
def encoded_header_json(value: object) -> str:
payload = json.dumps(value).encode("utf-8")
@@ -81,6 +85,16 @@ class WebRTCOfferMetadataTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(raised.exception.status_code, 400)
async def test_invalid_assistant_returns_actionable_bad_request(self):
with patch.object(
voice_webrtc, "_handle_offer_payload",
new=AsyncMock(side_effect=ValueError("助手不存在: missing")),
):
with self.assertRaises(HTTPException) as raised:
await voice_webrtc.http_offer(offer_request(), {"type": "offer"})
self.assertEqual(raised.exception.status_code, 400)
self.assertIn("助手不存在", raised.exception.detail)
class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase):
async def asyncTearDown(self):
@@ -89,6 +103,9 @@ class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase):
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
await asyncio.sleep(0)
if lifecycle.connection_cleanup_tasks:
await asyncio.gather(*list(lifecycle.connection_cleanup_tasks))
voice_webrtc._pipeline_tasks.clear()
voice_webrtc._connection_tasks.clear()
voice_webrtc._active_connections.clear()
@@ -111,6 +128,54 @@ class WebRTCPipelineLifecycleTest(unittest.IsolatedAsyncioTestCase):
self.assertNotIn(task, voice_webrtc._pipeline_tasks)
self.assertNotIn(connection, voice_webrtc._connection_tasks)
async def test_failed_startup_disconnects_and_releases_connection(self):
connection = FakeConnection()
async def pipeline():
raise ValueError("provider construction failed")
task = voice_webrtc._start_pipeline_task(connection, pipeline())
with self.assertRaises(ValueError):
await task
await asyncio.sleep(0)
if lifecycle.connection_cleanup_tasks:
await asyncio.gather(*list(lifecycle.connection_cleanup_tasks))
connection.disconnect.assert_awaited_once()
self.assertNotIn(connection, voice_webrtc._active_connections)
self.assertNotIn(connection, voice_webrtc._connection_tasks)
async def test_cleanup_closed_handler_can_wait_for_completed_pipeline(self):
connection = FakeConnection()
async def pipeline():
return None
task = voice_webrtc._start_pipeline_task(connection, pipeline())
async def disconnect():
await lifecycle.wait_for_pipeline_close(
task, connection_id=connection.pc_id, timeout=0.01,
)
connection.disconnect.side_effect = disconnect
await task
await asyncio.sleep(0)
if lifecycle.connection_cleanup_tasks:
await asyncio.wait_for(
asyncio.gather(*list(lifecycle.connection_cleanup_tasks)), 0.5,
)
connection.disconnect.assert_awaited_once()
async def test_already_closed_connection_is_not_disconnected_twice(self):
connection = FakeConnection()
release = asyncio.Event()
task = voice_webrtc._start_pipeline_task(connection, release.wait())
voice_webrtc._active_connections.discard(connection)
release.set()
await task
await asyncio.sleep(0)
connection.disconnect.assert_not_awaited()
async def test_stuck_pipeline_is_cancelled_after_close_grace_period(self):
connection = FakeConnection()
cancelled = asyncio.Event()