feat: add stateful responses API
This commit is contained in:
Xin Wang
@@ -0,0 +1,358 @@
|
||||
"""Public stateful OpenAI Responses-compatible text endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import settings
|
||||
from db.session import SessionLocal
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from loguru import logger
|
||||
from services.openai_realtime.auth import (
|
||||
RealtimeAuthError,
|
||||
RealtimeCredential,
|
||||
authenticate_bearer,
|
||||
bearer_from_authorization,
|
||||
)
|
||||
from services.openai_responses.protocol import (
|
||||
ResponseRequest,
|
||||
ResponsesAPIError,
|
||||
error_payload,
|
||||
new_id,
|
||||
parse_request,
|
||||
response_payload,
|
||||
)
|
||||
from services.openai_responses.runner import ResponseRunError, ResponsesTextRunner
|
||||
from services.openai_responses.state import (
|
||||
PreparedResponse,
|
||||
finish_response,
|
||||
get_response,
|
||||
prepare_response,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter(prefix="/v1/responses", tags=["openai-responses"])
|
||||
|
||||
|
||||
async def _credential(request: Request) -> RealtimeCredential:
|
||||
token = bearer_from_authorization(request.headers.get("authorization"))
|
||||
async with SessionLocal() as db:
|
||||
credential = await authenticate_bearer(db, token)
|
||||
if credential.ephemeral:
|
||||
raise RealtimeAuthError("A long-lived API key is required")
|
||||
return credential
|
||||
|
||||
|
||||
def _json_error(error: ResponsesAPIError) -> JSONResponse:
|
||||
return JSONResponse(status_code=error.status_code, content=error_payload(error))
|
||||
|
||||
|
||||
def _sse(event: dict[str, Any]) -> bytes:
|
||||
return (
|
||||
f"event: {event['type']}\n"
|
||||
f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _text_item(item_id: str, text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": item_id,
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": text,
|
||||
"annotations": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _output_items(
|
||||
*,
|
||||
text: str,
|
||||
text_item_id: str,
|
||||
function_calls: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
if text:
|
||||
items.append(_text_item(text_item_id, text))
|
||||
items.extend(function_calls)
|
||||
return items
|
||||
|
||||
|
||||
async def _run_prepared(
|
||||
prepared: PreparedResponse,
|
||||
*,
|
||||
on_delta=None,
|
||||
text_item_id: str,
|
||||
):
|
||||
try:
|
||||
async with asyncio.timeout(settings.RESPONSES_REQUEST_TIMEOUT_SECONDS):
|
||||
result = await ResponsesTextRunner().run(
|
||||
prepared.config,
|
||||
history_messages=prepared.history_messages,
|
||||
current_messages=prepared.current_messages,
|
||||
runtime_state=prepared.runtime_state,
|
||||
client_tool_results=prepared.client_tool_results,
|
||||
on_delta=on_delta,
|
||||
)
|
||||
return await finish_response(
|
||||
prepared.row.id,
|
||||
status="completed",
|
||||
output_items=_output_items(
|
||||
text=result.text,
|
||||
text_item_id=text_item_id,
|
||||
function_calls=result.function_calls,
|
||||
),
|
||||
context_messages=result.context_messages,
|
||||
dynamic_variables=result.dynamic_variables,
|
||||
runtime_state=result.runtime_state,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
await finish_response(
|
||||
prepared.row.id,
|
||||
status="failed",
|
||||
output_items=[],
|
||||
error={"code": "response_timeout", "message": "Response timed out"},
|
||||
)
|
||||
raise ResponsesAPIError(
|
||||
"Response timed out", code="response_timeout", status_code=504
|
||||
) from exc
|
||||
except asyncio.CancelledError:
|
||||
await finish_response(
|
||||
prepared.row.id,
|
||||
status="cancelled",
|
||||
output_items=[],
|
||||
error={"code": "client_disconnected", "message": "Client disconnected"},
|
||||
)
|
||||
raise
|
||||
except ResponseRunError as exc:
|
||||
await finish_response(
|
||||
prepared.row.id,
|
||||
status="failed",
|
||||
output_items=[],
|
||||
error={"code": "response_failed", "message": str(exc)},
|
||||
)
|
||||
raise ResponsesAPIError(
|
||||
"Response generation failed",
|
||||
code="response_failed",
|
||||
status_code=500,
|
||||
) from exc
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception(f"Responses generation failed: {exc}")
|
||||
await finish_response(
|
||||
prepared.row.id,
|
||||
status="failed",
|
||||
output_items=[],
|
||||
error={"code": "internal_error", "message": "Response generation failed"},
|
||||
)
|
||||
raise ResponsesAPIError(
|
||||
"Response generation failed", code="internal_error", status_code=500
|
||||
) from exc
|
||||
|
||||
|
||||
async def _stream_response(
|
||||
prepared: PreparedResponse,
|
||||
*,
|
||||
text_item_id: str,
|
||||
) -> AsyncIterator[bytes]:
|
||||
queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
|
||||
text_started = False
|
||||
|
||||
async def on_delta(delta: str) -> None:
|
||||
await queue.put({"kind": "delta", "delta": delta})
|
||||
|
||||
async def generate() -> None:
|
||||
try:
|
||||
row = await _run_prepared(
|
||||
prepared,
|
||||
on_delta=on_delta,
|
||||
text_item_id=text_item_id,
|
||||
)
|
||||
await queue.put({"kind": "completed", "row": row})
|
||||
except ResponsesAPIError as exc:
|
||||
await queue.put({"kind": "failed", "error": exc})
|
||||
finally:
|
||||
await queue.put(None)
|
||||
|
||||
task = asyncio.create_task(generate(), name=f"stream-{prepared.row.id}")
|
||||
try:
|
||||
created = response_payload(prepared.row)
|
||||
yield _sse({"type": "response.created", "response": created})
|
||||
yield _sse({"type": "response.in_progress", "response": created})
|
||||
while True:
|
||||
event = await queue.get()
|
||||
if event is None:
|
||||
break
|
||||
if event["kind"] == "delta":
|
||||
if not text_started:
|
||||
text_started = True
|
||||
pending_item = _text_item(text_item_id, "")
|
||||
pending_item["status"] = "in_progress"
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": pending_item,
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.content_part.added",
|
||||
"item_id": text_item_id,
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"part": {
|
||||
"type": "output_text",
|
||||
"text": "",
|
||||
"annotations": [],
|
||||
},
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"item_id": text_item_id,
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": event["delta"],
|
||||
}
|
||||
)
|
||||
continue
|
||||
if event["kind"] == "failed":
|
||||
error: ResponsesAPIError = event["error"]
|
||||
yield _sse({"type": "error", **error_payload(error)})
|
||||
failed = await get_response(
|
||||
prepared.row.id, api_key_id=prepared.row.api_key_id
|
||||
)
|
||||
yield _sse(
|
||||
{"type": "response.failed", "response": response_payload(failed)}
|
||||
)
|
||||
continue
|
||||
|
||||
row = event["row"]
|
||||
payload = response_payload(row)
|
||||
output_index = 0
|
||||
for item in row.output_items or []:
|
||||
if item.get("type") == "message":
|
||||
text = str((item.get("content") or [{}])[0].get("text") or "")
|
||||
if not text_started:
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": output_index,
|
||||
"item": {
|
||||
**item,
|
||||
"status": "in_progress",
|
||||
"content": [],
|
||||
},
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.content_part.added",
|
||||
"item_id": item["id"],
|
||||
"output_index": output_index,
|
||||
"content_index": 0,
|
||||
"part": {
|
||||
"type": "output_text",
|
||||
"text": "",
|
||||
"annotations": [],
|
||||
},
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_text.done",
|
||||
"item_id": item["id"],
|
||||
"output_index": output_index,
|
||||
"content_index": 0,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.content_part.done",
|
||||
"item_id": item["id"],
|
||||
"output_index": output_index,
|
||||
"content_index": 0,
|
||||
"part": item["content"][0],
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": output_index,
|
||||
"item": item,
|
||||
}
|
||||
)
|
||||
else:
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": output_index,
|
||||
"item": item,
|
||||
}
|
||||
)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": output_index,
|
||||
"item": item,
|
||||
}
|
||||
)
|
||||
output_index += 1
|
||||
yield _sse({"type": "response.completed", "response": payload})
|
||||
finally:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_response(request: Request):
|
||||
try:
|
||||
credential = await _credential(request)
|
||||
body = await request.json()
|
||||
parsed: ResponseRequest = parse_request(body)
|
||||
prepared = await prepare_response(parsed, api_key_id=credential.api_key_id)
|
||||
text_item_id = new_id("msg")
|
||||
if parsed.stream:
|
||||
return StreamingResponse(
|
||||
_stream_response(prepared, text_item_id=text_item_id),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
row = await _run_prepared(prepared, text_item_id=text_item_id)
|
||||
return response_payload(row)
|
||||
except RealtimeAuthError as exc:
|
||||
return _json_error(
|
||||
ResponsesAPIError(str(exc), code="invalid_api_key", status_code=401)
|
||||
)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
return _json_error(ResponsesAPIError("Request body must be valid JSON"))
|
||||
except ResponsesAPIError as exc:
|
||||
return _json_error(exc)
|
||||
except ValueError as exc:
|
||||
return _json_error(ResponsesAPIError(str(exc)))
|
||||
|
||||
|
||||
@router.get("/{response_id}")
|
||||
async def retrieve_response(response_id: str, request: Request):
|
||||
try:
|
||||
credential = await _credential(request)
|
||||
row = await get_response(response_id, api_key_id=credential.api_key_id)
|
||||
return response_payload(row)
|
||||
except RealtimeAuthError as exc:
|
||||
return _json_error(
|
||||
ResponsesAPIError(str(exc), code="invalid_api_key", status_code=401)
|
||||
)
|
||||
except ResponsesAPIError as exc:
|
||||
return _json_error(exc)
|
||||
Reference in New Issue
Block a user