feat: unify debug runtime timelines

This commit is contained in:
Xin Wang
2026-08-13 09:30:43 +08:00
parent 7c18cea16d
commit eb111a3890
21 changed files with 1380 additions and 533 deletions
+26
View File
@@ -231,6 +231,22 @@ class PromptBrain(BaseBrain):
speech,
opening_message,
)
await self._emit_trace(
"message_started",
phase="opening",
hasSpeech=bool(speech),
showsMessage=True,
completionPolicy=MESSAGE_CONFIRMATION,
requiresConfirmation=True,
messageTitle=message_spec.display.title,
messageContent=message_spec.display.message,
messageActions=[
{
"id": "confirmed",
"label": message_spec.display.confirm_label,
}
],
)
message_result = await self._message_stages.run(
message_spec,
speak=self._speak_opening,
@@ -242,10 +258,20 @@ class PromptBrain(BaseBrain):
release_input_on_failure=False,
)
if not message_result.succeeded:
await self._emit_trace(
"message_failed",
phase="opening",
error=message_result.error or "开场消息显示失败",
)
await self._fail_opening(
message_result.error or "开场消息显示失败"
)
return
await self._emit_trace(
"message_completed",
phase="opening",
action=message_result.action,
)
context_message = confirmation_context_message(
message_result,
source="prompt-opening",
+14
View File
@@ -1557,6 +1557,20 @@ class WorkflowBrain(BaseBrain):
showsMessage=require_confirmation,
completionPolicy=completion_policy,
requiresConfirmation=require_confirmation,
**(
{
"messageTitle": display.title,
"messageContent": display.message,
"messageActions": [
{
"id": "confirmed",
"label": display.confirm_label,
}
],
}
if display is not None
else {}
),
),
)
if result.succeeded:
+1
View File
@@ -194,6 +194,7 @@ class ClientToolBroker(FrameProcessor):
toolCallId=tool_call_id,
functionName=pending.function_name,
status=status,
internalStage=pending.function_name == "show_message",
**({"userAction": user_action} if user_action else {}),
**(
{"error": str(message.get("message") or "客户端工具执行失败")}
+2
View File
@@ -120,6 +120,8 @@ class ConversationRecorder:
async def _append_workflow_event(self, message: dict) -> None:
"""Persist bounded control-plane history without creating chat rows."""
if message.get("internalStage") is True:
return
event_id = str(message.get("eventId") or "")
event_name = str(message.get("event") or "")
event_key = f"workflow:{event_id}"
+63 -4
View File
@@ -5,6 +5,8 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from typing import Any
from uuid import uuid4
from loguru import logger
from pipecat.frames.frames import (
@@ -14,6 +16,38 @@ from pipecat.frames.frames import (
InterruptionFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.time import time_now_iso8601
CALL_END_EVENT_BY_STATE = {
"pending": "conversation_end_requested",
"speaking": "conversation_end_speaking",
"cancelled": "conversation_end_cancelled",
"ended": "conversation_ended",
}
def call_end_trace_event(
state: str,
*,
reason: str = "",
interruptible: bool = False,
voice_interruptible: bool = False,
) -> dict[str, Any]:
"""Build one shared record used by live calls and headless tests."""
return {
"type": "workflow-event",
"eventId": f"wfe_{uuid4().hex[:20]}",
"event": CALL_END_EVENT_BY_STATE.get(
state,
"conversation_end_state_changed",
),
"timestamp": time_now_iso8601(),
**({"reason": reason} if reason else {}),
"interruptible": interruptible,
"voiceInterruptible": voice_interruptible,
}
class SpeechPlaybackCompletion:
@@ -76,6 +110,7 @@ class CallEndCoordinator:
):
self._queue_end = queue_end
self._on_state_change = on_state_change
self._state_tasks: set[asyncio.Task[None]] = set()
self._ending = False
self._interruptible = False
self._armed = False
@@ -86,7 +121,7 @@ class CallEndCoordinator:
self._tracked_speech_completions: set[SpeechPlaybackCompletion] = set()
self._finish_after_tracked_speech = False
self._finished = False
self._reason = "completed"
self._reason = ""
@property
def ending(self) -> bool:
@@ -106,6 +141,12 @@ class CallEndCoordinator:
"""Whether transport output is currently producing bot speech."""
return self._speaking
@property
def reason(self) -> str:
"""Current requested ending reason for client feedback and records."""
return self._reason
async def wait_until_silent(self) -> None:
"""Wait for the transport-owned bot speech boundary."""
await self._speech_stopped.wait()
@@ -124,6 +165,7 @@ class CallEndCoordinator:
"""Cancel one pending ending when its tool explicitly allows it."""
if not self.interruptible:
return False
was_interruptible = self._interruptible
self._ending = False
self._interruptible = False
self._armed = False
@@ -132,12 +174,26 @@ class CallEndCoordinator:
# Transport speech state remains authoritative: if audio is currently
# playing, the interruption path will still emit BotStoppedSpeakingFrame.
logger.info("用户打断结束语,已撤销待挂断状态")
self._schedule_state("cancelled")
self._schedule_state("cancelled", interruptible=was_interruptible)
return True
def _schedule_state(self, state: str) -> None:
def _schedule_state(
self,
state: str,
*,
interruptible: bool | None = None,
) -> None:
if self._on_state_change is not None:
asyncio.create_task(self._on_state_change(state, self.interruptible))
task = asyncio.create_task(
self._on_state_change(
state,
self.interruptible
if interruptible is None
else interruptible,
)
)
self._state_tasks.add(task)
task.add_done_callback(self._state_tasks.discard)
def begin_response(self) -> None:
"""Start tracking speech produced by one LLM response."""
@@ -196,6 +252,9 @@ class CallEndCoordinator:
return
self._finished = True
self._interruptible = False
pending_states = tuple(self._state_tasks)
if pending_states:
await asyncio.gather(*pending_states, return_exceptions=True)
await self._queue_end(self._reason)
async def observe(self, frame) -> None:
+56 -6
View File
@@ -17,6 +17,7 @@ from services.conversation_history import ConversationRecorder
from services.pipecat.call_lifecycle import (
CallEndCoordinator,
EndCallAfterSpeechProcessor,
call_end_trace_event,
)
from services.pipecat.service_factory import (
config_with_resource,
@@ -208,6 +209,7 @@ def _call_end_state_frame(
*,
interruptible: bool,
voice_interruptible: bool,
reason: str = "",
) -> OutputTransportMessageUrgentFrame:
"""Build the client event used to render the pending call-end state."""
return OutputTransportMessageUrgentFrame(
@@ -216,10 +218,30 @@ def _call_end_state_frame(
"state": state,
"interruptible": interruptible,
"voice_interruptible": voice_interruptible,
**({"reason": reason} if reason else {}),
}
)
def _call_lifecycle_trace_frame(
state: str,
*,
reason: str = "",
interruptible: bool = False,
voice_interruptible: bool = False,
) -> OutputTransportMessageUrgentFrame:
"""Build one persistent runtime record for the call-ending lifecycle."""
return OutputTransportMessageUrgentFrame(
message=call_end_trace_event(
state,
reason=reason,
interruptible=interruptible,
voice_interruptible=voice_interruptible,
)
)
async def run_pipeline(
transport,
cfg: AssistantConfig,
@@ -307,6 +329,12 @@ async def run_pipeline(
if worker is None:
return
logger.info(f"结束通话: reason={reason}")
await worker.queue_frame(
_call_lifecycle_trace_frame(
"ended",
reason=reason,
)
)
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={"type": "call-ended", "reason": reason}
@@ -318,13 +346,21 @@ async def run_pipeline(
worker = worker_holder.get("worker")
if worker is None:
return
voice_interruptible = interruptible and current_enable_interrupt
await worker.queue_frame(
_call_end_state_frame(
state,
interruptible=interruptible,
voice_interruptible=(
interruptible and current_enable_interrupt
),
voice_interruptible=voice_interruptible,
reason=call_end.reason,
)
)
await worker.queue_frame(
_call_lifecycle_trace_frame(
state,
reason=call_end.reason,
interruptible=interruptible,
voice_interruptible=voice_interruptible,
)
)
@@ -1043,6 +1079,12 @@ async def run_realtime_pipeline(
if worker is None:
return
logger.info(f"结束 Realtime 通话: reason={reason}")
await worker.queue_frame(
_call_lifecycle_trace_frame(
"ended",
reason=reason,
)
)
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={"type": "call-ended", "reason": reason}
@@ -1054,13 +1096,21 @@ async def run_realtime_pipeline(
worker = worker_holder.get("worker")
if worker is None:
return
voice_interruptible = interruptible and cfg.enableInterrupt
await worker.queue_frame(
_call_end_state_frame(
state,
interruptible=interruptible,
voice_interruptible=(
interruptible and cfg.enableInterrupt
),
voice_interruptible=voice_interruptible,
reason=call_end.reason,
)
)
await worker.queue_frame(
_call_lifecycle_trace_frame(
state,
reason=call_end.reason,
interruptible=interruptible,
voice_interruptible=voice_interruptible,
)
)
+2
View File
@@ -163,6 +163,7 @@ class EvaluationEngine:
user_input=expected_turn.user_input,
assistant_reply="",
tool_calls=[],
runtime_records=[],
)
evaluations = [
await self._evaluate_behavior(behavior, raw)
@@ -175,6 +176,7 @@ class EvaluationEngine:
user_input=raw.user_input,
assistant_reply=raw.assistant_reply,
tool_calls=[item.as_result() for item in raw.tool_calls],
runtime_records=list(raw.runtime_records),
evaluations=evaluations,
)
)
@@ -302,6 +302,7 @@ class TestRunOrchestrator:
"greeting": "",
"assistantOutputs": [],
"autoConfirmedMessages": [],
"runtimeRecords": [],
},
"overallCriteria": [],
}
@@ -324,6 +325,7 @@ class TestRunOrchestrator:
"autoConfirmedMessages": list(
startup.auto_confirmed_messages
),
"runtimeRecords": list(startup.runtime_records),
}
row.result = result
await session.commit()
@@ -346,6 +348,7 @@ class TestRunOrchestrator:
"userInput": turn.user_input,
"assistantReply": turn.assistant_reply,
"toolCalls": [item.as_result() for item in turn.tool_calls],
"runtimeRecords": list(turn.runtime_records),
"evaluations": [],
}
turns = [item for item in turns if item.get("id") != turn.id]
@@ -382,6 +385,7 @@ class TestRunOrchestrator:
"autoConfirmedMessages": list(
startup.auto_confirmed_messages
),
"runtimeRecords": list(startup.runtime_records),
},
"overallCriteria": [
item.model_dump(mode="json", by_alias=True)
+98 -2
View File
@@ -26,6 +26,7 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesAppendFrame,
ManuallySwitchServiceFrame,
OutputTransportMessageUrgentFrame,
TTSSpeakFrame,
)
from pipecat.pipeline.pipeline import Pipeline
@@ -44,6 +45,7 @@ from services.knowledge import search as search_knowledge
from services.pipecat.call_lifecycle import (
CallEndCoordinator,
FixedSpeechPlaybackMarkerFrame,
call_end_trace_event,
)
from services.pipecat.processors import (
KnowledgeRetrievalProcessor,
@@ -102,6 +104,7 @@ class RawTurnResult:
user_input: str
assistant_reply: str
tool_calls: list[RawToolCall]
runtime_records: list[dict[str, Any]] = field(default_factory=list)
@dataclass
@@ -111,6 +114,7 @@ class TextStartupResult:
greeting: str = ""
assistant_outputs: list[str] = field(default_factory=list)
auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list)
runtime_records: list[dict[str, Any]] = field(default_factory=list)
@dataclass
@@ -148,6 +152,30 @@ class _ResponseWindow:
return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools
def _opening_speech_from_message(message: Any) -> str:
"""Extract only the transcript emitted by Prompt's opening Message stage."""
if (
not isinstance(message, dict)
or message.get("type") != "transcript"
or message.get("source") != "prompt-opening-speech"
):
return ""
return str(message.get("content") or "").strip()
def _runtime_record_from_message(message: Any) -> dict[str, Any] | None:
"""Keep the same runtime-record envelope used by live conversation history."""
if (
not isinstance(message, dict)
or message.get("type") != "workflow-event"
or message.get("internalStage") is True
):
return None
return dict(message)
class _TextCaptureProcessor(FrameProcessor):
"""Capture assistant/tool output and expose a reliable response boundary."""
@@ -156,6 +184,8 @@ class _TextCaptureProcessor(FrameProcessor):
self.context = context
self.window = _ResponseWindow()
self.outputs: list[str] = []
self.opening_speech_outputs: list[str] = []
self.runtime_records: list[dict[str, Any]] = []
self.tool_calls: list[RawToolCall] = []
self._tool_by_id: dict[str, RawToolCall] = {}
self.error: TestExecutionError | None = None
@@ -195,6 +225,14 @@ class _TextCaptureProcessor(FrameProcessor):
await self.on_fixed_speech_complete()
return
if isinstance(frame, OutputTransportMessageUrgentFrame):
opening_speech = _opening_speech_from_message(frame.message)
if opening_speech:
self.opening_speech_outputs.append(opening_speech)
runtime_record = _runtime_record_from_message(frame.message)
if runtime_record is not None:
self.runtime_records.append(runtime_record)
if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
await frame.completion.mark_played()
self.window.touch()
@@ -248,6 +286,27 @@ class _TextCaptureProcessor(FrameProcessor):
await self.push_frame(frame, direction)
def _resolve_startup_greeting(
shared_greeting: str,
opening_speech_outputs: list[str],
) -> str:
"""Return the opening speech shown by the headless run.
Prompt confirmation mode deliberately suppresses the shared greeting
because its atomic Message stage owns both speech and the dialog. In text
tests the fixed speech transcript carries a dedicated source marker, so
preserve that exact output instead of guessing from unrelated fixed speech.
"""
greeting = shared_greeting.strip()
if greeting:
return greeting
return next(
(text.strip() for text in opening_speech_outputs if text.strip()),
"",
)
def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
messages: list[dict[str, Any]] = []
latest_call_by_name: dict[str, str] = {}
@@ -494,9 +553,34 @@ class TextPipelineRunner:
async def queue_call_end(reason: str) -> None:
logger.debug(f"文本测试收到会话结束请求: {reason}")
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message=call_end_trace_event(
"ended",
reason=reason,
)
)
)
await worker.queue_frame(EndFrame(reason=reason))
call_end = CallEndCoordinator(queue_call_end)
async def record_call_end_state(
state: str,
interruptible: bool,
) -> None:
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message=call_end_trace_event(
state,
reason=call_end.reason,
interruptible=interruptible,
)
)
)
call_end = CallEndCoordinator(
queue_call_end,
on_state_change=record_call_end_state,
)
async def finish_fixed_speech_if_ending() -> None:
if call_end.ending:
@@ -667,6 +751,7 @@ class TextPipelineRunner:
runner_task: asyncio.Task[None] | None = None
try:
startup_runtime_record_marker = len(capture.runtime_records)
await brain.run_preflight()
if executor.unmocked_calls:
raise TestExecutionError(
@@ -693,12 +778,19 @@ class TextPipelineRunner:
runner_task,
marker,
)
startup_greeting = _resolve_startup_greeting(
greeting,
capture.opening_speech_outputs,
)
startup = TextStartupResult(
greeting=greeting,
greeting=startup_greeting,
assistant_outputs=list(capture.outputs),
auto_confirmed_messages=list(
text_client_tools.auto_confirmed_messages
),
runtime_records=list(
capture.runtime_records[startup_runtime_record_marker:]
),
)
if on_startup_complete is not None:
await on_startup_complete(startup)
@@ -775,6 +867,7 @@ class TextPipelineRunner:
activity_marker = capture.window.activity_count
output_marker = len(capture.outputs)
tool_marker = len(capture.tool_calls)
runtime_record_marker = len(capture.runtime_records)
await worker.queue_frame(
LLMMessagesAppendFrame(
messages=[{"role": "user", "content": turn.user_input}],
@@ -795,4 +888,7 @@ class TextPipelineRunner:
user_input=turn.user_input,
assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
tool_calls=list(capture.tool_calls[tool_marker:]),
runtime_records=list(
capture.runtime_records[runtime_record_marker:]
),
)
+2
View File
@@ -364,6 +364,7 @@ class BatchTurnResult(CamelModel):
user_input: str
assistant_reply: str
tool_calls: list[BatchToolCallRecord] = Field(default_factory=list)
runtime_records: list[dict[str, Any]] = Field(default_factory=list)
evaluations: list[BatchEvaluationResult] = Field(default_factory=list)
@@ -372,6 +373,7 @@ class BatchStartupResult(CamelModel):
greeting: str = ""
assistant_outputs: list[str] = Field(default_factory=list)
auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list)
runtime_records: list[dict[str, Any]] = Field(default_factory=list)
class BatchRunCaseOut(CamelModel):
+18
View File
@@ -521,6 +521,24 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertIn("点击“确认”", confirmation_messages[0]["content"])
self.assertIn("弹窗标题:重要提示", confirmation_messages[0]["content"])
self.assertIn("弹窗内容:请确认已阅读。", confirmation_messages[0]["content"])
message_traces = [
frame.message
for frame in queued
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "workflow-event"
and frame.message.get("event")
in {"message_started", "message_completed"}
]
self.assertEqual(
[event["event"] for event in message_traces],
["message_started", "message_completed"],
)
self.assertEqual(message_traces[0]["messageTitle"], "重要提示")
self.assertEqual(
message_traces[0]["messageContent"],
"请确认已阅读。",
)
self.assertEqual(message_traces[1]["action"], "confirmed")
self.assertEqual(
sum(isinstance(frame, LLMRunFrame) for frame in queued),
1,
+17 -1
View File
@@ -11,12 +11,27 @@ from pipecat.frames.frames import (
)
from services.pipecat.call_lifecycle import (
CallEndCoordinator,
call_end_trace_event,
playback_marker_for,
)
from services.pipecat.processors import CallEndingUserMuteStrategy
class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
def test_shared_trace_event_uses_runtime_record_contract(self):
event = call_end_trace_event(
"pending",
reason="用户已完成咨询",
interruptible=True,
voice_interruptible=True,
)
self.assertEqual(event["type"], "workflow-event")
self.assertEqual(event["event"], "conversation_end_requested")
self.assertEqual(event["reason"], "用户已完成咨询")
self.assertTrue(event["interruptible"])
self.assertTrue(event["voiceInterruptible"])
async def asyncSetUp(self):
self.reasons: list[str] = []
self.states: list[tuple[str, bool]] = []
@@ -87,7 +102,7 @@ class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
self.assertTrue(self.coordinator.accepts_input)
self.assertTrue(self.coordinator.cancel_pending_end())
await asyncio.sleep(0)
self.assertEqual(self.states[-1], ("cancelled", False))
self.assertEqual(self.states[-1], ("cancelled", True))
self.assertFalse(self.coordinator.ending)
self.assertTrue(self.coordinator.accepts_input)
self.assertTrue(self.coordinator.speaking)
@@ -95,6 +110,7 @@ class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
await self.coordinator.observe(BotStoppedSpeakingFrame())
self.assertFalse(self.coordinator.speaking)
self.assertEqual(self.reasons, [])
self.assertEqual(self.coordinator.reason, "user_goodbye")
async def test_committed_end_rejects_cancellation(self):
self.coordinator.begin("required_shutdown")
@@ -144,6 +144,22 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(saved["sessionId"], "conv_test")
self.assertEqual(saved["sequence"], 1)
async def test_internal_message_stage_transport_trace_is_not_persisted_twice(self):
recorder = ConversationRecorder("conv_test")
event = {
"type": "workflow-event",
"eventId": "wfe_internal",
"event": "client_tool_completed",
"functionName": "show_message",
"internalStage": True,
}
with patch("services.conversation_history.SessionLocal") as session_local:
await recorder.record_transport_message(event)
session_local.assert_not_called()
self.assertEqual(recorder._trace_sequence, 0)
async def test_image_is_persisted_as_message_with_artifact(self):
conversation = SimpleNamespace(message_count=0)
+91 -1
View File
@@ -10,10 +10,13 @@ from services.runtime_variables import DynamicVariableStore, prepare_dynamic_con
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from services.test_runs.text_runner import (
_TextCaptureProcessor,
_opening_speech_from_message,
_resolve_startup_greeting,
_runtime_record_from_message,
_wait_for_startup,
_wait_for_quiescence,
)
from test_schemas import BatchRunCreate
from test_schemas import BatchRunCreate, BatchStartupResult
class FakeLLM:
@@ -126,6 +129,25 @@ class DebugAutoTestContractTests(unittest.TestCase):
"你好,测试用户",
)
def test_startup_result_exposes_real_runtime_records(self):
result = BatchStartupResult.model_validate(
{
"completed": True,
"runtimeRecords": [
{
"type": "workflow-event",
"eventId": "wfe_opening",
"event": "message_started",
}
],
}
)
self.assertEqual(
result.runtime_records[0]["event"],
"message_started",
)
def test_text_client_port_records_automatic_message_confirmation(self):
port = TextClientToolPort()
@@ -142,6 +164,51 @@ class DebugAutoTestContractTests(unittest.TestCase):
[{"title": "重要提示", "message": "请确认"}],
)
def test_confirmation_owned_speech_is_kept_as_startup_greeting(self):
self.assertEqual(
_resolve_startup_greeting(
"",
["您好,请先阅读弹窗。"],
),
"您好,请先阅读弹窗。",
)
self.assertEqual(
_resolve_startup_greeting(
"普通开场白",
["其他固定播报"],
),
"普通开场白",
)
self.assertEqual(
_resolve_startup_greeting(
"",
[],
),
"",
)
def test_only_prompt_opening_transcript_is_identified_as_greeting(self):
self.assertEqual(
_opening_speech_from_message(
{
"type": "transcript",
"content": "您好,请先阅读弹窗。",
"source": "prompt-opening-speech",
}
),
"您好,请先阅读弹窗。",
)
self.assertEqual(
_opening_speech_from_message(
{
"type": "transcript",
"content": "普通 Workflow 播报",
"source": "workflow-message-speech",
}
),
"",
)
class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
@@ -180,6 +247,29 @@ class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(params.result["action"], "ending_call")
self.assertEqual(executor_holder["executor"].unmocked_calls, [])
async def test_text_capture_keeps_runtime_records_for_auto_test_ui(self):
event = {
"type": "workflow-event",
"eventId": "wfe_end",
"event": "conversation_end_requested",
"reason": "测试完成",
}
self.assertEqual(_runtime_record_from_message(event), event)
self.assertIsNone(
_runtime_record_from_message({"type": "call-end-state"})
)
self.assertIsNone(
_runtime_record_from_message(
{
"type": "workflow-event",
"eventId": "wfe_internal",
"event": "client_tool_completed",
"internalStage": True,
}
)
)
async def test_startup_waits_for_queued_entry_reply(self):
capture = _TextCaptureProcessor(LLMContext(messages=[]))
@@ -22,6 +22,10 @@ import {
X,
} from "lucide-react";
import {
DebugTimeline,
type DebugTimelineItem,
} from "@/components/debug-timeline";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
@@ -252,17 +256,32 @@ function CasePicker({
export function AutoTestLivePanel({
testCase,
state,
onStart,
}: {
testCase: TestCase | null;
state: AutoTestRunState;
onStart: () => void;
}) {
if (!testCase) {
return (
<EmptyPanel
icon={<FlaskConical size={22} />}
title="选择一个测试用例"
description="运行真实 Pipeline,并在这里查看逐轮回复、工具调用和判断结果。"
/>
<div className="flex min-h-0 flex-1 flex-col items-center justify-center px-6 py-8 text-center">
<span className="flex h-10 w-10 items-center justify-center rounded-full bg-surface-strong text-muted-soft">
<FlaskConical size={19} />
</span>
<div className="mt-3 text-sm font-medium text-foreground">
选择测试用例后开始
</div>
<Button
type="button"
variant="secondary"
className="mt-4 h-10 gap-2 rounded-full px-5"
disabled
title="请先在下方选择测试用例"
>
<Play size={14} />
开始测试
</Button>
</div>
);
}
@@ -270,20 +289,20 @@ export function AutoTestLivePanel({
const validationMessage = runnableMessage(testCase);
return (
<div className="flex min-h-0 flex-1 flex-col items-center justify-center overflow-y-auto px-6 py-8 text-center">
<div className="max-w-sm">
<div className="text-base font-medium text-foreground">
<div className="w-full max-w-sm">
<div className="text-lg font-medium tracking-[-0.01em] text-foreground">
{testCase.name}
</div>
<div className="mt-2 flex flex-wrap items-center justify-center gap-2">
<Badge
variant="secondary"
className="rounded-full px-2 py-0 text-[10px] font-medium"
>
{TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]}
</Badge>
<span className="text-[11px] text-muted-soft">
{testCase.turns.length} 轮
</span>
<div className="mt-2 flex flex-wrap items-center justify-center gap-2 text-xs text-muted-foreground">
<span>{TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]}</span>
<span aria-hidden className="text-hairline-strong">·</span>
<span>{testCase.turns.length} 轮</span>
{state.evaluatorName && (
<>
<span aria-hidden className="text-hairline-strong">·</span>
<span>{state.evaluatorName}</span>
</>
)}
</div>
{testCase.description && (
<p className="mt-3 text-xs leading-5 text-muted-foreground">
@@ -294,11 +313,22 @@ export function AutoTestLivePanel({
<div className="mt-4 rounded-xl border border-amber-500/20 bg-amber-500/[0.06] px-3 py-2 text-xs leading-5 text-amber-600 dark:text-amber-400">
{validationMessage}
</div>
) : state.evaluatorName ? (
<p className="mt-4 text-[11px] text-muted-soft">
评估模型 · {state.evaluatorName}
</p>
) : null}
<Button
type="button"
className="mx-auto mt-5 h-10 gap-2 rounded-full px-5"
disabled={Boolean(state.blockedMessage)}
onClick={onStart}
title={state.blockedMessage || "开始测试"}
>
<Play size={14} />
开始测试
</Button>
{state.blockedMessage && !validationMessage && (
<p className="mt-2 text-[10px] leading-4 text-muted-soft">
{state.blockedMessage}
</p>
)}
</div>
</div>
);
@@ -327,8 +357,11 @@ export function AutoTestLivePanel({
if (startup?.completed) {
return (
<div className="scrollbar-subtle min-h-0 flex-1 overflow-y-auto px-5 py-4">
<StartupDetails startup={startup} />
<StartupAssistantMessages startup={startup} />
<AutoTestTimeline
startup={startup}
turns={[]}
maxTurns={state.maxTurns}
/>
<div className="flex items-center justify-center gap-2 py-4 text-xs text-muted-foreground">
<Loader2 size={13} className="animate-spin" />
正在执行第 1 轮固定输入…
@@ -351,62 +384,11 @@ export function AutoTestLivePanel({
return (
<div className="scrollbar-subtle flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4">
<div className="flex flex-col gap-4">
{startup?.completed && <StartupDetails startup={startup} />}
{startup?.completed && <StartupAssistantMessages startup={startup} />}
{turns.map((turn) => (
<section key={turn.id} className="flex flex-col gap-3">
<div className="flex max-w-[88%] flex-col items-end gap-1 self-end">
<span className="flex items-center gap-2 px-1 text-[11px] text-muted-soft">
<span>固定输入</span>
<span className="rounded-full bg-surface-strong px-2 py-0.5 text-[10px]">
{turn.index + 1} / {state.maxTurns}
</span>
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
{turn.userInput}
</div>
</div>
<div className="flex max-w-[88%] flex-col gap-1 self-start">
<span className="px-1 text-[11px] text-muted-soft">助手</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{turn.assistantReply || "(没有生成文字回复)"}
</div>
</div>
{turn.toolCalls.length > 0 && (
<div className="w-full space-y-2 rounded-xl border border-violet-500/20 bg-violet-500/[0.04] px-3 py-2.5">
<div className="flex items-center gap-1.5 text-[11px] font-medium text-foreground">
<Wrench size={12} />
工具调用
</div>
{turn.toolCalls.map((toolCall) => (
<div key={toolCall.id} className="text-[11px] leading-5">
<code className="text-foreground">{toolCall.functionName}</code>
<span className="ml-2 text-muted-soft">
{toolCall.outcome === "success" ? "成功" : "错误"} ·{" "}
{toolCall.durationMs} ms
</span>
<pre className="mt-1 max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
{toolCall.argumentsJson}
</pre>
</div>
))}
</div>
)}
{turn.evaluations.length > 0 && (
<div className="w-full space-y-2">
{turn.evaluations.map((evaluation) => (
<EvaluationDetails
key={evaluation.id}
evaluation={evaluation}
<AutoTestTimeline
startup={startup}
turns={turns}
maxTurns={state.maxTurns}
/>
))}
</div>
)}
</section>
))}
{state.phase === "running" && (
<div className="flex items-center justify-center gap-2 py-2 text-xs text-muted-foreground">
@@ -446,78 +428,15 @@ export function AutoTestLivePanel({
);
}
function StartupDetails({ startup }: { startup: BatchStartupResult }) {
function startupAssistantMessages(startup: BatchStartupResult): string[] {
const greeting = startup.greeting.trim();
return (
<section className="w-full space-y-3 rounded-xl border border-hairline bg-canvas-soft px-3 py-3">
<div className="flex flex-wrap items-start justify-between gap-2">
<div>
<div className="text-xs font-medium text-foreground">开场记录</div>
<p className="mt-0.5 text-[10px] leading-4 text-muted-soft">
已按真实开场流程模拟,不计入脚本轮次断言
</p>
</div>
<Badge
variant="secondary"
className="rounded-full px-2 py-0 text-[10px] font-medium"
>
已完成
</Badge>
</div>
{greeting && (
<StartupTextBlock label="开场白 · 按已播放处理" value={greeting} />
)}
{startup.autoConfirmedMessages.map((message, index) => {
const confirmedAction =
message.actions?.find((action) => action.id === "confirmed") ??
message.actions?.[0];
return (
<div
key={`${index}-${message.title ?? message.message ?? "message"}`}
className="rounded-xl border border-hairline bg-background/70 px-3 py-2.5"
>
<div className="text-[10px] font-medium text-muted-soft">
消息弹窗{startup.autoConfirmedMessages.length > 1 ? ` ${index + 1}` : ""}
</div>
<div className="mt-1 text-xs font-medium text-foreground">
{message.title?.trim() || "未命名消息"}
</div>
<p className="mt-1 whitespace-pre-wrap break-words text-[11px] leading-5 text-muted-foreground">
{message.message?.trim() || "(没有消息正文)"}
</p>
<div className="mt-2 inline-flex items-center gap-1 rounded-full border border-success/20 px-2 py-0.5 text-[10px] text-success">
<Check size={10} />
自动点击“{confirmedAction?.label?.trim() || "确认"}”
</div>
</div>
);
})}
</section>
);
}
function StartupAssistantMessages({
startup,
}: {
startup: BatchStartupResult;
}) {
const openingTexts = new Set(
[
startup.greeting,
...startup.autoConfirmedMessages.map((message) => message.message ?? ""),
]
.map(normalizeDisplayText)
.filter(Boolean),
);
const greetingText = normalizeDisplayText(greeting);
const seen = new Set<string>();
const assistantMessages = startup.assistantOutputs.filter((output) => {
const normalizedOutput = normalizeDisplayText(output);
if (
!normalizedOutput ||
openingTexts.has(normalizedOutput) ||
normalizedOutput === greetingText ||
seen.has(normalizedOutput)
) {
return false;
@@ -525,43 +444,235 @@ function StartupAssistantMessages({
seen.add(normalizedOutput);
return true;
});
return [greeting, ...assistantMessages].filter(Boolean);
}
if (assistantMessages.length === 0) return null;
function legacyStartupRuntimeRecords(
startup: BatchStartupResult,
): Array<Record<string, unknown>> {
return startup.autoConfirmedMessages.flatMap((message, index) => {
const confirmedAction =
message.actions?.find((action) => action.id === "confirmed") ??
message.actions?.[0];
const legacyId = `${index}-${message.title ?? message.message ?? "message"}`;
return [
{
eventId: `legacy-message-started-${legacyId}`,
event: "message_started",
phase: "opening",
requiresConfirmation: true,
messageTitle: message.title?.trim() || "未命名消息",
messageContent: message.message?.trim() || "(没有消息正文)",
messageActions: message.actions ?? [],
},
{
eventId: `legacy-message-completed-${legacyId}`,
event: "message_completed",
phase: "opening",
action: confirmedAction?.id?.trim() || "confirmed",
},
];
});
}
const label =
startup.autoConfirmedMessages.length > 0
? "助手 · 确认后自动输出"
: "助手 · 开场自动输出";
return (
<div className="flex w-full flex-col gap-3">
{assistantMessages.map((message, index) => (
<div
key={`${index}-${message}`}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">{label}</span>
<div className="whitespace-pre-wrap break-words rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.trim()}
</div>
</div>
))}
</div>
);
function runtimeRecordKey(
event: Record<string, unknown>,
prefix: string,
index: number,
): string {
const id = typeof event.eventId === "string" ? event.eventId : "";
const name = typeof event.event === "string" ? event.event : "record";
return id || `${prefix}-${name}-${index}`;
}
function normalizeDisplayText(value: string): string {
return value.trim().replace(/\s+/g, " ");
}
function StartupTextBlock({ label, value }: { label: string; value: string }) {
return (
<div className="rounded-xl border border-hairline bg-background/70 px-3 py-2.5">
<div className="text-[10px] font-medium text-muted-soft">{label}</div>
<p className="mt-1 whitespace-pre-wrap break-words text-[11px] leading-5 text-muted-foreground">
{value}
</p>
function AutoTestTimeline({
startup,
turns,
maxTurns,
}: {
startup?: BatchStartupResult;
turns: BatchRunCase["turns"];
maxTurns: number;
}) {
const items: DebugTimelineItem[] = [];
if (startup?.completed) {
items.push({
id: "startup-boundary",
kind: "content",
content: (
<div className="flex items-center gap-2 py-1 text-[11px] text-muted-soft">
<span className="h-px flex-1 bg-hairline" />
<span>开场阶段 · 自动执行,不计入脚本断言</span>
<span className="h-px flex-1 bg-hairline" />
</div>
),
});
const openingMessages = startupAssistantMessages(startup);
const records =
(startup.runtimeRecords?.length ?? 0) > 0
? startup.runtimeRecords ?? []
: legacyStartupRuntimeRecords(startup);
const firstMessageStartedIndex = records.findIndex(
(event) => event.event === "message_started",
);
const firstMessageCompletedIndex = records.findIndex(
(event) => event.event === "message_completed",
);
const appendOpeningMessage = (message: string, index: number) => {
items.push({
id: `startup-message-${index}`,
kind: "message",
message: {
id: `startup-message-${index}`,
role: "assistant",
content: message,
},
});
};
const appendRuntimeRecord = (
event: Record<string, unknown>,
index: number,
) => {
const id = runtimeRecordKey(event, "startup-record", index);
items.push({
id,
kind: "runtime-record",
record: {
id,
event,
timestamp:
typeof event.timestamp === "string" ? event.timestamp : undefined,
},
});
};
if (firstMessageStartedIndex < 0) {
records.forEach(appendRuntimeRecord);
openingMessages.forEach(appendOpeningMessage);
} else {
records.forEach((event, index) => {
appendRuntimeRecord(event, index);
if (index === firstMessageStartedIndex && openingMessages[0]) {
appendOpeningMessage(openingMessages[0], 0);
}
if (index === firstMessageCompletedIndex) {
openingMessages.slice(1).forEach((message, outputIndex) => {
appendOpeningMessage(message, outputIndex + 1);
});
}
});
if (firstMessageCompletedIndex < 0) {
openingMessages.slice(1).forEach((message, outputIndex) => {
appendOpeningMessage(message, outputIndex + 1);
});
}
}
}
turns.forEach((turn) => {
items.push({
id: `turn-user-${turn.id}`,
kind: "message",
message: {
id: `turn-user-${turn.id}`,
role: "user",
label: `固定输入 · ${turn.index + 1} / ${maxTurns}`,
content: turn.userInput,
},
});
items.push({
id: `turn-assistant-${turn.id}`,
kind: "message",
message: {
id: `turn-assistant-${turn.id}`,
role: "assistant",
content: turn.assistantReply || "(没有生成文字回复)",
},
});
turn.runtimeRecords?.forEach((event, index) => {
const id = runtimeRecordKey(event, `turn-record-${turn.id}`, index);
items.push({
id,
kind: "runtime-record",
record: {
id,
event,
timestamp:
typeof event.timestamp === "string" ? event.timestamp : undefined,
},
});
});
if (turn.toolCalls.length > 0) {
items.push({
id: `turn-tools-${turn.id}`,
kind: "content",
content: <AutoTestToolDetails toolCalls={turn.toolCalls} />,
});
}
if (turn.evaluations.length > 0) {
items.push({
id: `turn-evaluations-${turn.id}`,
kind: "content",
content: (
<div className="w-full space-y-2">
{turn.evaluations.map((evaluation) => (
<EvaluationDetails
key={evaluation.id}
evaluation={evaluation}
/>
))}
</div>
),
});
}
});
return <DebugTimeline items={items} recordContext="auto-test" />;
}
function AutoTestToolDetails({
toolCalls,
}: {
toolCalls: BatchRunCase["turns"][number]["toolCalls"];
}) {
return (
<details className="w-full rounded-xl border border-hairline bg-canvas-soft px-3 py-2.5 text-xs">
<summary className="flex cursor-pointer list-none items-center gap-1.5 font-medium text-foreground">
<Wrench size={12} />
工具调用详情
<span className="ml-auto text-[10px] font-normal text-muted-soft">
{toolCalls.length} 次
</span>
</summary>
<div className="mt-2 space-y-3 border-t border-hairline pt-2">
{toolCalls.map((toolCall) => (
<div key={toolCall.id} className="text-[11px] leading-5">
<div>
<code className="text-foreground">{toolCall.functionName}</code>
<span className="ml-2 text-muted-soft">
{toolCall.outcome === "success" ? "成功" : "错误"} ·{" "}
{toolCall.durationMs} ms
</span>
</div>
<div className="mt-1 grid gap-1.5">
<pre className="max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
{toolCall.argumentsJson}
</pre>
<pre className="max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
{toolCall.resultJson}
</pre>
</div>
</div>
))}
</div>
</details>
);
}
@@ -671,7 +782,6 @@ export function AutoTestRunBar({
loading,
onSelectCase,
onClearCase,
onStart,
onStop,
onRerun,
}: {
@@ -682,7 +792,6 @@ export function AutoTestRunBar({
loading: boolean;
onSelectCase: (id: string) => void;
onClearCase: () => void;
onStart: () => void;
onStop: () => void;
onRerun: () => void;
}) {
@@ -818,17 +927,6 @@ export function AutoTestRunBar({
</button>
)}
</div>
<Button
type="button"
size="sm"
className="h-8 shrink-0 gap-1.5 rounded-full px-3"
disabled={Boolean(state.blockedMessage)}
onClick={onStart}
title={state.blockedMessage || "开始测试"}
>
<Play size={13} />
开始测试
</Button>
</div>
{state.blockedMessage && testCase && (
<p className="truncate px-3 text-[10px] leading-4 text-muted-soft">
@@ -29,6 +29,10 @@ import {
DebugModeTabs,
useAutoTestRunner,
} from "@/components/assistant-editor/debug-auto-test";
import {
DebugTimeline,
type DebugTimelineItem,
} from "@/components/debug-timeline";
import { NetworkQualityIndicator } from "@/components/network-quality-indicator";
import { ClientMessageDialog } from "@/components/client-message-dialog";
import { AuraVisualizer } from "@/components/ui/aura-visualizer";
@@ -60,6 +64,7 @@ import {
type ChatMessage,
type CallEndState,
type ClientToolDefinition,
type LiveRuntimeRecord,
type UserInputPart,
type VoicePreview,
type VoicePreviewStatus,
@@ -383,7 +388,11 @@ function AutoTestPanel({
return (
<div className="flex min-h-0 flex-1 flex-col overflow-hidden">
<div className="flex min-h-0 flex-1 flex-col overflow-hidden">
<AutoTestLivePanel testCase={selectedCase} state={state} />
<AutoTestLivePanel
testCase={selectedCase}
state={state}
onStart={start}
/>
</div>
<div className="shrink-0 border-t border-hairline bg-card p-3">
<AutoTestRunBar
@@ -394,7 +403,6 @@ function AutoTestPanel({
loading={catalogLoading}
onSelectCase={selectCase}
onClearCase={clearCase}
onStart={start}
onStop={stop}
onRerun={rerun}
/>
@@ -895,6 +903,7 @@ function DebugVoicePanel({
localStream,
remoteStream,
messages,
runtimeRecords,
sendText,
sendUserInput,
appendUserImage,
@@ -924,7 +933,8 @@ function DebugVoicePanel({
const idleOrFailed = status === "idle" || status === "failed";
const showIdleHub =
idleOrFailed &&
(view === "video" || (messages.length === 0 && inChatView));
(view === "video" ||
(messages.length === 0 && runtimeRecords.length === 0 && inChatView));
const startBlockedMessage = !assistantId
? "请先保存助手,再开始对话。"
: hasUnsavedChanges
@@ -1087,6 +1097,7 @@ function DebugVoicePanel({
view={view}
onViewChange={onViewChange}
messages={messages}
runtimeRecords={runtimeRecords}
recording={recording}
camera={camera}
videoStream={preview.videoStream}
@@ -1115,7 +1126,11 @@ function DebugVoicePanel({
connect={startConversation}
/>
) : (
<DebugTranscriptPanel messages={messages} recording={recording} />
<DebugTranscriptPanel
messages={messages}
runtimeRecords={runtimeRecords}
recording={recording}
/>
)
) : (
<div className="scrollbar-subtle relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
@@ -1496,6 +1511,7 @@ function DebugVisionWorkspace({
view,
onViewChange,
messages,
runtimeRecords,
recording,
camera,
videoStream,
@@ -1507,6 +1523,7 @@ function DebugVisionWorkspace({
view: DebugView;
onViewChange: (view: DebugView) => void;
messages: ChatMessage[];
runtimeRecords: LiveRuntimeRecord[];
recording: boolean;
camera: CameraPreview;
videoStream: MediaStream | null;
@@ -1599,7 +1616,11 @@ function DebugVisionWorkspace({
return (
<div ref={containerRef} className="relative flex min-h-0 flex-1 overflow-hidden">
<DebugTranscriptPanel messages={messages} recording={recording} />
<DebugTranscriptPanel
messages={messages}
runtimeRecords={runtimeRecords}
recording={recording}
/>
<button
type="button"
style={{ left: position.x, top: position.y }}
@@ -1788,19 +1809,13 @@ function DebugConnectionStatus({
);
}
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
function formatMessageTime(iso: string): string {
const d = new Date(iso);
if (Number.isNaN(d.getTime())) return "";
const pad = (n: number) => String(n).padStart(2, "0");
return `${pad(d.getHours())}:${pad(d.getMinutes())}`;
}
function DebugTranscriptPanel({
messages,
runtimeRecords = [],
recording = false,
}: {
messages: ChatMessage[];
runtimeRecords?: LiveRuntimeRecord[];
recording?: boolean;
}) {
const scrollRef = useRef<HTMLDivElement>(null);
@@ -1814,7 +1829,7 @@ function DebugTranscriptPanel({
// Scroll when message list changes (before paint).
useLayoutEffect(() => {
scrollToBottom();
}, [messages, scrollToBottom]);
}, [messages, runtimeRecords, scrollToBottom]);
// Images load after the message row mounts; re-scroll when content height grows.
useEffect(() => {
@@ -1825,7 +1840,7 @@ function DebugTranscriptPanel({
return () => observer.disconnect();
}, [scrollToBottom]);
if (messages.length === 0) {
if (messages.length === 0 && runtimeRecords.length === 0) {
return (
<div
className={[
@@ -1848,59 +1863,36 @@ function DebugTranscriptPanel({
);
}
const timelineItems: Array<DebugTimelineItem & { sequence: number }> = [
...messages.map((message) => ({
id: message.id,
kind: "message" as const,
sequence: message.sequence,
message,
})),
...runtimeRecords.map((record) => ({
id: record.id,
kind: "runtime-record" as const,
sequence: record.sequence,
record: {
id: record.id,
event: record.data,
timestamp: record.timestamp,
},
})),
];
timelineItems.sort((left, right) => left.sequence - right.sequence);
return (
<div
ref={scrollRef}
className="scrollbar-subtle flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4"
>
<div ref={contentRef} className="flex flex-col gap-4">
{messages.map((message) => {
const time = formatMessageTime(message.timestamp);
return message.role === "assistant" ? (
<div
key={message.id}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">
助手{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content || (message.streaming ? "…" : "")}
</div>
</div>
) : (
<div
key={message.id}
className="flex max-w-[88%] flex-col items-end gap-1 self-end"
>
<span className="px-1 text-[11px] text-muted-soft">
我{time ? ` · ${time}` : ""}
</span>
<div
className={[
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
message.attachments?.length ? "p-1" : "px-4 py-2.5",
].join(" ")}
>
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-72 w-full rounded-[0.8rem] object-cover"
onLoad={scrollToBottom}
<div ref={contentRef}>
<DebugTimeline
items={timelineItems}
onContentResize={scrollToBottom}
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
{message.content}
</div>
)}
</div>
</div>
);
})}
</div>
</div>
);
+133
View File
@@ -0,0 +1,133 @@
"use client";
import type React from "react";
import {
RuntimeRecordCard,
type RuntimeRecordContext,
type RuntimeRecordEvent,
} from "@/components/runtime-record-card";
import { cn } from "@/lib/utils";
export type DebugTimelineAttachment = {
id: string;
type: "image";
url: string;
alt: string;
};
export type DebugTimelineMessage = {
id: string;
role: "user" | "assistant";
content: string;
label?: string;
timestamp?: string;
streaming?: boolean;
attachments?: DebugTimelineAttachment[];
};
export type DebugTimelineRecord = {
id: string;
event: RuntimeRecordEvent;
timestamp?: string;
};
export type DebugTimelineItem =
| { id: string; kind: "message"; message: DebugTimelineMessage }
| { id: string; kind: "runtime-record"; record: DebugTimelineRecord }
| { id: string; kind: "content"; content: React.ReactNode };
function formatTimelineTime(iso: string | undefined): string {
if (!iso) return "";
const date = new Date(iso);
if (Number.isNaN(date.getTime())) return "";
const pad = (value: number) => String(value).padStart(2, "0");
return `${pad(date.getHours())}:${pad(date.getMinutes())}`;
}
export function DebugTimeline({
items,
className,
onContentResize,
recordContext = "live",
}: {
items: DebugTimelineItem[];
className?: string;
onContentResize?: () => void;
recordContext?: RuntimeRecordContext;
}) {
return (
<div className={cn("flex flex-col gap-4", className)}>
{items.map((item) => {
if (item.kind === "content") {
return <div key={item.id}>{item.content}</div>;
}
if (item.kind === "runtime-record") {
return (
<div key={item.id} className="w-full py-0.5">
<RuntimeRecordCard
compact
event={item.record.event}
context={recordContext}
timestampLabel={formatTimelineTime(item.record.timestamp)}
/>
</div>
);
}
const message = item.message;
const time = formatTimelineTime(message.timestamp);
const label =
message.label ?? (message.role === "assistant" ? "助手" : "我");
if (message.role === "assistant") {
return (
<div
key={item.id}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">
{label}{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap break-words rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content || (message.streaming ? "…" : "")}
</div>
</div>
);
}
return (
<div
key={item.id}
className="flex max-w-[88%] flex-col items-end gap-1 self-end"
>
<span className="flex items-center gap-2 px-1 text-[11px] text-muted-soft">
{label}{time ? ` · ${time}` : ""}
</span>
<div
className={cn(
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
message.attachments?.length ? "p-1" : "px-4 py-2.5",
)}
>
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-72 w-full rounded-[0.8rem] object-cover"
onLoad={onContentResize}
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
{message.content}
</div>
)}
</div>
</div>
);
})}
</div>
);
}
+20 -248
View File
@@ -1,32 +1,32 @@
"use client";
import {
Activity,
ArrowRight,
Camera,
ChartLine,
CheckCircle2,
ChevronDown,
ChevronUp,
CircleDot,
Eye,
GitBranch,
ImageIcon,
Loader2,
MessageSquareText,
MonitorSmartphone,
MoreHorizontal,
Pause,
Play,
Server,
Trash2,
Wrench,
XCircle,
type LucideIcon,
} from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { SectionAnchorTabs } from "@/components/editor/section-anchor-tabs";
import {
RuntimeRecordCard,
runtimeRecordNumber,
runtimeRecordText,
runtimeRecordValue,
type RuntimeRecordEvent,
type RuntimeRecordNode,
} from "@/components/runtime-record-card";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { DataList, type DataListColumn } from "@/components/ui/data-list";
@@ -115,29 +115,19 @@ function statusLabel(status: string): string {
return "已结束";
}
type TraceEvent = Record<string, unknown>;
type WorkflowNode = { label: string; type: string };
type TraceTone = "default" | "success" | "error" | "user";
type TracePresentation = {
title: string;
description: string;
icon: LucideIcon;
tone: TraceTone;
};
type TraceEvent = RuntimeRecordEvent;
type WorkflowNode = RuntimeRecordNode;
function recordValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
return runtimeRecordValue(value);
}
function textValue(value: unknown): string {
return typeof value === "string" ? value : "";
return runtimeRecordText(value);
}
function numberValue(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
return runtimeRecordNumber(value);
}
function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
@@ -163,196 +153,6 @@ function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
);
}
function nodeLabel(nodes: Map<string, WorkflowNode>, nodeId: unknown): string {
const id = textValue(nodeId);
const node = nodes.get(id);
return node ? `「${node.label}」` : id ? `「${id}」` : "当前节点";
}
function toolLabel(toolType: unknown): string {
if (toolType === "client") return "客户端工具";
if (toolType === "http") return "HTTP 服务端工具";
if (toolType === "mcp") return "MCP 服务端工具";
if (toolType === "system") return "系统工具";
return "工具";
}
function tracePresentation(
event: TraceEvent,
nodes: Map<string, WorkflowNode>,
startedEvent?: TraceEvent,
): TracePresentation {
const eventName = textValue(event.event);
const node = nodeLabel(nodes, event.nodeId);
const outcome = recordValue(event.outcome);
const duration = numberValue(outcome.durationMs);
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
switch (eventName) {
case "node_entered":
return {
title: `进入节点 ${node}`,
description: `节点类型:${textValue(event.nodeType) || nodes.get(textValue(event.nodeId))?.type || "未知"}`,
icon: CircleDot,
tone: "default",
};
case "node_exited":
return {
title: `离开节点 ${node}`,
description: "当前节点处理完成,准备选择下一步。",
icon: ArrowRight,
tone: "default",
};
case "edge_selected":
return {
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)} → ${nodeLabel(nodes, event.targetNodeId)}`,
description: `路由方式:${textValue(event.edgeMode) || "always"}`,
icon: GitBranch,
tone: "default",
};
case "action_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "action_completed":
return {
title: `${tool}执行成功`,
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "action_failed":
case "action_cancelled":
return {
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
description: textValue(recordValue(outcome.error).message) || node,
icon: XCircle,
tone: "error",
};
case "tool_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolName) || textValue(event.functionName) || textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "tool_completed":
return {
title: `${tool}调用成功`,
description: `${textValue(event.toolName) || textValue(event.functionName) || node}${numberValue(event.durationMs) === null ? "" : ` · ${numberValue(event.durationMs)} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "tool_failed":
return {
title: `${tool}调用失败`,
description: textValue(event.error) || textValue(event.toolName) || node,
icon: XCircle,
tone: "error",
};
case "client_tool_started":
return {
title:
event.functionName === "show_message"
? "向客户端显示交互消息"
: "向客户端下发工具调用",
description: textValue(event.functionName) || "客户端工具",
icon: MonitorSmartphone,
tone: "default",
};
case "client_tool_completed": {
const userAction = textValue(event.userAction);
return {
title: userAction === "confirmed" ? "用户已确认" : "客户端交互已完成",
description: userAction
? `用户操作:${userAction}`
: textValue(event.functionName) || textValue(event.status) || "执行成功",
icon: CheckCircle2,
tone: userAction ? "user" : "success",
};
}
case "client_tool_failed":
return {
title: "客户端交互失败",
description: textValue(event.error) || textValue(event.functionName),
icon: XCircle,
tone: "error",
};
case "message_started":
return event.requiresConfirmation
? {
title: "等待用户确认",
description: `${node}向客户端显示了确认消息。`,
icon: MonitorSmartphone,
tone: "user",
}
: {
title: "开始播放固定消息",
description: node,
icon: Activity,
tone: "default",
};
case "message_completed": {
const action = textValue(event.action);
return {
title: action === "confirmed" ? "用户已确认" : "消息步骤已完成",
description: `${node}${action ? ` · 操作:${action}` : ""}`,
icon: CheckCircle2,
tone: action ? "user" : "success",
};
}
case "message_interrupted":
return {
title: "用户输入打断消息",
description: node,
icon: Activity,
tone: "user",
};
case "message_failed":
return {
title: "消息步骤失败",
description: textValue(event.error) || node,
icon: XCircle,
tone: "error",
};
case "variables_updated": {
const names = Array.isArray(event.variableNames)
? event.variableNames.filter((name): name is string => typeof name === "string")
: [];
return {
title: "会话变量已更新",
description: names.length ? names.join("、") : node,
icon: Wrench,
tone: "default",
};
}
case "variables_snapshot":
return {
title: "记录会话变量快照",
description: node,
icon: Wrench,
tone: "default",
};
case "workflow_ended":
return {
title: "工作流已结束",
description: `${node} · ${textValue(event.outcome) || "success"}`,
icon: CheckCircle2,
tone: "success",
};
default:
return {
title: eventName || "运行事件",
description: node,
icon: Activity,
tone: "default",
};
}
}
function artifactUrl(path: string): string {
if (/^(https?:|data:|blob:)/.test(path)) return path;
return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`;
@@ -1237,41 +1037,13 @@ function TimelineTrace({
nodes: Map<string, WorkflowNode>;
startedEvent?: TraceEvent;
}) {
const presentation = tracePresentation(event, nodes, startedEvent);
const Icon = presentation.icon;
const iconClass = {
default: "bg-surface-strong text-muted-foreground",
success: "bg-success/10 text-success",
error: "bg-destructive/10 text-destructive",
user: "bg-primary/10 text-primary",
}[presentation.tone];
return (
<div className="flex gap-3 rounded-xl border border-hairline-soft bg-background/65 p-3">
<div className={cn("flex size-8 shrink-0 items-center justify-center rounded-full", iconClass)}>
<Icon size={15} />
</div>
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1">
<div>
<div className="text-sm font-medium text-foreground">{presentation.title}</div>
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
{presentation.description}
</div>
</div>
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
{formatDate(textValue(event.timestamp))}
</time>
</div>
<details className="mt-2 text-xs text-muted-foreground">
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
详细数据
</summary>
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
{JSON.stringify(event, null, 2)}
</pre>
</details>
</div>
</div>
<RuntimeRecordCard
event={event}
nodes={nodes}
startedEvent={startedEvent}
timestampLabel={formatDate(textValue(event.timestamp))}
showDetails
/>
);
}
@@ -0,0 +1,402 @@
"use client";
import {
Activity,
ArrowRight,
CheckCircle2,
CircleDot,
GitBranch,
MonitorSmartphone,
PhoneOff,
Server,
Undo2,
Volume2,
Wrench,
XCircle,
type LucideIcon,
} from "lucide-react";
import type React from "react";
import { cn } from "@/lib/utils";
export type RuntimeRecordEvent = Record<string, unknown>;
export type RuntimeRecordNode = { label: string; type: string };
export type RuntimeRecordTone = "default" | "success" | "error" | "user";
export type RuntimeRecordContext = "live" | "auto-test";
export type RuntimeRecordPresentation = {
title: string;
description: string;
icon: LucideIcon;
tone: RuntimeRecordTone;
content?: React.ReactNode;
};
export function runtimeRecordValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
export function runtimeRecordText(value: unknown): string {
return typeof value === "string" ? value : "";
}
export function runtimeRecordNumber(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function nodeLabel(
nodes: Map<string, RuntimeRecordNode>,
nodeId: unknown,
): string {
const id = runtimeRecordText(nodeId);
const node = nodes.get(id);
return node ? `「${node.label}」` : id ? `「${id}」` : "当前节点";
}
function toolLabel(toolType: unknown): string {
if (toolType === "client") return "客户端工具";
if (toolType === "http") return "HTTP 服务端工具";
if (toolType === "mcp") return "MCP 服务端工具";
if (toolType === "system") return "系统工具";
return "工具";
}
function messageContent(event: RuntimeRecordEvent): React.ReactNode {
const title = runtimeRecordText(event.messageTitle);
const message = runtimeRecordText(event.messageContent);
const actions = Array.isArray(event.messageActions)
? event.messageActions.map(runtimeRecordValue)
: [];
if (!title && !message && actions.length === 0) return undefined;
return (
<div className="mt-2 rounded-xl border border-hairline bg-canvas-soft px-3 py-2.5">
{title && <div className="text-xs font-medium text-foreground">{title}</div>}
{message && (
<p className="mt-1 whitespace-pre-wrap break-words text-[11px] leading-5 text-muted-foreground">
{message}
</p>
)}
{actions.length > 0 && (
<div className="mt-2 flex flex-wrap gap-1.5">
{actions.map((action, index) => (
<span
key={`${runtimeRecordText(action.id)}-${index}`}
className="rounded-full border border-hairline bg-background px-2 py-0.5 text-[10px] text-muted-foreground"
>
{runtimeRecordText(action.label) || runtimeRecordText(action.id) || "操作"}
</span>
))}
</div>
)}
</div>
);
}
export function runtimeRecordPresentation(
event: RuntimeRecordEvent,
nodes: Map<string, RuntimeRecordNode> = new Map(),
startedEvent?: RuntimeRecordEvent,
context: RuntimeRecordContext = "live",
): RuntimeRecordPresentation {
const eventName = runtimeRecordText(event.event);
const node = nodeLabel(nodes, event.nodeId);
const outcome = runtimeRecordValue(event.outcome);
const duration = runtimeRecordNumber(outcome.durationMs);
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
const reason = runtimeRecordText(event.reason);
switch (eventName) {
case "node_entered":
return {
title: `进入节点 ${node}`,
description: `节点类型:${runtimeRecordText(event.nodeType) || nodes.get(runtimeRecordText(event.nodeId))?.type || "未知"}`,
icon: CircleDot,
tone: "default",
};
case "node_exited":
return {
title: `离开节点 ${node}`,
description: "当前节点处理完成,准备选择下一步。",
icon: ArrowRight,
tone: "default",
};
case "edge_selected":
return {
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)} → ${nodeLabel(nodes, event.targetNodeId)}`,
description: `路由方式:${runtimeRecordText(event.edgeMode) || "always"}`,
icon: GitBranch,
tone: "default",
};
case "action_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${runtimeRecordText(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "action_completed":
return {
title: `${tool}执行成功`,
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "action_failed":
case "action_cancelled":
return {
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
description: runtimeRecordText(runtimeRecordValue(outcome.error).message) || node,
icon: XCircle,
tone: "error",
};
case "tool_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${runtimeRecordText(event.toolName) || runtimeRecordText(event.functionName) || runtimeRecordText(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "tool_completed":
return {
title: `${tool}调用成功`,
description: `${runtimeRecordText(event.toolName) || runtimeRecordText(event.functionName) || node}${runtimeRecordNumber(event.durationMs) === null ? "" : ` · ${runtimeRecordNumber(event.durationMs)} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "tool_failed":
return {
title: `${tool}调用失败`,
description: runtimeRecordText(event.error) || runtimeRecordText(event.toolName) || node,
icon: XCircle,
tone: "error",
};
case "client_tool_started":
return {
title:
event.functionName === "show_message"
? "显示确认弹窗"
: "向客户端下发工具调用",
description:
event.functionName === "show_message"
? "等待用户完成弹窗操作。"
: runtimeRecordText(event.functionName) || "客户端工具",
icon: MonitorSmartphone,
tone: event.functionName === "show_message" ? "user" : "default",
content: messageContent(event),
};
case "client_tool_completed": {
const userAction = runtimeRecordText(event.userAction);
return {
title: userAction === "confirmed" ? "用户已确认弹窗" : "客户端交互已完成",
description: userAction
? `用户操作:${userAction}`
: runtimeRecordText(event.functionName) || runtimeRecordText(event.status) || "执行成功",
icon: CheckCircle2,
tone: userAction ? "user" : "success",
};
}
case "client_tool_failed":
return {
title: "客户端交互失败",
description: runtimeRecordText(event.error) || runtimeRecordText(event.functionName),
icon: XCircle,
tone: "error",
};
case "message_started":
return event.requiresConfirmation
? {
title: "显示确认弹窗",
description:
context === "auto-test"
? "自动测试将模拟确认弹窗操作。"
: event.phase === "opening"
? "开场阶段等待用户完成弹窗操作。"
: `${node}向客户端显示了确认消息。`,
icon: MonitorSmartphone,
tone: "user",
content: messageContent(event),
}
: {
title: "开始播放固定消息",
description: node,
icon: Activity,
tone: "default",
};
case "message_completed": {
const action = runtimeRecordText(event.action);
const automaticallyConfirmed =
context === "auto-test" && action === "confirmed";
return {
title: automaticallyConfirmed
? "自动测试已确认"
: action === "confirmed"
? "用户已确认"
: "消息步骤已完成",
description: automaticallyConfirmed
? "已模拟点击确认,继续执行后续开场流程。"
: `${node}${action ? ` · 操作:${action}` : ""}`,
icon: CheckCircle2,
tone: action ? "user" : "success",
};
}
case "message_interrupted":
return {
title: "用户输入打断消息",
description: node,
icon: Activity,
tone: "user",
};
case "message_failed":
return {
title: "消息步骤失败",
description: runtimeRecordText(event.error) || node,
icon: XCircle,
tone: "error",
};
case "variables_updated": {
const names = Array.isArray(event.variableNames)
? event.variableNames.filter((name): name is string => typeof name === "string")
: [];
return {
title: "会话变量已更新",
description: names.length ? names.join("、") : node,
icon: Wrench,
tone: "default",
};
}
case "variables_snapshot":
return {
title: "记录会话变量快照",
description: node,
icon: Wrench,
tone: "default",
};
case "workflow_ended":
return {
title: "工作流已结束",
description: `${node} · ${runtimeRecordText(event.outcome) || "success"}`,
icon: CheckCircle2,
tone: "success",
};
case "conversation_end_requested":
return {
title: "助手请求结束对话",
description: `${reason || "未记录结束原因"}${event.interruptible === true ? " · 可由用户打断" : " · 不可撤销"}`,
icon: PhoneOff,
tone: "default",
};
case "conversation_end_speaking":
return {
title: "正在播放结束语",
description:
event.interruptible === true
? `播放期间仍可${event.voiceInterruptible === true ? "通过语音或文字" : "通过文字"}打断。`
: "结束语播放完毕后将结束通话。",
icon: Volume2,
tone: "default",
};
case "conversation_end_cancelled":
return {
title: "用户打断,已撤销结束",
description: reason || "对话继续进行。",
icon: Undo2,
tone: "user",
};
case "conversation_ended":
return {
title: "通话已结束",
description: reason || "结束流程已完成。",
icon: PhoneOff,
tone: "success",
};
default:
return {
title: eventName || "运行事件",
description: node,
icon: Activity,
tone: "default",
};
}
}
export function RuntimeRecordCard({
event,
nodes,
startedEvent,
timestampLabel,
showDetails = false,
compact = false,
context = "live",
}: {
event: RuntimeRecordEvent;
nodes?: Map<string, RuntimeRecordNode>;
startedEvent?: RuntimeRecordEvent;
timestampLabel?: string;
showDetails?: boolean;
compact?: boolean;
context?: RuntimeRecordContext;
}) {
const presentation = runtimeRecordPresentation(
event,
nodes,
startedEvent,
context,
);
const Icon = presentation.icon;
const iconClass = {
default: "bg-surface-strong text-muted-foreground",
success: "bg-success/10 text-success",
error: "bg-destructive/10 text-destructive",
user: "bg-primary/10 text-primary",
}[presentation.tone];
return (
<div
className={cn(
"flex gap-3 rounded-xl border border-hairline-soft bg-background/65",
compact ? "p-2.5" : "p-3",
)}
>
<div
className={cn(
"flex shrink-0 items-center justify-center rounded-full",
compact ? "size-7" : "size-8",
iconClass,
)}
>
<Icon size={compact ? 13 : 15} />
</div>
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1">
<div className="min-w-0">
<div className={cn("font-medium text-foreground", compact ? "text-xs" : "text-sm")}>
{presentation.title}
</div>
<div className={cn("mt-0.5 leading-5 text-muted-foreground", compact ? "text-[11px]" : "text-xs")}>
{presentation.description}
</div>
</div>
{timestampLabel && (
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
{timestampLabel}
</time>
)}
</div>
{presentation.content}
{showDetails && (
<details className="mt-2 text-xs text-muted-foreground">
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
详细数据
</summary>
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
{JSON.stringify(event, null, 2)}
</pre>
</details>
)}
</div>
</div>
);
}
+2
View File
@@ -56,6 +56,7 @@ export type BatchTurnResult = {
userInput: string;
assistantReply: string;
toolCalls: BatchToolCallRecord[];
runtimeRecords?: Array<Record<string, unknown>>;
evaluations: BatchEvaluationResult[];
};
@@ -63,6 +64,7 @@ export type BatchStartupResult = {
completed: boolean;
greeting: string;
assistantOutputs: string[];
runtimeRecords?: Array<Record<string, unknown>>;
autoConfirmedMessages: Array<{
title?: string;
message?: string;
+59 -7
View File
@@ -27,6 +27,7 @@ export type CallEndState = {
const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
const SESSION_UPDATE_TIMEOUT_MS = 10_000;
const CALL_END_CANCELLED_FEEDBACK_MS = 1_500;
const MAX_LIVE_RUNTIME_RECORDS = 1_000;
type NetworkStatsSample = {
packetsSent: number;
@@ -62,6 +63,13 @@ export type ChatAttachment = {
alt: string;
};
export type LiveRuntimeRecord = {
id: string;
data: Record<string, unknown>;
timestamp: string;
sequence: number;
};
type AppMessage = Record<string, unknown> & { type?: string };
export type DynamicVariableValue = string | number | boolean;
export type UserInputPart =
@@ -296,6 +304,7 @@ export function useVoicePreview(
const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [runtimeRecords, setRuntimeRecords] = useState<LiveRuntimeRecord[]>([]);
const [sessionVariables, setSessionVariables] = useState<
Record<string, DynamicVariableValue>
>({});
@@ -318,6 +327,7 @@ export function useVoicePreview(
const connectionGenerationRef = useRef(0);
const startingRef = useRef(false);
const messageSeqRef = useRef(0);
const runtimeRecordIdsRef = useRef(new Set<string>());
const networkStatsRef = useRef<NetworkStatsSample | null>(null);
const pendingAssistantTurnsRef = useRef(
new Map<string, { timestamp: string }>(),
@@ -407,13 +417,18 @@ export function useVoicePreview(
onNodeActiveRef.current?.(null);
}, []);
const disconnect = useCallback(() => {
const closeConnection = useCallback(
(options: { preserveTimeline?: boolean } = {}) => {
releaseResources();
setLocalStream(null);
setVideoStream(null);
setRemoteStream(null);
if (!options.preserveTimeline) {
setMessages([]);
setRuntimeRecords([]);
messageSeqRef.current = 0;
runtimeRecordIdsRef.current.clear();
}
pendingAssistantTurnsRef.current.clear();
setError(null);
setMicWarning(null);
@@ -424,7 +439,13 @@ export function useVoicePreview(
voiceInterruptible: false,
});
setStatus("idle");
}, [releaseResources]);
},
[releaseResources],
);
const disconnect = useCallback(() => {
closeConnection();
}, [closeConnection]);
const fail = useCallback((message: string) => {
releaseResources();
@@ -641,6 +662,31 @@ export function useVoicePreview(
},
]),
);
} else if (
msg.type === "workflow-event" &&
typeof msg.event === "string" &&
msg.internalStage !== true
) {
const eventId =
typeof msg.eventId === "string" && msg.eventId
? msg.eventId
: `live-event-${messageSeqRef.current + 1}`;
if (runtimeRecordIdsRef.current.has(eventId)) return;
runtimeRecordIdsRef.current.add(eventId);
messageSeqRef.current += 1;
const timestamp =
typeof msg.timestamp === "string"
? msg.timestamp
: new Date().toISOString();
const record: LiveRuntimeRecord = {
id: eventId,
data: { ...msg, timestamp },
timestamp,
sequence: messageSeqRef.current,
};
setRuntimeRecords((previous) =>
[...previous, record].slice(-MAX_LIVE_RUNTIME_RECORDS),
);
} else if (msg.type === "node-active" && typeof msg.nodeId === "string") {
onNodeActiveRef.current?.(msg.nodeId);
} else if (msg.type === "workflow-variables") {
@@ -676,9 +722,11 @@ export function useVoicePreview(
} else if (msg.type === "call-ended") {
endedByServerRef.current = true;
setCallEnded(true);
disconnect();
// Keep the completed timeline visible in the Debug Drawer. Starting a
// new session (or manually disconnecting) still clears it normally.
closeConnection({ preserveTimeline: true });
}
}, [disconnect, dispatchClientTool]);
}, [closeConnection, dispatchClientTool]);
const connect = useCallback(async (options: ConnectOptions = {}) => {
if (startingRef.current || transportRef.current) return;
@@ -695,6 +743,8 @@ export function useVoicePreview(
setError(null);
setMicWarning(null);
setMessages([]);
setRuntimeRecords([]);
runtimeRecordIdsRef.current.clear();
setSessionVariables(publicVariableSnapshot(options.dynamicVariables ?? {}));
pendingAssistantTurnsRef.current.clear();
setCallEnded(false);
@@ -722,8 +772,9 @@ export function useVoicePreview(
onConnected: () => setStatus("connected"),
onDisconnected: () => {
if (transportRef.current !== transport) return;
if (endedByServerRef.current) disconnect();
else fail("WebRTC 连接已断开。");
if (endedByServerRef.current) {
closeConnection({ preserveTimeline: true });
} else fail("WebRTC 连接已断开。");
},
onTrackStarted: (track) => {
if (track.kind !== "audio") return;
@@ -811,7 +862,7 @@ export function useVoicePreview(
}, [
assistantId,
applyOutputDevice,
disconnect,
closeConnection,
fail,
handleAppMessage,
refreshDevices,
@@ -1055,6 +1106,7 @@ export function useVoicePreview(
videoStream,
remoteStream,
messages,
runtimeRecords,
sessionVariables,
clientTools,
callEnded,