feat: unify debug runtime timelines

This commit is contained in:
Xin Wang
2026-08-13 09:30:43 +08:00
parent 7c18cea16d
commit eb111a3890
21 changed files with 1380 additions and 533 deletions
+26
View File
@@ -231,6 +231,22 @@ class PromptBrain(BaseBrain):
speech, speech,
opening_message, opening_message,
) )
await self._emit_trace(
"message_started",
phase="opening",
hasSpeech=bool(speech),
showsMessage=True,
completionPolicy=MESSAGE_CONFIRMATION,
requiresConfirmation=True,
messageTitle=message_spec.display.title,
messageContent=message_spec.display.message,
messageActions=[
{
"id": "confirmed",
"label": message_spec.display.confirm_label,
}
],
)
message_result = await self._message_stages.run( message_result = await self._message_stages.run(
message_spec, message_spec,
speak=self._speak_opening, speak=self._speak_opening,
@@ -242,10 +258,20 @@ class PromptBrain(BaseBrain):
release_input_on_failure=False, release_input_on_failure=False,
) )
if not message_result.succeeded: if not message_result.succeeded:
await self._emit_trace(
"message_failed",
phase="opening",
error=message_result.error or "开场消息显示失败",
)
await self._fail_opening( await self._fail_opening(
message_result.error or "开场消息显示失败" message_result.error or "开场消息显示失败"
) )
return return
await self._emit_trace(
"message_completed",
phase="opening",
action=message_result.action,
)
context_message = confirmation_context_message( context_message = confirmation_context_message(
message_result, message_result,
source="prompt-opening", source="prompt-opening",
+14
View File
@@ -1557,6 +1557,20 @@ class WorkflowBrain(BaseBrain):
showsMessage=require_confirmation, showsMessage=require_confirmation,
completionPolicy=completion_policy, completionPolicy=completion_policy,
requiresConfirmation=require_confirmation, requiresConfirmation=require_confirmation,
**(
{
"messageTitle": display.title,
"messageContent": display.message,
"messageActions": [
{
"id": "confirmed",
"label": display.confirm_label,
}
],
}
if display is not None
else {}
),
), ),
) )
if result.succeeded: if result.succeeded:
+1
View File
@@ -194,6 +194,7 @@ class ClientToolBroker(FrameProcessor):
toolCallId=tool_call_id, toolCallId=tool_call_id,
functionName=pending.function_name, functionName=pending.function_name,
status=status, status=status,
internalStage=pending.function_name == "show_message",
**({"userAction": user_action} if user_action else {}), **({"userAction": user_action} if user_action else {}),
**( **(
{"error": str(message.get("message") or "客户端工具执行失败")} {"error": str(message.get("message") or "客户端工具执行失败")}
+2
View File
@@ -120,6 +120,8 @@ class ConversationRecorder:
async def _append_workflow_event(self, message: dict) -> None: async def _append_workflow_event(self, message: dict) -> None:
"""Persist bounded control-plane history without creating chat rows.""" """Persist bounded control-plane history without creating chat rows."""
if message.get("internalStage") is True:
return
event_id = str(message.get("eventId") or "") event_id = str(message.get("eventId") or "")
event_name = str(message.get("event") or "") event_name = str(message.get("event") or "")
event_key = f"workflow:{event_id}" event_key = f"workflow:{event_id}"
+63 -4
View File
@@ -5,6 +5,8 @@ from __future__ import annotations
import asyncio import asyncio
from collections.abc import Awaitable, Callable from collections.abc import Awaitable, Callable
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any
from uuid import uuid4
from loguru import logger from loguru import logger
from pipecat.frames.frames import ( from pipecat.frames.frames import (
@@ -14,6 +16,38 @@ from pipecat.frames.frames import (
InterruptionFrame, InterruptionFrame,
) )
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.time import time_now_iso8601
CALL_END_EVENT_BY_STATE = {
"pending": "conversation_end_requested",
"speaking": "conversation_end_speaking",
"cancelled": "conversation_end_cancelled",
"ended": "conversation_ended",
}
def call_end_trace_event(
state: str,
*,
reason: str = "",
interruptible: bool = False,
voice_interruptible: bool = False,
) -> dict[str, Any]:
"""Build one shared record used by live calls and headless tests."""
return {
"type": "workflow-event",
"eventId": f"wfe_{uuid4().hex[:20]}",
"event": CALL_END_EVENT_BY_STATE.get(
state,
"conversation_end_state_changed",
),
"timestamp": time_now_iso8601(),
**({"reason": reason} if reason else {}),
"interruptible": interruptible,
"voiceInterruptible": voice_interruptible,
}
class SpeechPlaybackCompletion: class SpeechPlaybackCompletion:
@@ -76,6 +110,7 @@ class CallEndCoordinator:
): ):
self._queue_end = queue_end self._queue_end = queue_end
self._on_state_change = on_state_change self._on_state_change = on_state_change
self._state_tasks: set[asyncio.Task[None]] = set()
self._ending = False self._ending = False
self._interruptible = False self._interruptible = False
self._armed = False self._armed = False
@@ -86,7 +121,7 @@ class CallEndCoordinator:
self._tracked_speech_completions: set[SpeechPlaybackCompletion] = set() self._tracked_speech_completions: set[SpeechPlaybackCompletion] = set()
self._finish_after_tracked_speech = False self._finish_after_tracked_speech = False
self._finished = False self._finished = False
self._reason = "completed" self._reason = ""
@property @property
def ending(self) -> bool: def ending(self) -> bool:
@@ -106,6 +141,12 @@ class CallEndCoordinator:
"""Whether transport output is currently producing bot speech.""" """Whether transport output is currently producing bot speech."""
return self._speaking return self._speaking
@property
def reason(self) -> str:
"""Current requested ending reason for client feedback and records."""
return self._reason
async def wait_until_silent(self) -> None: async def wait_until_silent(self) -> None:
"""Wait for the transport-owned bot speech boundary.""" """Wait for the transport-owned bot speech boundary."""
await self._speech_stopped.wait() await self._speech_stopped.wait()
@@ -124,6 +165,7 @@ class CallEndCoordinator:
"""Cancel one pending ending when its tool explicitly allows it.""" """Cancel one pending ending when its tool explicitly allows it."""
if not self.interruptible: if not self.interruptible:
return False return False
was_interruptible = self._interruptible
self._ending = False self._ending = False
self._interruptible = False self._interruptible = False
self._armed = False self._armed = False
@@ -132,12 +174,26 @@ class CallEndCoordinator:
# Transport speech state remains authoritative: if audio is currently # Transport speech state remains authoritative: if audio is currently
# playing, the interruption path will still emit BotStoppedSpeakingFrame. # playing, the interruption path will still emit BotStoppedSpeakingFrame.
logger.info("用户打断结束语,已撤销待挂断状态") logger.info("用户打断结束语,已撤销待挂断状态")
self._schedule_state("cancelled") self._schedule_state("cancelled", interruptible=was_interruptible)
return True return True
def _schedule_state(self, state: str) -> None: def _schedule_state(
self,
state: str,
*,
interruptible: bool | None = None,
) -> None:
if self._on_state_change is not None: if self._on_state_change is not None:
asyncio.create_task(self._on_state_change(state, self.interruptible)) task = asyncio.create_task(
self._on_state_change(
state,
self.interruptible
if interruptible is None
else interruptible,
)
)
self._state_tasks.add(task)
task.add_done_callback(self._state_tasks.discard)
def begin_response(self) -> None: def begin_response(self) -> None:
"""Start tracking speech produced by one LLM response.""" """Start tracking speech produced by one LLM response."""
@@ -196,6 +252,9 @@ class CallEndCoordinator:
return return
self._finished = True self._finished = True
self._interruptible = False self._interruptible = False
pending_states = tuple(self._state_tasks)
if pending_states:
await asyncio.gather(*pending_states, return_exceptions=True)
await self._queue_end(self._reason) await self._queue_end(self._reason)
async def observe(self, frame) -> None: async def observe(self, frame) -> None:
+56 -6
View File
@@ -17,6 +17,7 @@ from services.conversation_history import ConversationRecorder
from services.pipecat.call_lifecycle import ( from services.pipecat.call_lifecycle import (
CallEndCoordinator, CallEndCoordinator,
EndCallAfterSpeechProcessor, EndCallAfterSpeechProcessor,
call_end_trace_event,
) )
from services.pipecat.service_factory import ( from services.pipecat.service_factory import (
config_with_resource, config_with_resource,
@@ -208,6 +209,7 @@ def _call_end_state_frame(
*, *,
interruptible: bool, interruptible: bool,
voice_interruptible: bool, voice_interruptible: bool,
reason: str = "",
) -> OutputTransportMessageUrgentFrame: ) -> OutputTransportMessageUrgentFrame:
"""Build the client event used to render the pending call-end state.""" """Build the client event used to render the pending call-end state."""
return OutputTransportMessageUrgentFrame( return OutputTransportMessageUrgentFrame(
@@ -216,10 +218,30 @@ def _call_end_state_frame(
"state": state, "state": state,
"interruptible": interruptible, "interruptible": interruptible,
"voice_interruptible": voice_interruptible, "voice_interruptible": voice_interruptible,
**({"reason": reason} if reason else {}),
} }
) )
def _call_lifecycle_trace_frame(
state: str,
*,
reason: str = "",
interruptible: bool = False,
voice_interruptible: bool = False,
) -> OutputTransportMessageUrgentFrame:
"""Build one persistent runtime record for the call-ending lifecycle."""
return OutputTransportMessageUrgentFrame(
message=call_end_trace_event(
state,
reason=reason,
interruptible=interruptible,
voice_interruptible=voice_interruptible,
)
)
async def run_pipeline( async def run_pipeline(
transport, transport,
cfg: AssistantConfig, cfg: AssistantConfig,
@@ -307,6 +329,12 @@ async def run_pipeline(
if worker is None: if worker is None:
return return
logger.info(f"结束通话: reason={reason}") logger.info(f"结束通话: reason={reason}")
await worker.queue_frame(
_call_lifecycle_trace_frame(
"ended",
reason=reason,
)
)
await worker.queue_frame( await worker.queue_frame(
OutputTransportMessageUrgentFrame( OutputTransportMessageUrgentFrame(
message={"type": "call-ended", "reason": reason} message={"type": "call-ended", "reason": reason}
@@ -318,13 +346,21 @@ async def run_pipeline(
worker = worker_holder.get("worker") worker = worker_holder.get("worker")
if worker is None: if worker is None:
return return
voice_interruptible = interruptible and current_enable_interrupt
await worker.queue_frame( await worker.queue_frame(
_call_end_state_frame( _call_end_state_frame(
state, state,
interruptible=interruptible, interruptible=interruptible,
voice_interruptible=( voice_interruptible=voice_interruptible,
interruptible and current_enable_interrupt reason=call_end.reason,
), )
)
await worker.queue_frame(
_call_lifecycle_trace_frame(
state,
reason=call_end.reason,
interruptible=interruptible,
voice_interruptible=voice_interruptible,
) )
) )
@@ -1043,6 +1079,12 @@ async def run_realtime_pipeline(
if worker is None: if worker is None:
return return
logger.info(f"结束 Realtime 通话: reason={reason}") logger.info(f"结束 Realtime 通话: reason={reason}")
await worker.queue_frame(
_call_lifecycle_trace_frame(
"ended",
reason=reason,
)
)
await worker.queue_frame( await worker.queue_frame(
OutputTransportMessageUrgentFrame( OutputTransportMessageUrgentFrame(
message={"type": "call-ended", "reason": reason} message={"type": "call-ended", "reason": reason}
@@ -1054,13 +1096,21 @@ async def run_realtime_pipeline(
worker = worker_holder.get("worker") worker = worker_holder.get("worker")
if worker is None: if worker is None:
return return
voice_interruptible = interruptible and cfg.enableInterrupt
await worker.queue_frame( await worker.queue_frame(
_call_end_state_frame( _call_end_state_frame(
state, state,
interruptible=interruptible, interruptible=interruptible,
voice_interruptible=( voice_interruptible=voice_interruptible,
interruptible and cfg.enableInterrupt reason=call_end.reason,
), )
)
await worker.queue_frame(
_call_lifecycle_trace_frame(
state,
reason=call_end.reason,
interruptible=interruptible,
voice_interruptible=voice_interruptible,
) )
) )
+2
View File
@@ -163,6 +163,7 @@ class EvaluationEngine:
user_input=expected_turn.user_input, user_input=expected_turn.user_input,
assistant_reply="", assistant_reply="",
tool_calls=[], tool_calls=[],
runtime_records=[],
) )
evaluations = [ evaluations = [
await self._evaluate_behavior(behavior, raw) await self._evaluate_behavior(behavior, raw)
@@ -175,6 +176,7 @@ class EvaluationEngine:
user_input=raw.user_input, user_input=raw.user_input,
assistant_reply=raw.assistant_reply, assistant_reply=raw.assistant_reply,
tool_calls=[item.as_result() for item in raw.tool_calls], tool_calls=[item.as_result() for item in raw.tool_calls],
runtime_records=list(raw.runtime_records),
evaluations=evaluations, evaluations=evaluations,
) )
) )
@@ -302,6 +302,7 @@ class TestRunOrchestrator:
"greeting": "", "greeting": "",
"assistantOutputs": [], "assistantOutputs": [],
"autoConfirmedMessages": [], "autoConfirmedMessages": [],
"runtimeRecords": [],
}, },
"overallCriteria": [], "overallCriteria": [],
} }
@@ -324,6 +325,7 @@ class TestRunOrchestrator:
"autoConfirmedMessages": list( "autoConfirmedMessages": list(
startup.auto_confirmed_messages startup.auto_confirmed_messages
), ),
"runtimeRecords": list(startup.runtime_records),
} }
row.result = result row.result = result
await session.commit() await session.commit()
@@ -346,6 +348,7 @@ class TestRunOrchestrator:
"userInput": turn.user_input, "userInput": turn.user_input,
"assistantReply": turn.assistant_reply, "assistantReply": turn.assistant_reply,
"toolCalls": [item.as_result() for item in turn.tool_calls], "toolCalls": [item.as_result() for item in turn.tool_calls],
"runtimeRecords": list(turn.runtime_records),
"evaluations": [], "evaluations": [],
} }
turns = [item for item in turns if item.get("id") != turn.id] turns = [item for item in turns if item.get("id") != turn.id]
@@ -382,6 +385,7 @@ class TestRunOrchestrator:
"autoConfirmedMessages": list( "autoConfirmedMessages": list(
startup.auto_confirmed_messages startup.auto_confirmed_messages
), ),
"runtimeRecords": list(startup.runtime_records),
}, },
"overallCriteria": [ "overallCriteria": [
item.model_dump(mode="json", by_alias=True) item.model_dump(mode="json", by_alias=True)
+98 -2
View File
@@ -26,6 +26,7 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame, LLMFullResponseStartFrame,
LLMMessagesAppendFrame, LLMMessagesAppendFrame,
ManuallySwitchServiceFrame, ManuallySwitchServiceFrame,
OutputTransportMessageUrgentFrame,
TTSSpeakFrame, TTSSpeakFrame,
) )
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
@@ -44,6 +45,7 @@ from services.knowledge import search as search_knowledge
from services.pipecat.call_lifecycle import ( from services.pipecat.call_lifecycle import (
CallEndCoordinator, CallEndCoordinator,
FixedSpeechPlaybackMarkerFrame, FixedSpeechPlaybackMarkerFrame,
call_end_trace_event,
) )
from services.pipecat.processors import ( from services.pipecat.processors import (
KnowledgeRetrievalProcessor, KnowledgeRetrievalProcessor,
@@ -102,6 +104,7 @@ class RawTurnResult:
user_input: str user_input: str
assistant_reply: str assistant_reply: str
tool_calls: list[RawToolCall] tool_calls: list[RawToolCall]
runtime_records: list[dict[str, Any]] = field(default_factory=list)
@dataclass @dataclass
@@ -111,6 +114,7 @@ class TextStartupResult:
greeting: str = "" greeting: str = ""
assistant_outputs: list[str] = field(default_factory=list) assistant_outputs: list[str] = field(default_factory=list)
auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list) auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list)
runtime_records: list[dict[str, Any]] = field(default_factory=list)
@dataclass @dataclass
@@ -148,6 +152,30 @@ class _ResponseWindow:
return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools
def _opening_speech_from_message(message: Any) -> str:
"""Extract only the transcript emitted by Prompt's opening Message stage."""
if (
not isinstance(message, dict)
or message.get("type") != "transcript"
or message.get("source") != "prompt-opening-speech"
):
return ""
return str(message.get("content") or "").strip()
def _runtime_record_from_message(message: Any) -> dict[str, Any] | None:
"""Keep the same runtime-record envelope used by live conversation history."""
if (
not isinstance(message, dict)
or message.get("type") != "workflow-event"
or message.get("internalStage") is True
):
return None
return dict(message)
class _TextCaptureProcessor(FrameProcessor): class _TextCaptureProcessor(FrameProcessor):
"""Capture assistant/tool output and expose a reliable response boundary.""" """Capture assistant/tool output and expose a reliable response boundary."""
@@ -156,6 +184,8 @@ class _TextCaptureProcessor(FrameProcessor):
self.context = context self.context = context
self.window = _ResponseWindow() self.window = _ResponseWindow()
self.outputs: list[str] = [] self.outputs: list[str] = []
self.opening_speech_outputs: list[str] = []
self.runtime_records: list[dict[str, Any]] = []
self.tool_calls: list[RawToolCall] = [] self.tool_calls: list[RawToolCall] = []
self._tool_by_id: dict[str, RawToolCall] = {} self._tool_by_id: dict[str, RawToolCall] = {}
self.error: TestExecutionError | None = None self.error: TestExecutionError | None = None
@@ -195,6 +225,14 @@ class _TextCaptureProcessor(FrameProcessor):
await self.on_fixed_speech_complete() await self.on_fixed_speech_complete()
return return
if isinstance(frame, OutputTransportMessageUrgentFrame):
opening_speech = _opening_speech_from_message(frame.message)
if opening_speech:
self.opening_speech_outputs.append(opening_speech)
runtime_record = _runtime_record_from_message(frame.message)
if runtime_record is not None:
self.runtime_records.append(runtime_record)
if isinstance(frame, FixedSpeechPlaybackMarkerFrame): if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
await frame.completion.mark_played() await frame.completion.mark_played()
self.window.touch() self.window.touch()
@@ -248,6 +286,27 @@ class _TextCaptureProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
def _resolve_startup_greeting(
shared_greeting: str,
opening_speech_outputs: list[str],
) -> str:
"""Return the opening speech shown by the headless run.
Prompt confirmation mode deliberately suppresses the shared greeting
because its atomic Message stage owns both speech and the dialog. In text
tests the fixed speech transcript carries a dedicated source marker, so
preserve that exact output instead of guessing from unrelated fixed speech.
"""
greeting = shared_greeting.strip()
if greeting:
return greeting
return next(
(text.strip() for text in opening_speech_outputs if text.strip()),
"",
)
def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]: def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
messages: list[dict[str, Any]] = [] messages: list[dict[str, Any]] = []
latest_call_by_name: dict[str, str] = {} latest_call_by_name: dict[str, str] = {}
@@ -494,9 +553,34 @@ class TextPipelineRunner:
async def queue_call_end(reason: str) -> None: async def queue_call_end(reason: str) -> None:
logger.debug(f"文本测试收到会话结束请求: {reason}") logger.debug(f"文本测试收到会话结束请求: {reason}")
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message=call_end_trace_event(
"ended",
reason=reason,
)
)
)
await worker.queue_frame(EndFrame(reason=reason)) await worker.queue_frame(EndFrame(reason=reason))
call_end = CallEndCoordinator(queue_call_end) async def record_call_end_state(
state: str,
interruptible: bool,
) -> None:
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message=call_end_trace_event(
state,
reason=call_end.reason,
interruptible=interruptible,
)
)
)
call_end = CallEndCoordinator(
queue_call_end,
on_state_change=record_call_end_state,
)
async def finish_fixed_speech_if_ending() -> None: async def finish_fixed_speech_if_ending() -> None:
if call_end.ending: if call_end.ending:
@@ -667,6 +751,7 @@ class TextPipelineRunner:
runner_task: asyncio.Task[None] | None = None runner_task: asyncio.Task[None] | None = None
try: try:
startup_runtime_record_marker = len(capture.runtime_records)
await brain.run_preflight() await brain.run_preflight()
if executor.unmocked_calls: if executor.unmocked_calls:
raise TestExecutionError( raise TestExecutionError(
@@ -693,12 +778,19 @@ class TextPipelineRunner:
runner_task, runner_task,
marker, marker,
) )
startup_greeting = _resolve_startup_greeting(
greeting,
capture.opening_speech_outputs,
)
startup = TextStartupResult( startup = TextStartupResult(
greeting=greeting, greeting=startup_greeting,
assistant_outputs=list(capture.outputs), assistant_outputs=list(capture.outputs),
auto_confirmed_messages=list( auto_confirmed_messages=list(
text_client_tools.auto_confirmed_messages text_client_tools.auto_confirmed_messages
), ),
runtime_records=list(
capture.runtime_records[startup_runtime_record_marker:]
),
) )
if on_startup_complete is not None: if on_startup_complete is not None:
await on_startup_complete(startup) await on_startup_complete(startup)
@@ -775,6 +867,7 @@ class TextPipelineRunner:
activity_marker = capture.window.activity_count activity_marker = capture.window.activity_count
output_marker = len(capture.outputs) output_marker = len(capture.outputs)
tool_marker = len(capture.tool_calls) tool_marker = len(capture.tool_calls)
runtime_record_marker = len(capture.runtime_records)
await worker.queue_frame( await worker.queue_frame(
LLMMessagesAppendFrame( LLMMessagesAppendFrame(
messages=[{"role": "user", "content": turn.user_input}], messages=[{"role": "user", "content": turn.user_input}],
@@ -795,4 +888,7 @@ class TextPipelineRunner:
user_input=turn.user_input, user_input=turn.user_input,
assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(), assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
tool_calls=list(capture.tool_calls[tool_marker:]), tool_calls=list(capture.tool_calls[tool_marker:]),
runtime_records=list(
capture.runtime_records[runtime_record_marker:]
),
) )
+2
View File
@@ -364,6 +364,7 @@ class BatchTurnResult(CamelModel):
user_input: str user_input: str
assistant_reply: str assistant_reply: str
tool_calls: list[BatchToolCallRecord] = Field(default_factory=list) tool_calls: list[BatchToolCallRecord] = Field(default_factory=list)
runtime_records: list[dict[str, Any]] = Field(default_factory=list)
evaluations: list[BatchEvaluationResult] = Field(default_factory=list) evaluations: list[BatchEvaluationResult] = Field(default_factory=list)
@@ -372,6 +373,7 @@ class BatchStartupResult(CamelModel):
greeting: str = "" greeting: str = ""
assistant_outputs: list[str] = Field(default_factory=list) assistant_outputs: list[str] = Field(default_factory=list)
auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list) auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list)
runtime_records: list[dict[str, Any]] = Field(default_factory=list)
class BatchRunCaseOut(CamelModel): class BatchRunCaseOut(CamelModel):
+18
View File
@@ -521,6 +521,24 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertIn("点击“确认”", confirmation_messages[0]["content"]) self.assertIn("点击“确认”", confirmation_messages[0]["content"])
self.assertIn("弹窗标题:重要提示", confirmation_messages[0]["content"]) self.assertIn("弹窗标题:重要提示", confirmation_messages[0]["content"])
self.assertIn("弹窗内容:请确认已阅读。", confirmation_messages[0]["content"]) self.assertIn("弹窗内容:请确认已阅读。", confirmation_messages[0]["content"])
message_traces = [
frame.message
for frame in queued
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "workflow-event"
and frame.message.get("event")
in {"message_started", "message_completed"}
]
self.assertEqual(
[event["event"] for event in message_traces],
["message_started", "message_completed"],
)
self.assertEqual(message_traces[0]["messageTitle"], "重要提示")
self.assertEqual(
message_traces[0]["messageContent"],
"请确认已阅读。",
)
self.assertEqual(message_traces[1]["action"], "confirmed")
self.assertEqual( self.assertEqual(
sum(isinstance(frame, LLMRunFrame) for frame in queued), sum(isinstance(frame, LLMRunFrame) for frame in queued),
1, 1,
+17 -1
View File
@@ -11,12 +11,27 @@ from pipecat.frames.frames import (
) )
from services.pipecat.call_lifecycle import ( from services.pipecat.call_lifecycle import (
CallEndCoordinator, CallEndCoordinator,
call_end_trace_event,
playback_marker_for, playback_marker_for,
) )
from services.pipecat.processors import CallEndingUserMuteStrategy from services.pipecat.processors import CallEndingUserMuteStrategy
class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase): class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
def test_shared_trace_event_uses_runtime_record_contract(self):
event = call_end_trace_event(
"pending",
reason="用户已完成咨询",
interruptible=True,
voice_interruptible=True,
)
self.assertEqual(event["type"], "workflow-event")
self.assertEqual(event["event"], "conversation_end_requested")
self.assertEqual(event["reason"], "用户已完成咨询")
self.assertTrue(event["interruptible"])
self.assertTrue(event["voiceInterruptible"])
async def asyncSetUp(self): async def asyncSetUp(self):
self.reasons: list[str] = [] self.reasons: list[str] = []
self.states: list[tuple[str, bool]] = [] self.states: list[tuple[str, bool]] = []
@@ -87,7 +102,7 @@ class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
self.assertTrue(self.coordinator.accepts_input) self.assertTrue(self.coordinator.accepts_input)
self.assertTrue(self.coordinator.cancel_pending_end()) self.assertTrue(self.coordinator.cancel_pending_end())
await asyncio.sleep(0) await asyncio.sleep(0)
self.assertEqual(self.states[-1], ("cancelled", False)) self.assertEqual(self.states[-1], ("cancelled", True))
self.assertFalse(self.coordinator.ending) self.assertFalse(self.coordinator.ending)
self.assertTrue(self.coordinator.accepts_input) self.assertTrue(self.coordinator.accepts_input)
self.assertTrue(self.coordinator.speaking) self.assertTrue(self.coordinator.speaking)
@@ -95,6 +110,7 @@ class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
await self.coordinator.observe(BotStoppedSpeakingFrame()) await self.coordinator.observe(BotStoppedSpeakingFrame())
self.assertFalse(self.coordinator.speaking) self.assertFalse(self.coordinator.speaking)
self.assertEqual(self.reasons, []) self.assertEqual(self.reasons, [])
self.assertEqual(self.coordinator.reason, "user_goodbye")
async def test_committed_end_rejects_cancellation(self): async def test_committed_end_rejects_cancellation(self):
self.coordinator.begin("required_shutdown") self.coordinator.begin("required_shutdown")
@@ -144,6 +144,22 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(saved["sessionId"], "conv_test") self.assertEqual(saved["sessionId"], "conv_test")
self.assertEqual(saved["sequence"], 1) self.assertEqual(saved["sequence"], 1)
async def test_internal_message_stage_transport_trace_is_not_persisted_twice(self):
recorder = ConversationRecorder("conv_test")
event = {
"type": "workflow-event",
"eventId": "wfe_internal",
"event": "client_tool_completed",
"functionName": "show_message",
"internalStage": True,
}
with patch("services.conversation_history.SessionLocal") as session_local:
await recorder.record_transport_message(event)
session_local.assert_not_called()
self.assertEqual(recorder._trace_sequence, 0)
async def test_image_is_persisted_as_message_with_artifact(self): async def test_image_is_persisted_as_message_with_artifact(self):
conversation = SimpleNamespace(message_count=0) conversation = SimpleNamespace(message_count=0)
+91 -1
View File
@@ -10,10 +10,13 @@ from services.runtime_variables import DynamicVariableStore, prepare_dynamic_con
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from services.test_runs.text_runner import ( from services.test_runs.text_runner import (
_TextCaptureProcessor, _TextCaptureProcessor,
_opening_speech_from_message,
_resolve_startup_greeting,
_runtime_record_from_message,
_wait_for_startup, _wait_for_startup,
_wait_for_quiescence, _wait_for_quiescence,
) )
from test_schemas import BatchRunCreate from test_schemas import BatchRunCreate, BatchStartupResult
class FakeLLM: class FakeLLM:
@@ -126,6 +129,25 @@ class DebugAutoTestContractTests(unittest.TestCase):
"你好,测试用户", "你好,测试用户",
) )
def test_startup_result_exposes_real_runtime_records(self):
result = BatchStartupResult.model_validate(
{
"completed": True,
"runtimeRecords": [
{
"type": "workflow-event",
"eventId": "wfe_opening",
"event": "message_started",
}
],
}
)
self.assertEqual(
result.runtime_records[0]["event"],
"message_started",
)
def test_text_client_port_records_automatic_message_confirmation(self): def test_text_client_port_records_automatic_message_confirmation(self):
port = TextClientToolPort() port = TextClientToolPort()
@@ -142,6 +164,51 @@ class DebugAutoTestContractTests(unittest.TestCase):
[{"title": "重要提示", "message": "请确认"}], [{"title": "重要提示", "message": "请确认"}],
) )
def test_confirmation_owned_speech_is_kept_as_startup_greeting(self):
self.assertEqual(
_resolve_startup_greeting(
"",
["您好,请先阅读弹窗。"],
),
"您好,请先阅读弹窗。",
)
self.assertEqual(
_resolve_startup_greeting(
"普通开场白",
["其他固定播报"],
),
"普通开场白",
)
self.assertEqual(
_resolve_startup_greeting(
"",
[],
),
"",
)
def test_only_prompt_opening_transcript_is_identified_as_greeting(self):
self.assertEqual(
_opening_speech_from_message(
{
"type": "transcript",
"content": "您好,请先阅读弹窗。",
"source": "prompt-opening-speech",
}
),
"您好,请先阅读弹窗。",
)
self.assertEqual(
_opening_speech_from_message(
{
"type": "transcript",
"content": "普通 Workflow 播报",
"source": "workflow-message-speech",
}
),
"",
)
class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase): class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self): async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self):
@@ -180,6 +247,29 @@ class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(params.result["action"], "ending_call") self.assertEqual(params.result["action"], "ending_call")
self.assertEqual(executor_holder["executor"].unmocked_calls, []) self.assertEqual(executor_holder["executor"].unmocked_calls, [])
async def test_text_capture_keeps_runtime_records_for_auto_test_ui(self):
event = {
"type": "workflow-event",
"eventId": "wfe_end",
"event": "conversation_end_requested",
"reason": "测试完成",
}
self.assertEqual(_runtime_record_from_message(event), event)
self.assertIsNone(
_runtime_record_from_message({"type": "call-end-state"})
)
self.assertIsNone(
_runtime_record_from_message(
{
"type": "workflow-event",
"eventId": "wfe_internal",
"event": "client_tool_completed",
"internalStage": True,
}
)
)
async def test_startup_waits_for_queued_entry_reply(self): async def test_startup_waits_for_queued_entry_reply(self):
capture = _TextCaptureProcessor(LLMContext(messages=[])) capture = _TextCaptureProcessor(LLMContext(messages=[]))
@@ -22,6 +22,10 @@ import {
X, X,
} from "lucide-react"; } from "lucide-react";
import {
DebugTimeline,
type DebugTimelineItem,
} from "@/components/debug-timeline";
import { Badge } from "@/components/ui/badge"; import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { import {
@@ -252,17 +256,32 @@ function CasePicker({
export function AutoTestLivePanel({ export function AutoTestLivePanel({
testCase, testCase,
state, state,
onStart,
}: { }: {
testCase: TestCase | null; testCase: TestCase | null;
state: AutoTestRunState; state: AutoTestRunState;
onStart: () => void;
}) { }) {
if (!testCase) { if (!testCase) {
return ( return (
<EmptyPanel <div className="flex min-h-0 flex-1 flex-col items-center justify-center px-6 py-8 text-center">
icon={<FlaskConical size={22} />} <span className="flex h-10 w-10 items-center justify-center rounded-full bg-surface-strong text-muted-soft">
title="选择一个测试用例" <FlaskConical size={19} />
description="运行真实 Pipeline,并在这里查看逐轮回复、工具调用和判断结果。" </span>
/> <div className="mt-3 text-sm font-medium text-foreground">
选择测试用例后开始
</div>
<Button
type="button"
variant="secondary"
className="mt-4 h-10 gap-2 rounded-full px-5"
disabled
title="请先在下方选择测试用例"
>
<Play size={14} />
开始测试
</Button>
</div>
); );
} }
@@ -270,20 +289,20 @@ export function AutoTestLivePanel({
const validationMessage = runnableMessage(testCase); const validationMessage = runnableMessage(testCase);
return ( return (
<div className="flex min-h-0 flex-1 flex-col items-center justify-center overflow-y-auto px-6 py-8 text-center"> <div className="flex min-h-0 flex-1 flex-col items-center justify-center overflow-y-auto px-6 py-8 text-center">
<div className="max-w-sm"> <div className="w-full max-w-sm">
<div className="text-base font-medium text-foreground"> <div className="text-lg font-medium tracking-[-0.01em] text-foreground">
{testCase.name} {testCase.name}
</div> </div>
<div className="mt-2 flex flex-wrap items-center justify-center gap-2"> <div className="mt-2 flex flex-wrap items-center justify-center gap-2 text-xs text-muted-foreground">
<Badge <span>{TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]}</span>
variant="secondary" <span aria-hidden className="text-hairline-strong">·</span>
className="rounded-full px-2 py-0 text-[10px] font-medium" <span>{testCase.turns.length} 轮</span>
> {state.evaluatorName && (
{TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]} <>
</Badge> <span aria-hidden className="text-hairline-strong">·</span>
<span className="text-[11px] text-muted-soft"> <span>{state.evaluatorName}</span>
{testCase.turns.length} 轮 </>
</span> )}
</div> </div>
{testCase.description && ( {testCase.description && (
<p className="mt-3 text-xs leading-5 text-muted-foreground"> <p className="mt-3 text-xs leading-5 text-muted-foreground">
@@ -294,11 +313,22 @@ export function AutoTestLivePanel({
<div className="mt-4 rounded-xl border border-amber-500/20 bg-amber-500/[0.06] px-3 py-2 text-xs leading-5 text-amber-600 dark:text-amber-400"> <div className="mt-4 rounded-xl border border-amber-500/20 bg-amber-500/[0.06] px-3 py-2 text-xs leading-5 text-amber-600 dark:text-amber-400">
{validationMessage} {validationMessage}
</div> </div>
) : state.evaluatorName ? (
<p className="mt-4 text-[11px] text-muted-soft">
评估模型 · {state.evaluatorName}
</p>
) : null} ) : null}
<Button
type="button"
className="mx-auto mt-5 h-10 gap-2 rounded-full px-5"
disabled={Boolean(state.blockedMessage)}
onClick={onStart}
title={state.blockedMessage || "开始测试"}
>
<Play size={14} />
开始测试
</Button>
{state.blockedMessage && !validationMessage && (
<p className="mt-2 text-[10px] leading-4 text-muted-soft">
{state.blockedMessage}
</p>
)}
</div> </div>
</div> </div>
); );
@@ -327,8 +357,11 @@ export function AutoTestLivePanel({
if (startup?.completed) { if (startup?.completed) {
return ( return (
<div className="scrollbar-subtle min-h-0 flex-1 overflow-y-auto px-5 py-4"> <div className="scrollbar-subtle min-h-0 flex-1 overflow-y-auto px-5 py-4">
<StartupDetails startup={startup} /> <AutoTestTimeline
<StartupAssistantMessages startup={startup} /> startup={startup}
turns={[]}
maxTurns={state.maxTurns}
/>
<div className="flex items-center justify-center gap-2 py-4 text-xs text-muted-foreground"> <div className="flex items-center justify-center gap-2 py-4 text-xs text-muted-foreground">
<Loader2 size={13} className="animate-spin" /> <Loader2 size={13} className="animate-spin" />
正在执行第 1 轮固定输入… 正在执行第 1 轮固定输入…
@@ -351,62 +384,11 @@ export function AutoTestLivePanel({
return ( return (
<div className="scrollbar-subtle flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4"> <div className="scrollbar-subtle flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4">
<div className="flex flex-col gap-4"> <div className="flex flex-col gap-4">
{startup?.completed && <StartupDetails startup={startup} />} <AutoTestTimeline
{startup?.completed && <StartupAssistantMessages startup={startup} />} startup={startup}
{turns.map((turn) => ( turns={turns}
<section key={turn.id} className="flex flex-col gap-3"> maxTurns={state.maxTurns}
<div className="flex max-w-[88%] flex-col items-end gap-1 self-end">
<span className="flex items-center gap-2 px-1 text-[11px] text-muted-soft">
<span>固定输入</span>
<span className="rounded-full bg-surface-strong px-2 py-0.5 text-[10px]">
{turn.index + 1} / {state.maxTurns}
</span>
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
{turn.userInput}
</div>
</div>
<div className="flex max-w-[88%] flex-col gap-1 self-start">
<span className="px-1 text-[11px] text-muted-soft">助手</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{turn.assistantReply || "(没有生成文字回复)"}
</div>
</div>
{turn.toolCalls.length > 0 && (
<div className="w-full space-y-2 rounded-xl border border-violet-500/20 bg-violet-500/[0.04] px-3 py-2.5">
<div className="flex items-center gap-1.5 text-[11px] font-medium text-foreground">
<Wrench size={12} />
工具调用
</div>
{turn.toolCalls.map((toolCall) => (
<div key={toolCall.id} className="text-[11px] leading-5">
<code className="text-foreground">{toolCall.functionName}</code>
<span className="ml-2 text-muted-soft">
{toolCall.outcome === "success" ? "成功" : "错误"} ·{" "}
{toolCall.durationMs} ms
</span>
<pre className="mt-1 max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
{toolCall.argumentsJson}
</pre>
</div>
))}
</div>
)}
{turn.evaluations.length > 0 && (
<div className="w-full space-y-2">
{turn.evaluations.map((evaluation) => (
<EvaluationDetails
key={evaluation.id}
evaluation={evaluation}
/> />
))}
</div>
)}
</section>
))}
{state.phase === "running" && ( {state.phase === "running" && (
<div className="flex items-center justify-center gap-2 py-2 text-xs text-muted-foreground"> <div className="flex items-center justify-center gap-2 py-2 text-xs text-muted-foreground">
@@ -446,78 +428,15 @@ export function AutoTestLivePanel({
); );
} }
function StartupDetails({ startup }: { startup: BatchStartupResult }) { function startupAssistantMessages(startup: BatchStartupResult): string[] {
const greeting = startup.greeting.trim(); const greeting = startup.greeting.trim();
const greetingText = normalizeDisplayText(greeting);
return (
<section className="w-full space-y-3 rounded-xl border border-hairline bg-canvas-soft px-3 py-3">
<div className="flex flex-wrap items-start justify-between gap-2">
<div>
<div className="text-xs font-medium text-foreground">开场记录</div>
<p className="mt-0.5 text-[10px] leading-4 text-muted-soft">
已按真实开场流程模拟,不计入脚本轮次断言
</p>
</div>
<Badge
variant="secondary"
className="rounded-full px-2 py-0 text-[10px] font-medium"
>
已完成
</Badge>
</div>
{greeting && (
<StartupTextBlock label="开场白 · 按已播放处理" value={greeting} />
)}
{startup.autoConfirmedMessages.map((message, index) => {
const confirmedAction =
message.actions?.find((action) => action.id === "confirmed") ??
message.actions?.[0];
return (
<div
key={`${index}-${message.title ?? message.message ?? "message"}`}
className="rounded-xl border border-hairline bg-background/70 px-3 py-2.5"
>
<div className="text-[10px] font-medium text-muted-soft">
消息弹窗{startup.autoConfirmedMessages.length > 1 ? ` ${index + 1}` : ""}
</div>
<div className="mt-1 text-xs font-medium text-foreground">
{message.title?.trim() || "未命名消息"}
</div>
<p className="mt-1 whitespace-pre-wrap break-words text-[11px] leading-5 text-muted-foreground">
{message.message?.trim() || "(没有消息正文)"}
</p>
<div className="mt-2 inline-flex items-center gap-1 rounded-full border border-success/20 px-2 py-0.5 text-[10px] text-success">
<Check size={10} />
自动点击“{confirmedAction?.label?.trim() || "确认"}”
</div>
</div>
);
})}
</section>
);
}
function StartupAssistantMessages({
startup,
}: {
startup: BatchStartupResult;
}) {
const openingTexts = new Set(
[
startup.greeting,
...startup.autoConfirmedMessages.map((message) => message.message ?? ""),
]
.map(normalizeDisplayText)
.filter(Boolean),
);
const seen = new Set<string>(); const seen = new Set<string>();
const assistantMessages = startup.assistantOutputs.filter((output) => { const assistantMessages = startup.assistantOutputs.filter((output) => {
const normalizedOutput = normalizeDisplayText(output); const normalizedOutput = normalizeDisplayText(output);
if ( if (
!normalizedOutput || !normalizedOutput ||
openingTexts.has(normalizedOutput) || normalizedOutput === greetingText ||
seen.has(normalizedOutput) seen.has(normalizedOutput)
) { ) {
return false; return false;
@@ -525,43 +444,235 @@ function StartupAssistantMessages({
seen.add(normalizedOutput); seen.add(normalizedOutput);
return true; return true;
}); });
return [greeting, ...assistantMessages].filter(Boolean);
}
if (assistantMessages.length === 0) return null; function legacyStartupRuntimeRecords(
startup: BatchStartupResult,
): Array<Record<string, unknown>> {
return startup.autoConfirmedMessages.flatMap((message, index) => {
const confirmedAction =
message.actions?.find((action) => action.id === "confirmed") ??
message.actions?.[0];
const legacyId = `${index}-${message.title ?? message.message ?? "message"}`;
return [
{
eventId: `legacy-message-started-${legacyId}`,
event: "message_started",
phase: "opening",
requiresConfirmation: true,
messageTitle: message.title?.trim() || "未命名消息",
messageContent: message.message?.trim() || "(没有消息正文)",
messageActions: message.actions ?? [],
},
{
eventId: `legacy-message-completed-${legacyId}`,
event: "message_completed",
phase: "opening",
action: confirmedAction?.id?.trim() || "confirmed",
},
];
});
}
const label = function runtimeRecordKey(
startup.autoConfirmedMessages.length > 0 event: Record<string, unknown>,
? "助手 · 确认后自动输出" prefix: string,
: "助手 · 开场自动输出"; index: number,
): string {
return ( const id = typeof event.eventId === "string" ? event.eventId : "";
<div className="flex w-full flex-col gap-3"> const name = typeof event.event === "string" ? event.event : "record";
{assistantMessages.map((message, index) => ( return id || `${prefix}-${name}-${index}`;
<div
key={`${index}-${message}`}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">{label}</span>
<div className="whitespace-pre-wrap break-words rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.trim()}
</div>
</div>
))}
</div>
);
} }
function normalizeDisplayText(value: string): string { function normalizeDisplayText(value: string): string {
return value.trim().replace(/\s+/g, " "); return value.trim().replace(/\s+/g, " ");
} }
function StartupTextBlock({ label, value }: { label: string; value: string }) { function AutoTestTimeline({
return ( startup,
<div className="rounded-xl border border-hairline bg-background/70 px-3 py-2.5"> turns,
<div className="text-[10px] font-medium text-muted-soft">{label}</div> maxTurns,
<p className="mt-1 whitespace-pre-wrap break-words text-[11px] leading-5 text-muted-foreground"> }: {
{value} startup?: BatchStartupResult;
</p> turns: BatchRunCase["turns"];
maxTurns: number;
}) {
const items: DebugTimelineItem[] = [];
if (startup?.completed) {
items.push({
id: "startup-boundary",
kind: "content",
content: (
<div className="flex items-center gap-2 py-1 text-[11px] text-muted-soft">
<span className="h-px flex-1 bg-hairline" />
<span>开场阶段 · 自动执行,不计入脚本断言</span>
<span className="h-px flex-1 bg-hairline" />
</div> </div>
),
});
const openingMessages = startupAssistantMessages(startup);
const records =
(startup.runtimeRecords?.length ?? 0) > 0
? startup.runtimeRecords ?? []
: legacyStartupRuntimeRecords(startup);
const firstMessageStartedIndex = records.findIndex(
(event) => event.event === "message_started",
);
const firstMessageCompletedIndex = records.findIndex(
(event) => event.event === "message_completed",
);
const appendOpeningMessage = (message: string, index: number) => {
items.push({
id: `startup-message-${index}`,
kind: "message",
message: {
id: `startup-message-${index}`,
role: "assistant",
content: message,
},
});
};
const appendRuntimeRecord = (
event: Record<string, unknown>,
index: number,
) => {
const id = runtimeRecordKey(event, "startup-record", index);
items.push({
id,
kind: "runtime-record",
record: {
id,
event,
timestamp:
typeof event.timestamp === "string" ? event.timestamp : undefined,
},
});
};
if (firstMessageStartedIndex < 0) {
records.forEach(appendRuntimeRecord);
openingMessages.forEach(appendOpeningMessage);
} else {
records.forEach((event, index) => {
appendRuntimeRecord(event, index);
if (index === firstMessageStartedIndex && openingMessages[0]) {
appendOpeningMessage(openingMessages[0], 0);
}
if (index === firstMessageCompletedIndex) {
openingMessages.slice(1).forEach((message, outputIndex) => {
appendOpeningMessage(message, outputIndex + 1);
});
}
});
if (firstMessageCompletedIndex < 0) {
openingMessages.slice(1).forEach((message, outputIndex) => {
appendOpeningMessage(message, outputIndex + 1);
});
}
}
}
turns.forEach((turn) => {
items.push({
id: `turn-user-${turn.id}`,
kind: "message",
message: {
id: `turn-user-${turn.id}`,
role: "user",
label: `固定输入 · ${turn.index + 1} / ${maxTurns}`,
content: turn.userInput,
},
});
items.push({
id: `turn-assistant-${turn.id}`,
kind: "message",
message: {
id: `turn-assistant-${turn.id}`,
role: "assistant",
content: turn.assistantReply || "(没有生成文字回复)",
},
});
turn.runtimeRecords?.forEach((event, index) => {
const id = runtimeRecordKey(event, `turn-record-${turn.id}`, index);
items.push({
id,
kind: "runtime-record",
record: {
id,
event,
timestamp:
typeof event.timestamp === "string" ? event.timestamp : undefined,
},
});
});
if (turn.toolCalls.length > 0) {
items.push({
id: `turn-tools-${turn.id}`,
kind: "content",
content: <AutoTestToolDetails toolCalls={turn.toolCalls} />,
});
}
if (turn.evaluations.length > 0) {
items.push({
id: `turn-evaluations-${turn.id}`,
kind: "content",
content: (
<div className="w-full space-y-2">
{turn.evaluations.map((evaluation) => (
<EvaluationDetails
key={evaluation.id}
evaluation={evaluation}
/>
))}
</div>
),
});
}
});
return <DebugTimeline items={items} recordContext="auto-test" />;
}
function AutoTestToolDetails({
toolCalls,
}: {
toolCalls: BatchRunCase["turns"][number]["toolCalls"];
}) {
return (
<details className="w-full rounded-xl border border-hairline bg-canvas-soft px-3 py-2.5 text-xs">
<summary className="flex cursor-pointer list-none items-center gap-1.5 font-medium text-foreground">
<Wrench size={12} />
工具调用详情
<span className="ml-auto text-[10px] font-normal text-muted-soft">
{toolCalls.length} 次
</span>
</summary>
<div className="mt-2 space-y-3 border-t border-hairline pt-2">
{toolCalls.map((toolCall) => (
<div key={toolCall.id} className="text-[11px] leading-5">
<div>
<code className="text-foreground">{toolCall.functionName}</code>
<span className="ml-2 text-muted-soft">
{toolCall.outcome === "success" ? "成功" : "错误"} ·{" "}
{toolCall.durationMs} ms
</span>
</div>
<div className="mt-1 grid gap-1.5">
<pre className="max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
{toolCall.argumentsJson}
</pre>
<pre className="max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
{toolCall.resultJson}
</pre>
</div>
</div>
))}
</div>
</details>
); );
} }
@@ -671,7 +782,6 @@ export function AutoTestRunBar({
loading, loading,
onSelectCase, onSelectCase,
onClearCase, onClearCase,
onStart,
onStop, onStop,
onRerun, onRerun,
}: { }: {
@@ -682,7 +792,6 @@ export function AutoTestRunBar({
loading: boolean; loading: boolean;
onSelectCase: (id: string) => void; onSelectCase: (id: string) => void;
onClearCase: () => void; onClearCase: () => void;
onStart: () => void;
onStop: () => void; onStop: () => void;
onRerun: () => void; onRerun: () => void;
}) { }) {
@@ -818,17 +927,6 @@ export function AutoTestRunBar({
</button> </button>
)} )}
</div> </div>
<Button
type="button"
size="sm"
className="h-8 shrink-0 gap-1.5 rounded-full px-3"
disabled={Boolean(state.blockedMessage)}
onClick={onStart}
title={state.blockedMessage || "开始测试"}
>
<Play size={13} />
开始测试
</Button>
</div> </div>
{state.blockedMessage && testCase && ( {state.blockedMessage && testCase && (
<p className="truncate px-3 text-[10px] leading-4 text-muted-soft"> <p className="truncate px-3 text-[10px] leading-4 text-muted-soft">
@@ -29,6 +29,10 @@ import {
DebugModeTabs, DebugModeTabs,
useAutoTestRunner, useAutoTestRunner,
} from "@/components/assistant-editor/debug-auto-test"; } from "@/components/assistant-editor/debug-auto-test";
import {
DebugTimeline,
type DebugTimelineItem,
} from "@/components/debug-timeline";
import { NetworkQualityIndicator } from "@/components/network-quality-indicator"; import { NetworkQualityIndicator } from "@/components/network-quality-indicator";
import { ClientMessageDialog } from "@/components/client-message-dialog"; import { ClientMessageDialog } from "@/components/client-message-dialog";
import { AuraVisualizer } from "@/components/ui/aura-visualizer"; import { AuraVisualizer } from "@/components/ui/aura-visualizer";
@@ -60,6 +64,7 @@ import {
type ChatMessage, type ChatMessage,
type CallEndState, type CallEndState,
type ClientToolDefinition, type ClientToolDefinition,
type LiveRuntimeRecord,
type UserInputPart, type UserInputPart,
type VoicePreview, type VoicePreview,
type VoicePreviewStatus, type VoicePreviewStatus,
@@ -383,7 +388,11 @@ function AutoTestPanel({
return ( return (
<div className="flex min-h-0 flex-1 flex-col overflow-hidden"> <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
<div className="flex min-h-0 flex-1 flex-col overflow-hidden"> <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
<AutoTestLivePanel testCase={selectedCase} state={state} /> <AutoTestLivePanel
testCase={selectedCase}
state={state}
onStart={start}
/>
</div> </div>
<div className="shrink-0 border-t border-hairline bg-card p-3"> <div className="shrink-0 border-t border-hairline bg-card p-3">
<AutoTestRunBar <AutoTestRunBar
@@ -394,7 +403,6 @@ function AutoTestPanel({
loading={catalogLoading} loading={catalogLoading}
onSelectCase={selectCase} onSelectCase={selectCase}
onClearCase={clearCase} onClearCase={clearCase}
onStart={start}
onStop={stop} onStop={stop}
onRerun={rerun} onRerun={rerun}
/> />
@@ -895,6 +903,7 @@ function DebugVoicePanel({
localStream, localStream,
remoteStream, remoteStream,
messages, messages,
runtimeRecords,
sendText, sendText,
sendUserInput, sendUserInput,
appendUserImage, appendUserImage,
@@ -924,7 +933,8 @@ function DebugVoicePanel({
const idleOrFailed = status === "idle" || status === "failed"; const idleOrFailed = status === "idle" || status === "failed";
const showIdleHub = const showIdleHub =
idleOrFailed && idleOrFailed &&
(view === "video" || (messages.length === 0 && inChatView)); (view === "video" ||
(messages.length === 0 && runtimeRecords.length === 0 && inChatView));
const startBlockedMessage = !assistantId const startBlockedMessage = !assistantId
? "请先保存助手,再开始对话。" ? "请先保存助手,再开始对话。"
: hasUnsavedChanges : hasUnsavedChanges
@@ -1087,6 +1097,7 @@ function DebugVoicePanel({
view={view} view={view}
onViewChange={onViewChange} onViewChange={onViewChange}
messages={messages} messages={messages}
runtimeRecords={runtimeRecords}
recording={recording} recording={recording}
camera={camera} camera={camera}
videoStream={preview.videoStream} videoStream={preview.videoStream}
@@ -1115,7 +1126,11 @@ function DebugVoicePanel({
connect={startConversation} connect={startConversation}
/> />
) : ( ) : (
<DebugTranscriptPanel messages={messages} recording={recording} /> <DebugTranscriptPanel
messages={messages}
runtimeRecords={runtimeRecords}
recording={recording}
/>
) )
) : ( ) : (
<div className="scrollbar-subtle relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center"> <div className="scrollbar-subtle relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
@@ -1496,6 +1511,7 @@ function DebugVisionWorkspace({
view, view,
onViewChange, onViewChange,
messages, messages,
runtimeRecords,
recording, recording,
camera, camera,
videoStream, videoStream,
@@ -1507,6 +1523,7 @@ function DebugVisionWorkspace({
view: DebugView; view: DebugView;
onViewChange: (view: DebugView) => void; onViewChange: (view: DebugView) => void;
messages: ChatMessage[]; messages: ChatMessage[];
runtimeRecords: LiveRuntimeRecord[];
recording: boolean; recording: boolean;
camera: CameraPreview; camera: CameraPreview;
videoStream: MediaStream | null; videoStream: MediaStream | null;
@@ -1599,7 +1616,11 @@ function DebugVisionWorkspace({
return ( return (
<div ref={containerRef} className="relative flex min-h-0 flex-1 overflow-hidden"> <div ref={containerRef} className="relative flex min-h-0 flex-1 overflow-hidden">
<DebugTranscriptPanel messages={messages} recording={recording} /> <DebugTranscriptPanel
messages={messages}
runtimeRecords={runtimeRecords}
recording={recording}
/>
<button <button
type="button" type="button"
style={{ left: position.x, top: position.y }} style={{ left: position.x, top: position.y }}
@@ -1788,19 +1809,13 @@ function DebugConnectionStatus({
); );
} }
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
function formatMessageTime(iso: string): string {
const d = new Date(iso);
if (Number.isNaN(d.getTime())) return "";
const pad = (n: number) => String(n).padStart(2, "0");
return `${pad(d.getHours())}:${pad(d.getMinutes())}`;
}
function DebugTranscriptPanel({ function DebugTranscriptPanel({
messages, messages,
runtimeRecords = [],
recording = false, recording = false,
}: { }: {
messages: ChatMessage[]; messages: ChatMessage[];
runtimeRecords?: LiveRuntimeRecord[];
recording?: boolean; recording?: boolean;
}) { }) {
const scrollRef = useRef<HTMLDivElement>(null); const scrollRef = useRef<HTMLDivElement>(null);
@@ -1814,7 +1829,7 @@ function DebugTranscriptPanel({
// Scroll when message list changes (before paint). // Scroll when message list changes (before paint).
useLayoutEffect(() => { useLayoutEffect(() => {
scrollToBottom(); scrollToBottom();
}, [messages, scrollToBottom]); }, [messages, runtimeRecords, scrollToBottom]);
// Images load after the message row mounts; re-scroll when content height grows. // Images load after the message row mounts; re-scroll when content height grows.
useEffect(() => { useEffect(() => {
@@ -1825,7 +1840,7 @@ function DebugTranscriptPanel({
return () => observer.disconnect(); return () => observer.disconnect();
}, [scrollToBottom]); }, [scrollToBottom]);
if (messages.length === 0) { if (messages.length === 0 && runtimeRecords.length === 0) {
return ( return (
<div <div
className={[ className={[
@@ -1848,59 +1863,36 @@ function DebugTranscriptPanel({
); );
} }
const timelineItems: Array<DebugTimelineItem & { sequence: number }> = [
...messages.map((message) => ({
id: message.id,
kind: "message" as const,
sequence: message.sequence,
message,
})),
...runtimeRecords.map((record) => ({
id: record.id,
kind: "runtime-record" as const,
sequence: record.sequence,
record: {
id: record.id,
event: record.data,
timestamp: record.timestamp,
},
})),
];
timelineItems.sort((left, right) => left.sequence - right.sequence);
return ( return (
<div <div
ref={scrollRef} ref={scrollRef}
className="scrollbar-subtle flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4" className="scrollbar-subtle flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4"
> >
<div ref={contentRef} className="flex flex-col gap-4"> <div ref={contentRef}>
{messages.map((message) => { <DebugTimeline
const time = formatMessageTime(message.timestamp); items={timelineItems}
return message.role === "assistant" ? ( onContentResize={scrollToBottom}
<div
key={message.id}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">
助手{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content || (message.streaming ? "…" : "")}
</div>
</div>
) : (
<div
key={message.id}
className="flex max-w-[88%] flex-col items-end gap-1 self-end"
>
<span className="px-1 text-[11px] text-muted-soft">
我{time ? ` · ${time}` : ""}
</span>
<div
className={[
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
message.attachments?.length ? "p-1" : "px-4 py-2.5",
].join(" ")}
>
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-72 w-full rounded-[0.8rem] object-cover"
onLoad={scrollToBottom}
/> />
))}
{message.content && (
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
{message.content}
</div>
)}
</div>
</div>
);
})}
</div> </div>
</div> </div>
); );
+133
View File
@@ -0,0 +1,133 @@
"use client";
import type React from "react";
import {
RuntimeRecordCard,
type RuntimeRecordContext,
type RuntimeRecordEvent,
} from "@/components/runtime-record-card";
import { cn } from "@/lib/utils";
export type DebugTimelineAttachment = {
id: string;
type: "image";
url: string;
alt: string;
};
export type DebugTimelineMessage = {
id: string;
role: "user" | "assistant";
content: string;
label?: string;
timestamp?: string;
streaming?: boolean;
attachments?: DebugTimelineAttachment[];
};
export type DebugTimelineRecord = {
id: string;
event: RuntimeRecordEvent;
timestamp?: string;
};
export type DebugTimelineItem =
| { id: string; kind: "message"; message: DebugTimelineMessage }
| { id: string; kind: "runtime-record"; record: DebugTimelineRecord }
| { id: string; kind: "content"; content: React.ReactNode };
function formatTimelineTime(iso: string | undefined): string {
if (!iso) return "";
const date = new Date(iso);
if (Number.isNaN(date.getTime())) return "";
const pad = (value: number) => String(value).padStart(2, "0");
return `${pad(date.getHours())}:${pad(date.getMinutes())}`;
}
export function DebugTimeline({
items,
className,
onContentResize,
recordContext = "live",
}: {
items: DebugTimelineItem[];
className?: string;
onContentResize?: () => void;
recordContext?: RuntimeRecordContext;
}) {
return (
<div className={cn("flex flex-col gap-4", className)}>
{items.map((item) => {
if (item.kind === "content") {
return <div key={item.id}>{item.content}</div>;
}
if (item.kind === "runtime-record") {
return (
<div key={item.id} className="w-full py-0.5">
<RuntimeRecordCard
compact
event={item.record.event}
context={recordContext}
timestampLabel={formatTimelineTime(item.record.timestamp)}
/>
</div>
);
}
const message = item.message;
const time = formatTimelineTime(message.timestamp);
const label =
message.label ?? (message.role === "assistant" ? "助手" : "我");
if (message.role === "assistant") {
return (
<div
key={item.id}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">
{label}{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap break-words rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content || (message.streaming ? "…" : "")}
</div>
</div>
);
}
return (
<div
key={item.id}
className="flex max-w-[88%] flex-col items-end gap-1 self-end"
>
<span className="flex items-center gap-2 px-1 text-[11px] text-muted-soft">
{label}{time ? ` · ${time}` : ""}
</span>
<div
className={cn(
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
message.attachments?.length ? "p-1" : "px-4 py-2.5",
)}
>
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-72 w-full rounded-[0.8rem] object-cover"
onLoad={onContentResize}
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
{message.content}
</div>
)}
</div>
</div>
);
})}
</div>
);
}
+20 -248
View File
@@ -1,32 +1,32 @@
"use client"; "use client";
import { import {
Activity,
ArrowRight,
Camera, Camera,
ChartLine, ChartLine,
CheckCircle2,
ChevronDown, ChevronDown,
ChevronUp, ChevronUp,
CircleDot,
Eye, Eye,
GitBranch, GitBranch,
ImageIcon, ImageIcon,
Loader2, Loader2,
MessageSquareText, MessageSquareText,
MonitorSmartphone,
MoreHorizontal, MoreHorizontal,
Pause, Pause,
Play, Play,
Server,
Trash2, Trash2,
Wrench, Wrench,
XCircle,
type LucideIcon,
} from "lucide-react"; } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { SectionAnchorTabs } from "@/components/editor/section-anchor-tabs"; import { SectionAnchorTabs } from "@/components/editor/section-anchor-tabs";
import {
RuntimeRecordCard,
runtimeRecordNumber,
runtimeRecordText,
runtimeRecordValue,
type RuntimeRecordEvent,
type RuntimeRecordNode,
} from "@/components/runtime-record-card";
import { Badge } from "@/components/ui/badge"; import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { DataList, type DataListColumn } from "@/components/ui/data-list"; import { DataList, type DataListColumn } from "@/components/ui/data-list";
@@ -115,29 +115,19 @@ function statusLabel(status: string): string {
return "已结束"; return "已结束";
} }
type TraceEvent = Record<string, unknown>; type TraceEvent = RuntimeRecordEvent;
type WorkflowNode = { label: string; type: string }; type WorkflowNode = RuntimeRecordNode;
type TraceTone = "default" | "success" | "error" | "user";
type TracePresentation = {
title: string;
description: string;
icon: LucideIcon;
tone: TraceTone;
};
function recordValue(value: unknown): Record<string, unknown> { function recordValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value) return runtimeRecordValue(value);
? (value as Record<string, unknown>)
: {};
} }
function textValue(value: unknown): string { function textValue(value: unknown): string {
return typeof value === "string" ? value : ""; return runtimeRecordText(value);
} }
function numberValue(value: unknown): number | null { function numberValue(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null; return runtimeRecordNumber(value);
} }
function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> { function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
@@ -163,196 +153,6 @@ function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
); );
} }
function nodeLabel(nodes: Map<string, WorkflowNode>, nodeId: unknown): string {
const id = textValue(nodeId);
const node = nodes.get(id);
return node ? `「${node.label}」` : id ? `「${id}」` : "当前节点";
}
function toolLabel(toolType: unknown): string {
if (toolType === "client") return "客户端工具";
if (toolType === "http") return "HTTP 服务端工具";
if (toolType === "mcp") return "MCP 服务端工具";
if (toolType === "system") return "系统工具";
return "工具";
}
function tracePresentation(
event: TraceEvent,
nodes: Map<string, WorkflowNode>,
startedEvent?: TraceEvent,
): TracePresentation {
const eventName = textValue(event.event);
const node = nodeLabel(nodes, event.nodeId);
const outcome = recordValue(event.outcome);
const duration = numberValue(outcome.durationMs);
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
switch (eventName) {
case "node_entered":
return {
title: `进入节点 ${node}`,
description: `节点类型:${textValue(event.nodeType) || nodes.get(textValue(event.nodeId))?.type || "未知"}`,
icon: CircleDot,
tone: "default",
};
case "node_exited":
return {
title: `离开节点 ${node}`,
description: "当前节点处理完成,准备选择下一步。",
icon: ArrowRight,
tone: "default",
};
case "edge_selected":
return {
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)} → ${nodeLabel(nodes, event.targetNodeId)}`,
description: `路由方式:${textValue(event.edgeMode) || "always"}`,
icon: GitBranch,
tone: "default",
};
case "action_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "action_completed":
return {
title: `${tool}执行成功`,
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "action_failed":
case "action_cancelled":
return {
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
description: textValue(recordValue(outcome.error).message) || node,
icon: XCircle,
tone: "error",
};
case "tool_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolName) || textValue(event.functionName) || textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "tool_completed":
return {
title: `${tool}调用成功`,
description: `${textValue(event.toolName) || textValue(event.functionName) || node}${numberValue(event.durationMs) === null ? "" : ` · ${numberValue(event.durationMs)} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "tool_failed":
return {
title: `${tool}调用失败`,
description: textValue(event.error) || textValue(event.toolName) || node,
icon: XCircle,
tone: "error",
};
case "client_tool_started":
return {
title:
event.functionName === "show_message"
? "向客户端显示交互消息"
: "向客户端下发工具调用",
description: textValue(event.functionName) || "客户端工具",
icon: MonitorSmartphone,
tone: "default",
};
case "client_tool_completed": {
const userAction = textValue(event.userAction);
return {
title: userAction === "confirmed" ? "用户已确认" : "客户端交互已完成",
description: userAction
? `用户操作:${userAction}`
: textValue(event.functionName) || textValue(event.status) || "执行成功",
icon: CheckCircle2,
tone: userAction ? "user" : "success",
};
}
case "client_tool_failed":
return {
title: "客户端交互失败",
description: textValue(event.error) || textValue(event.functionName),
icon: XCircle,
tone: "error",
};
case "message_started":
return event.requiresConfirmation
? {
title: "等待用户确认",
description: `${node}向客户端显示了确认消息。`,
icon: MonitorSmartphone,
tone: "user",
}
: {
title: "开始播放固定消息",
description: node,
icon: Activity,
tone: "default",
};
case "message_completed": {
const action = textValue(event.action);
return {
title: action === "confirmed" ? "用户已确认" : "消息步骤已完成",
description: `${node}${action ? ` · 操作:${action}` : ""}`,
icon: CheckCircle2,
tone: action ? "user" : "success",
};
}
case "message_interrupted":
return {
title: "用户输入打断消息",
description: node,
icon: Activity,
tone: "user",
};
case "message_failed":
return {
title: "消息步骤失败",
description: textValue(event.error) || node,
icon: XCircle,
tone: "error",
};
case "variables_updated": {
const names = Array.isArray(event.variableNames)
? event.variableNames.filter((name): name is string => typeof name === "string")
: [];
return {
title: "会话变量已更新",
description: names.length ? names.join("、") : node,
icon: Wrench,
tone: "default",
};
}
case "variables_snapshot":
return {
title: "记录会话变量快照",
description: node,
icon: Wrench,
tone: "default",
};
case "workflow_ended":
return {
title: "工作流已结束",
description: `${node} · ${textValue(event.outcome) || "success"}`,
icon: CheckCircle2,
tone: "success",
};
default:
return {
title: eventName || "运行事件",
description: node,
icon: Activity,
tone: "default",
};
}
}
function artifactUrl(path: string): string { function artifactUrl(path: string): string {
if (/^(https?:|data:|blob:)/.test(path)) return path; if (/^(https?:|data:|blob:)/.test(path)) return path;
return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`; return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`;
@@ -1237,41 +1037,13 @@ function TimelineTrace({
nodes: Map<string, WorkflowNode>; nodes: Map<string, WorkflowNode>;
startedEvent?: TraceEvent; startedEvent?: TraceEvent;
}) { }) {
const presentation = tracePresentation(event, nodes, startedEvent);
const Icon = presentation.icon;
const iconClass = {
default: "bg-surface-strong text-muted-foreground",
success: "bg-success/10 text-success",
error: "bg-destructive/10 text-destructive",
user: "bg-primary/10 text-primary",
}[presentation.tone];
return ( return (
<div className="flex gap-3 rounded-xl border border-hairline-soft bg-background/65 p-3"> <RuntimeRecordCard
<div className={cn("flex size-8 shrink-0 items-center justify-center rounded-full", iconClass)}> event={event}
<Icon size={15} /> nodes={nodes}
</div> startedEvent={startedEvent}
<div className="min-w-0 flex-1"> timestampLabel={formatDate(textValue(event.timestamp))}
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1"> showDetails
<div> />
<div className="text-sm font-medium text-foreground">{presentation.title}</div>
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
{presentation.description}
</div>
</div>
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
{formatDate(textValue(event.timestamp))}
</time>
</div>
<details className="mt-2 text-xs text-muted-foreground">
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
详细数据
</summary>
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
{JSON.stringify(event, null, 2)}
</pre>
</details>
</div>
</div>
); );
} }
@@ -0,0 +1,402 @@
"use client";
import {
Activity,
ArrowRight,
CheckCircle2,
CircleDot,
GitBranch,
MonitorSmartphone,
PhoneOff,
Server,
Undo2,
Volume2,
Wrench,
XCircle,
type LucideIcon,
} from "lucide-react";
import type React from "react";
import { cn } from "@/lib/utils";
export type RuntimeRecordEvent = Record<string, unknown>;
export type RuntimeRecordNode = { label: string; type: string };
export type RuntimeRecordTone = "default" | "success" | "error" | "user";
export type RuntimeRecordContext = "live" | "auto-test";
export type RuntimeRecordPresentation = {
title: string;
description: string;
icon: LucideIcon;
tone: RuntimeRecordTone;
content?: React.ReactNode;
};
export function runtimeRecordValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
export function runtimeRecordText(value: unknown): string {
return typeof value === "string" ? value : "";
}
export function runtimeRecordNumber(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function nodeLabel(
nodes: Map<string, RuntimeRecordNode>,
nodeId: unknown,
): string {
const id = runtimeRecordText(nodeId);
const node = nodes.get(id);
return node ? `「${node.label}」` : id ? `「${id}」` : "当前节点";
}
function toolLabel(toolType: unknown): string {
if (toolType === "client") return "客户端工具";
if (toolType === "http") return "HTTP 服务端工具";
if (toolType === "mcp") return "MCP 服务端工具";
if (toolType === "system") return "系统工具";
return "工具";
}
function messageContent(event: RuntimeRecordEvent): React.ReactNode {
const title = runtimeRecordText(event.messageTitle);
const message = runtimeRecordText(event.messageContent);
const actions = Array.isArray(event.messageActions)
? event.messageActions.map(runtimeRecordValue)
: [];
if (!title && !message && actions.length === 0) return undefined;
return (
<div className="mt-2 rounded-xl border border-hairline bg-canvas-soft px-3 py-2.5">
{title && <div className="text-xs font-medium text-foreground">{title}</div>}
{message && (
<p className="mt-1 whitespace-pre-wrap break-words text-[11px] leading-5 text-muted-foreground">
{message}
</p>
)}
{actions.length > 0 && (
<div className="mt-2 flex flex-wrap gap-1.5">
{actions.map((action, index) => (
<span
key={`${runtimeRecordText(action.id)}-${index}`}
className="rounded-full border border-hairline bg-background px-2 py-0.5 text-[10px] text-muted-foreground"
>
{runtimeRecordText(action.label) || runtimeRecordText(action.id) || "操作"}
</span>
))}
</div>
)}
</div>
);
}
export function runtimeRecordPresentation(
event: RuntimeRecordEvent,
nodes: Map<string, RuntimeRecordNode> = new Map(),
startedEvent?: RuntimeRecordEvent,
context: RuntimeRecordContext = "live",
): RuntimeRecordPresentation {
const eventName = runtimeRecordText(event.event);
const node = nodeLabel(nodes, event.nodeId);
const outcome = runtimeRecordValue(event.outcome);
const duration = runtimeRecordNumber(outcome.durationMs);
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
const reason = runtimeRecordText(event.reason);
switch (eventName) {
case "node_entered":
return {
title: `进入节点 ${node}`,
description: `节点类型:${runtimeRecordText(event.nodeType) || nodes.get(runtimeRecordText(event.nodeId))?.type || "未知"}`,
icon: CircleDot,
tone: "default",
};
case "node_exited":
return {
title: `离开节点 ${node}`,
description: "当前节点处理完成,准备选择下一步。",
icon: ArrowRight,
tone: "default",
};
case "edge_selected":
return {
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)} → ${nodeLabel(nodes, event.targetNodeId)}`,
description: `路由方式:${runtimeRecordText(event.edgeMode) || "always"}`,
icon: GitBranch,
tone: "default",
};
case "action_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${runtimeRecordText(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "action_completed":
return {
title: `${tool}执行成功`,
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "action_failed":
case "action_cancelled":
return {
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
description: runtimeRecordText(runtimeRecordValue(outcome.error).message) || node,
icon: XCircle,
tone: "error",
};
case "tool_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${runtimeRecordText(event.toolName) || runtimeRecordText(event.functionName) || runtimeRecordText(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "tool_completed":
return {
title: `${tool}调用成功`,
description: `${runtimeRecordText(event.toolName) || runtimeRecordText(event.functionName) || node}${runtimeRecordNumber(event.durationMs) === null ? "" : ` · ${runtimeRecordNumber(event.durationMs)} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "tool_failed":
return {
title: `${tool}调用失败`,
description: runtimeRecordText(event.error) || runtimeRecordText(event.toolName) || node,
icon: XCircle,
tone: "error",
};
case "client_tool_started":
return {
title:
event.functionName === "show_message"
? "显示确认弹窗"
: "向客户端下发工具调用",
description:
event.functionName === "show_message"
? "等待用户完成弹窗操作。"
: runtimeRecordText(event.functionName) || "客户端工具",
icon: MonitorSmartphone,
tone: event.functionName === "show_message" ? "user" : "default",
content: messageContent(event),
};
case "client_tool_completed": {
const userAction = runtimeRecordText(event.userAction);
return {
title: userAction === "confirmed" ? "用户已确认弹窗" : "客户端交互已完成",
description: userAction
? `用户操作:${userAction}`
: runtimeRecordText(event.functionName) || runtimeRecordText(event.status) || "执行成功",
icon: CheckCircle2,
tone: userAction ? "user" : "success",
};
}
case "client_tool_failed":
return {
title: "客户端交互失败",
description: runtimeRecordText(event.error) || runtimeRecordText(event.functionName),
icon: XCircle,
tone: "error",
};
case "message_started":
return event.requiresConfirmation
? {
title: "显示确认弹窗",
description:
context === "auto-test"
? "自动测试将模拟确认弹窗操作。"
: event.phase === "opening"
? "开场阶段等待用户完成弹窗操作。"
: `${node}向客户端显示了确认消息。`,
icon: MonitorSmartphone,
tone: "user",
content: messageContent(event),
}
: {
title: "开始播放固定消息",
description: node,
icon: Activity,
tone: "default",
};
case "message_completed": {
const action = runtimeRecordText(event.action);
const automaticallyConfirmed =
context === "auto-test" && action === "confirmed";
return {
title: automaticallyConfirmed
? "自动测试已确认"
: action === "confirmed"
? "用户已确认"
: "消息步骤已完成",
description: automaticallyConfirmed
? "已模拟点击确认,继续执行后续开场流程。"
: `${node}${action ? ` · 操作:${action}` : ""}`,
icon: CheckCircle2,
tone: action ? "user" : "success",
};
}
case "message_interrupted":
return {
title: "用户输入打断消息",
description: node,
icon: Activity,
tone: "user",
};
case "message_failed":
return {
title: "消息步骤失败",
description: runtimeRecordText(event.error) || node,
icon: XCircle,
tone: "error",
};
case "variables_updated": {
const names = Array.isArray(event.variableNames)
? event.variableNames.filter((name): name is string => typeof name === "string")
: [];
return {
title: "会话变量已更新",
description: names.length ? names.join("、") : node,
icon: Wrench,
tone: "default",
};
}
case "variables_snapshot":
return {
title: "记录会话变量快照",
description: node,
icon: Wrench,
tone: "default",
};
case "workflow_ended":
return {
title: "工作流已结束",
description: `${node} · ${runtimeRecordText(event.outcome) || "success"}`,
icon: CheckCircle2,
tone: "success",
};
case "conversation_end_requested":
return {
title: "助手请求结束对话",
description: `${reason || "未记录结束原因"}${event.interruptible === true ? " · 可由用户打断" : " · 不可撤销"}`,
icon: PhoneOff,
tone: "default",
};
case "conversation_end_speaking":
return {
title: "正在播放结束语",
description:
event.interruptible === true
? `播放期间仍可${event.voiceInterruptible === true ? "通过语音或文字" : "通过文字"}打断。`
: "结束语播放完毕后将结束通话。",
icon: Volume2,
tone: "default",
};
case "conversation_end_cancelled":
return {
title: "用户打断,已撤销结束",
description: reason || "对话继续进行。",
icon: Undo2,
tone: "user",
};
case "conversation_ended":
return {
title: "通话已结束",
description: reason || "结束流程已完成。",
icon: PhoneOff,
tone: "success",
};
default:
return {
title: eventName || "运行事件",
description: node,
icon: Activity,
tone: "default",
};
}
}
export function RuntimeRecordCard({
event,
nodes,
startedEvent,
timestampLabel,
showDetails = false,
compact = false,
context = "live",
}: {
event: RuntimeRecordEvent;
nodes?: Map<string, RuntimeRecordNode>;
startedEvent?: RuntimeRecordEvent;
timestampLabel?: string;
showDetails?: boolean;
compact?: boolean;
context?: RuntimeRecordContext;
}) {
const presentation = runtimeRecordPresentation(
event,
nodes,
startedEvent,
context,
);
const Icon = presentation.icon;
const iconClass = {
default: "bg-surface-strong text-muted-foreground",
success: "bg-success/10 text-success",
error: "bg-destructive/10 text-destructive",
user: "bg-primary/10 text-primary",
}[presentation.tone];
return (
<div
className={cn(
"flex gap-3 rounded-xl border border-hairline-soft bg-background/65",
compact ? "p-2.5" : "p-3",
)}
>
<div
className={cn(
"flex shrink-0 items-center justify-center rounded-full",
compact ? "size-7" : "size-8",
iconClass,
)}
>
<Icon size={compact ? 13 : 15} />
</div>
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1">
<div className="min-w-0">
<div className={cn("font-medium text-foreground", compact ? "text-xs" : "text-sm")}>
{presentation.title}
</div>
<div className={cn("mt-0.5 leading-5 text-muted-foreground", compact ? "text-[11px]" : "text-xs")}>
{presentation.description}
</div>
</div>
{timestampLabel && (
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
{timestampLabel}
</time>
)}
</div>
{presentation.content}
{showDetails && (
<details className="mt-2 text-xs text-muted-foreground">
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
详细数据
</summary>
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
{JSON.stringify(event, null, 2)}
</pre>
</details>
)}
</div>
</div>
);
}
+2
View File
@@ -56,6 +56,7 @@ export type BatchTurnResult = {
userInput: string; userInput: string;
assistantReply: string; assistantReply: string;
toolCalls: BatchToolCallRecord[]; toolCalls: BatchToolCallRecord[];
runtimeRecords?: Array<Record<string, unknown>>;
evaluations: BatchEvaluationResult[]; evaluations: BatchEvaluationResult[];
}; };
@@ -63,6 +64,7 @@ export type BatchStartupResult = {
completed: boolean; completed: boolean;
greeting: string; greeting: string;
assistantOutputs: string[]; assistantOutputs: string[];
runtimeRecords?: Array<Record<string, unknown>>;
autoConfirmedMessages: Array<{ autoConfirmedMessages: Array<{
title?: string; title?: string;
message?: string; message?: string;
+59 -7
View File
@@ -27,6 +27,7 @@ export type CallEndState = {
const NETWORK_SAMPLE_INTERVAL_MS = 2_000; const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
const SESSION_UPDATE_TIMEOUT_MS = 10_000; const SESSION_UPDATE_TIMEOUT_MS = 10_000;
const CALL_END_CANCELLED_FEEDBACK_MS = 1_500; const CALL_END_CANCELLED_FEEDBACK_MS = 1_500;
const MAX_LIVE_RUNTIME_RECORDS = 1_000;
type NetworkStatsSample = { type NetworkStatsSample = {
packetsSent: number; packetsSent: number;
@@ -62,6 +63,13 @@ export type ChatAttachment = {
alt: string; alt: string;
}; };
export type LiveRuntimeRecord = {
id: string;
data: Record<string, unknown>;
timestamp: string;
sequence: number;
};
type AppMessage = Record<string, unknown> & { type?: string }; type AppMessage = Record<string, unknown> & { type?: string };
export type DynamicVariableValue = string | number | boolean; export type DynamicVariableValue = string | number | boolean;
export type UserInputPart = export type UserInputPart =
@@ -296,6 +304,7 @@ export function useVoicePreview(
const [videoStream, setVideoStream] = useState<MediaStream | null>(null); const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null); const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
const [messages, setMessages] = useState<ChatMessage[]>([]); const [messages, setMessages] = useState<ChatMessage[]>([]);
const [runtimeRecords, setRuntimeRecords] = useState<LiveRuntimeRecord[]>([]);
const [sessionVariables, setSessionVariables] = useState< const [sessionVariables, setSessionVariables] = useState<
Record<string, DynamicVariableValue> Record<string, DynamicVariableValue>
>({}); >({});
@@ -318,6 +327,7 @@ export function useVoicePreview(
const connectionGenerationRef = useRef(0); const connectionGenerationRef = useRef(0);
const startingRef = useRef(false); const startingRef = useRef(false);
const messageSeqRef = useRef(0); const messageSeqRef = useRef(0);
const runtimeRecordIdsRef = useRef(new Set<string>());
const networkStatsRef = useRef<NetworkStatsSample | null>(null); const networkStatsRef = useRef<NetworkStatsSample | null>(null);
const pendingAssistantTurnsRef = useRef( const pendingAssistantTurnsRef = useRef(
new Map<string, { timestamp: string }>(), new Map<string, { timestamp: string }>(),
@@ -407,13 +417,18 @@ export function useVoicePreview(
onNodeActiveRef.current?.(null); onNodeActiveRef.current?.(null);
}, []); }, []);
const disconnect = useCallback(() => { const closeConnection = useCallback(
(options: { preserveTimeline?: boolean } = {}) => {
releaseResources(); releaseResources();
setLocalStream(null); setLocalStream(null);
setVideoStream(null); setVideoStream(null);
setRemoteStream(null); setRemoteStream(null);
if (!options.preserveTimeline) {
setMessages([]); setMessages([]);
setRuntimeRecords([]);
messageSeqRef.current = 0; messageSeqRef.current = 0;
runtimeRecordIdsRef.current.clear();
}
pendingAssistantTurnsRef.current.clear(); pendingAssistantTurnsRef.current.clear();
setError(null); setError(null);
setMicWarning(null); setMicWarning(null);
@@ -424,7 +439,13 @@ export function useVoicePreview(
voiceInterruptible: false, voiceInterruptible: false,
}); });
setStatus("idle"); setStatus("idle");
}, [releaseResources]); },
[releaseResources],
);
const disconnect = useCallback(() => {
closeConnection();
}, [closeConnection]);
const fail = useCallback((message: string) => { const fail = useCallback((message: string) => {
releaseResources(); releaseResources();
@@ -641,6 +662,31 @@ export function useVoicePreview(
}, },
]), ]),
); );
} else if (
msg.type === "workflow-event" &&
typeof msg.event === "string" &&
msg.internalStage !== true
) {
const eventId =
typeof msg.eventId === "string" && msg.eventId
? msg.eventId
: `live-event-${messageSeqRef.current + 1}`;
if (runtimeRecordIdsRef.current.has(eventId)) return;
runtimeRecordIdsRef.current.add(eventId);
messageSeqRef.current += 1;
const timestamp =
typeof msg.timestamp === "string"
? msg.timestamp
: new Date().toISOString();
const record: LiveRuntimeRecord = {
id: eventId,
data: { ...msg, timestamp },
timestamp,
sequence: messageSeqRef.current,
};
setRuntimeRecords((previous) =>
[...previous, record].slice(-MAX_LIVE_RUNTIME_RECORDS),
);
} else if (msg.type === "node-active" && typeof msg.nodeId === "string") { } else if (msg.type === "node-active" && typeof msg.nodeId === "string") {
onNodeActiveRef.current?.(msg.nodeId); onNodeActiveRef.current?.(msg.nodeId);
} else if (msg.type === "workflow-variables") { } else if (msg.type === "workflow-variables") {
@@ -676,9 +722,11 @@ export function useVoicePreview(
} else if (msg.type === "call-ended") { } else if (msg.type === "call-ended") {
endedByServerRef.current = true; endedByServerRef.current = true;
setCallEnded(true); setCallEnded(true);
disconnect(); // Keep the completed timeline visible in the Debug Drawer. Starting a
// new session (or manually disconnecting) still clears it normally.
closeConnection({ preserveTimeline: true });
} }
}, [disconnect, dispatchClientTool]); }, [closeConnection, dispatchClientTool]);
const connect = useCallback(async (options: ConnectOptions = {}) => { const connect = useCallback(async (options: ConnectOptions = {}) => {
if (startingRef.current || transportRef.current) return; if (startingRef.current || transportRef.current) return;
@@ -695,6 +743,8 @@ export function useVoicePreview(
setError(null); setError(null);
setMicWarning(null); setMicWarning(null);
setMessages([]); setMessages([]);
setRuntimeRecords([]);
runtimeRecordIdsRef.current.clear();
setSessionVariables(publicVariableSnapshot(options.dynamicVariables ?? {})); setSessionVariables(publicVariableSnapshot(options.dynamicVariables ?? {}));
pendingAssistantTurnsRef.current.clear(); pendingAssistantTurnsRef.current.clear();
setCallEnded(false); setCallEnded(false);
@@ -722,8 +772,9 @@ export function useVoicePreview(
onConnected: () => setStatus("connected"), onConnected: () => setStatus("connected"),
onDisconnected: () => { onDisconnected: () => {
if (transportRef.current !== transport) return; if (transportRef.current !== transport) return;
if (endedByServerRef.current) disconnect(); if (endedByServerRef.current) {
else fail("WebRTC 连接已断开。"); closeConnection({ preserveTimeline: true });
} else fail("WebRTC 连接已断开。");
}, },
onTrackStarted: (track) => { onTrackStarted: (track) => {
if (track.kind !== "audio") return; if (track.kind !== "audio") return;
@@ -811,7 +862,7 @@ export function useVoicePreview(
}, [ }, [
assistantId, assistantId,
applyOutputDevice, applyOutputDevice,
disconnect, closeConnection,
fail, fail,
handleAppMessage, handleAppMessage,
refreshDevices, refreshDevices,
@@ -1055,6 +1106,7 @@ export function useVoicePreview(
videoStream, videoStream,
remoteStream, remoteStream,
messages, messages,
runtimeRecords,
sessionVariables, sessionVariables,
clientTools, clientTools,
callEnded, callEnded,