wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Make end call tool can be cancelled

Browse Source
This commit is contained in:
Xin Wang
2026-08-12 09:18:40 +08:00
parent fc026e765e
commit d0bb5b3868
15 changed files with 350 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/brains/base.py
+6 -1
View File
@@ -58,7 +58,12 @@ class CallEndPort(Protocol):
@property
def ending(self) -> bool: ...
def begin(self, reason: str) -> None: ...
@property
def accepts_input(self) -> bool: ...
def begin(self, reason: str, *, interruptible: bool = False) -> None: ...
def cancel_pending_end(self) -> bool: ...
def begin_response(self) -> None: ...
backend/services/brains/prompt_brain.py
+16 -9
View File
@@ -488,15 +488,12 @@ class PromptBrain(BaseBrain):
if content and not interrupted:
self._store.record("agent", content, completed_agent_turn=True)
self._refresh_prompt()
if (
self._waiting_for_generated_end_speech
and self._runtime is not None
and self._runtime.call_end.ending
):
if self._waiting_for_generated_end_speech:
self._waiting_for_generated_end_speech = False
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
if self._runtime is not None and self._runtime.call_end.ending:
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
def _refresh_prompt(self) -> None:
if self._dynamic_enabled and self._runtime is not None:
@@ -594,12 +591,18 @@ class PromptBrain(BaseBrain):
message_type = str(config.get("message_type") or "none")
custom_message = str(config.get("custom_message") or "").strip()
capture_reason = bool(config.get("capture_reason", True))
allow_end_interruption = bool(
config.get("allow_end_interruption", False)
)
async def end_call(params: FunctionCallParams) -> None:
reason = str(params.arguments.get("reason") or "end_call_tool").strip()
uses_custom_message = message_type == "custom" and bool(custom_message)
self._waiting_for_generated_end_speech = not uses_custom_message
runtime.call_end.begin(reason)
runtime.call_end.begin(
reason,
interruptible=allow_end_interruption,
)
await params.result_callback(
{"status": "success", "action": "ending_call"},
properties=FunctionCallResultProperties(run_llm=False),
@@ -610,6 +613,8 @@ class PromptBrain(BaseBrain):
# invoking this tool. Decide at assistant-text-end whether to
# wait for that TTS audio or finish immediately for tool-only calls.
return
if not runtime.call_end.ending:
return
turn_id = uuid4().hex
timestamp = time_now_iso8601()
@@ -634,6 +639,8 @@ class PromptBrain(BaseBrain):
await runtime.queue_frame(
OutputTransportMessageUrgentFrame(message=message)
)
if not runtime.call_end.ending:
return
runtime.call_end.arm_after_speech()
await runtime.queue_frame(
TTSSpeakFrame(custom_message, append_to_context=False)
backend/services/brains/workflow_brain.py
+12 -9
View File
@@ -603,15 +603,12 @@ class WorkflowBrain(BaseBrain):
self._state.consume_user_turn()
if self._engine.node_type(self._state.current_node_id) == "agent":
self._state.status = WorkflowStatus.WAITING_USER
if (
self._waiting_for_generated_end_speech
and self._runtime is not None
and self._runtime.call_end.ending
):
if self._waiting_for_generated_end_speech:
self._waiting_for_generated_end_speech = False
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
if self._runtime is not None and self._runtime.call_end.ending:
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
async def on_realtime_user_speech_started(self) -> None:
if self._realtime_controller is not None:
@@ -1090,13 +1087,19 @@ class WorkflowBrain(BaseBrain):
message_type = str(config.get("message_type") or "none")
custom_message = str(config.get("custom_message") or "").strip()
capture_reason = bool(config.get("capture_reason", True))
allow_end_interruption = bool(
config.get("allow_end_interruption", False)
)
async def handler(args, _flow_manager):
reason = str((args or {}).get("reason") or "end_conversation").strip()
uses_custom_message = message_type == "custom" and bool(custom_message)
self._waiting_for_generated_end_speech = not uses_custom_message
runtime = self._require_runtime()
runtime.call_end.begin(reason)
runtime.call_end.begin(
reason,
interruptible=allow_end_interruption,
)
if uses_custom_message:
await self._queue_visible_speech(
custom_message,
backend/services/pipecat/call_lifecycle.py
+39 -3
View File
@@ -64,11 +64,12 @@ def playback_marker_for(
class CallEndCoordinator:
"""End immediately or after the currently armed closing speech finishes."""
"""Coordinate committed and user-cancellable conversation endings."""
def __init__(self, queue_end: Callable[[str], Awaitable[None]]):
self._queue_end = queue_end
self._ending = False
self._interruptible = False
self._armed = False
self._speaking = False
self._speech_stopped = asyncio.Event()
@@ -83,6 +84,15 @@ class CallEndCoordinator:
def ending(self) -> bool:
return self._ending
@property
def accepts_input(self) -> bool:
"""Keep input open while an interruptible ending is still pending."""
return not self._finished and (not self._ending or self._interruptible)
@property
def interruptible(self) -> bool:
return self._ending and self._interruptible and not self._finished
@property
def speaking(self) -> bool:
"""Whether transport output is currently producing bot speech."""
@@ -92,16 +102,37 @@ class CallEndCoordinator:
"""Wait for the transport-owned bot speech boundary."""
await self._speech_stopped.wait()
def begin(self, reason: str) -> None:
def begin(self, reason: str, *, interruptible: bool = False) -> None:
if self._finished:
return
self._ending = True
self._interruptible = interruptible
self._armed = False
self._finish_after_tracked_speech = False
self._reason = reason or "completed"
def cancel_pending_end(self) -> bool:
"""Cancel one pending ending when its tool explicitly allows it."""
if not self.interruptible:
return False
self._ending = False
self._interruptible = False
self._armed = False
self._finish_after_tracked_speech = False
self._response_speech_started = False
# Transport speech state remains authoritative: if audio is currently
# playing, the interruption path will still emit BotStoppedSpeakingFrame.
logger.info("用户打断结束语,已撤销待挂断状态")
return True
def begin_response(self) -> None:
"""Start tracking speech produced by one LLM response."""
self._response_speech_started = False
def arm_after_speech(self) -> None:
"""Wait for the next observed bot speech to finish."""
if not self._ending:
return
self._armed = True
def track_speech(self) -> SpeechPlaybackCompletion:
@@ -128,12 +159,16 @@ class CallEndCoordinator:
async def arm_after_tracked_speech(self) -> None:
"""Finish after every already queued fixed utterance has played."""
if not self._ending:
return
self._finish_after_tracked_speech = True
if not self._tracked_speech_completions:
await self.finish()
async def finish_after_current_speech(self, *, has_text: bool) -> None:
"""Finish now if speech is absent/done, otherwise wait for its stop."""
if not self._ending:
return
if not has_text:
await self.finish()
return
@@ -143,9 +178,10 @@ class CallEndCoordinator:
self._armed = True
async def finish(self) -> None:
if self._finished:
if self._finished or not self._ending:
return
self._finished = True
self._interruptible = False
await self._queue_end(self._reason)
async def observe(self, frame) -> None:
backend/services/pipecat/pipeline.py
+23 -9
View File
@@ -68,6 +68,7 @@ from services.pipecat.turn_config import (
create_user_turn_strategies,
create_vad_analyzer,
create_vad_params,
is_configured_user_turn_start,
)
from services.pipecat.processors import (
KNOWLEDGE_CONTEXT_MARKER,
@@ -330,6 +331,16 @@ async def run_pipeline(
)
)
input_state = {"enabled": True}
current_enable_interrupt = cfg.enableInterrupt
current_turn_config = dict(cfg.turnConfig)
def is_voice_end_interruption(frame) -> bool:
return (
input_state["enabled"]
and current_enable_interrupt
and is_configured_user_turn_start(frame, current_turn_config)
)
tool_interruption_strategy = ToolInterruptionUserMuteStrategy(
{
tool.function_name: policy_for_tool(tool).execution_mode
@@ -355,7 +366,11 @@ async def run_pipeline(
vad_analyzer=create_vad_analyzer(cfg.turnConfig),
user_mute_strategies=[
CallEndingUserMuteStrategy(
lambda: call_end.ending or not input_state["enabled"]
lambda: (
not call_end.accepts_input or not input_state["enabled"]
),
is_user_interruption=is_voice_end_interruption,
cancel_pending_end=call_end.cancel_pending_end,
),
tool_interruption_strategy,
],
@@ -369,10 +384,11 @@ async def run_pipeline(
assistant_aggregator = PassthroughLLMAssistantAggregator(context)
user_input = UserInputProcessor(
should_ignore_input=lambda: (
call_end.ending
not call_end.accepts_input
or not input_state["enabled"]
or tool_interruption_strategy.is_muted
)
),
cancel_pending_end=call_end.cancel_pending_end,
)
session_update = SessionUpdateProcessor(
brain.on_session_update,
@@ -723,9 +739,6 @@ async def run_pipeline(
**current_voice_services,
},
)
current_enable_interrupt = cfg.enableInterrupt
current_turn_config = dict(cfg.turnConfig)
async def apply_workflow_turn_config(
enable_interrupt: bool,
turn_config: dict[str, Any],
@@ -983,11 +996,12 @@ async def run_realtime_pipeline(
client_tools.set_interrupt_handler(interrupt_for_client_tool_result)
user_input = RealtimeUserInputProcessor(
should_ignore_input=lambda: (
call_end.ending or not input_state["enabled"]
)
not call_end.accepts_input or not input_state["enabled"]
),
cancel_pending_end=call_end.cancel_pending_end,
)
input_gate = RealtimeInputAudioGateProcessor(
lambda: not call_end.ending and input_state["enabled"]
lambda: call_end.accepts_input and input_state["enabled"]
)
dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime)
backend/services/pipecat/processors.py
+31 -6
View File
@@ -230,9 +230,14 @@ def message_text(message: dict) -> str:
class UserInputProcessor(FrameProcessor):
"""Validate app inputs and coordinate their interruption boundary."""
def __init__(self, should_ignore_input: Callable[[], bool] | None = None):
def __init__(
self,
should_ignore_input: Callable[[], bool] | None = None,
cancel_pending_end: Callable[[], bool] | None = None,
):
super().__init__()
self._should_ignore_input = should_ignore_input or (lambda: False)
self._cancel_pending_end = cancel_pending_end or (lambda: False)
self._register_event_handler("on_user_input")
self._register_event_handler("on_client_ready")
@@ -256,6 +261,9 @@ class UserInputProcessor(FrameProcessor):
await self.push_frame(frame, direction)
return
if user_input.run_immediately and user_input.interrupt:
self._cancel_pending_end()
if self._should_ignore_input():
logger.debug("当前会话输入已暂停,忽略用户输入")
await self._emit_result(
@@ -371,15 +379,25 @@ class SessionUpdateProcessor(FrameProcessor):
class CallEndingUserMuteStrategy(BaseUserMuteStrategy):
"""Keep user media muted after an end-call tool starts terminating a call."""
"""Mute committed endings and cancel interruptible ones on user activity."""
def __init__(self, is_call_ending: Callable[[], bool]):
def __init__(
self,
should_mute: Callable[[], bool],
*,
is_user_interruption: Callable[[object], bool] | None = None,
cancel_pending_end: Callable[[], bool] | None = None,
):
super().__init__()
self._is_call_ending = is_call_ending
self._should_mute = should_mute
self._is_user_interruption = is_user_interruption or (lambda _frame: False)
self._cancel_pending_end = cancel_pending_end or (lambda: False)
async def process_frame(self, frame) -> bool:
await super().process_frame(frame)
return self._is_call_ending()
if self._is_user_interruption(frame):
self._cancel_pending_end()
return self._should_mute()
@dataclass
@@ -570,9 +588,14 @@ class RealtimeInputAudioGateProcessor(FrameProcessor):
class RealtimeUserInputProcessor(FrameProcessor):
"""Route text-only user-input messages to a realtime service."""
def __init__(self, should_ignore_input: Callable[[], bool] | None = None):
def __init__(
self,
should_ignore_input: Callable[[], bool] | None = None,
cancel_pending_end: Callable[[], bool] | None = None,
):
super().__init__()
self._should_ignore_input = should_ignore_input or (lambda: False)
self._cancel_pending_end = cancel_pending_end or (lambda: False)
self._register_event_handler("on_user_input")
async def process_frame(self, frame, direction: FrameDirection):
@@ -590,6 +613,8 @@ class RealtimeUserInputProcessor(FrameProcessor):
if user_input is None:
await self.push_frame(frame, direction)
return
if user_input.run_immediately and user_input.interrupt:
self._cancel_pending_end()
if self._should_ignore_input():
await self._emit_error(
user_input.input_id,
backend/services/pipecat/turn_config.py
+9
View File
@@ -7,6 +7,7 @@ from typing import Any
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.frames.frames import TranscriptionFrame, VADUserStartedSpeakingFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import (
LLMUserAggregator,
@@ -100,6 +101,14 @@ def create_user_turn_strategies(
return UserTurnStrategies(start=start, stop=stop)
def is_configured_user_turn_start(frame, turn_config: dict[str, Any]) -> bool:
"""Match the frame used by the configured barge-in start strategy."""
barge_in = _section(turn_config, "barge_in", "bargeIn")
if barge_in.get("strategy", "vad") == "transcription":
return isinstance(frame, TranscriptionFrame) and bool(frame.text.strip())
return isinstance(frame, VADUserStartedSpeakingFrame)
class ConfigurableLLMUserAggregator(LLMUserAggregator):
"""LLM user aggregator with one stable project-level runtime update API.
backend/services/workflow/realtime.py
+15 -5
View File
@@ -235,6 +235,7 @@ class WorkflowRealtimeController:
async def on_user_speech_started(self) -> None:
"""Let an interruptible Message finish its deterministic continuation."""
self._runtime.call_end.cancel_pending_end()
current = self._state.current_node_id
if (
self._engine.node_type(current) != "message"
@@ -263,11 +264,12 @@ class WorkflowRealtimeController:
self._state.consume_user_turn()
if self._engine.node_type(self._state.current_node_id) == "agent":
self._state.status = WorkflowStatus.WAITING_USER
if self._waiting_generated_end and self._runtime.call_end.ending:
if self._waiting_generated_end:
self._waiting_generated_end = False
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
if self._runtime.call_end.ending:
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
async def dispatch_tool(
self,
@@ -759,10 +761,18 @@ class WorkflowRealtimeController:
request_handoff,
)
if kind == "end_conversation":
config = (tool.definition or {}).get("config") or {}
allow_end_interruption = bool(
config.get("allow_end_interruption", False)
)
async def end_conversation(arguments: dict[str, Any]) -> RealtimeToolResult:
reason = str(arguments.get("reason") or "end_conversation")
self._waiting_generated_end = True
self._runtime.call_end.begin(reason)
self._runtime.call_end.begin(
reason,
interruptible=allow_end_interruption,
)
return RealtimeToolResult(
{"status": "success", "action": "ending_call"}
)