From 5bf5987fe40ba83e72283300840186561a49ef14 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 4 Aug 2026 13:29:21 +0800 Subject: [PATCH] feat: add prompt opening behavior modes --- backend/schemas.py | 17 ++- backend/services/brains/base.py | 5 + backend/services/brains/prompt_brain.py | 54 +++++-- backend/services/message_stage.py | 17 +-- backend/services/pipecat/pipeline_events.py | 3 + backend/tests/test_brains.py | 86 ++++++----- backend/tests/test_pipeline_events.py | 5 + backend/tests/test_startup_actions.py | 2 +- .../assistant-editor/prompt-editor.tsx | 138 ++++++++++++++---- .../src/components/pages/AssistantPage.tsx | 3 + frontend/src/lib/api.ts | 2 +- 11 files changed, 235 insertions(+), 97 deletions(-) diff --git a/backend/schemas.py b/backend/schemas.py index 6c49291..99e8f84 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -29,6 +29,7 @@ ToolExecutionMode = Literal["immediate", "async"] ClientToolResponseWaitMode = Literal["timeout", "session"] DynamicVariableType = Literal["string", "number", "boolean"] PromptEntryMode = Literal["wait_user", "generate"] +PromptOpeningMode = Literal["interruptible", "playback", "confirmation"] # 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵 EXTERNAL_TYPES = {"dify", "fastgpt", "opencode"} @@ -107,11 +108,11 @@ class OpeningMessageConfig(CamelModel): title: str = Field(default="重要提示", min_length=1, max_length=120) message: str = Field(min_length=1, max_length=2000) confirm_label: str = Field(default="确认", min_length=1, max_length=40) - skip_speech_on_confirm: bool = True class StartupConfig(CamelModel): execution_mode: Literal["sequential"] = "sequential" + opening_mode: PromptOpeningMode | None = None entry_mode: PromptEntryMode | None = None actions: list[StartupAction] = Field(default_factory=list, max_length=5) opening_message: OpeningMessageConfig | None = None @@ -121,11 +122,22 @@ class StartupConfig(CamelModel): ids = [action.id for action in self.actions] if len(ids) != len(set(ids)): raise ValueError("启动 Action 的 id 不能重复") + legacy_confirmation = ( + self.opening_mode is None and self.opening_message is not None + ) + if self.opening_mode is None: + self.opening_mode = ( + "confirmation" if legacy_confirmation else "interruptible" + ) + if self.opening_mode == "confirmation" and self.opening_message is None: + raise ValueError("确认后继续必须配置开场确认弹窗") + if self.opening_mode != "confirmation": + self.opening_message = None if self.entry_mode is None: # Preserve the behavior introduced before entryMode became an # explicit setting. Newly saved clients always send the field. self.entry_mode = ( - "generate" if self.opening_message is not None else "wait_user" + "generate" if legacy_confirmation else "wait_user" ) return self @@ -191,6 +203,7 @@ class AssistantUpsert(CamelModel): if self.runtime_mode == "realtime" and ( self.startup.actions or self.startup.opening_message is not None + or self.startup.opening_mode != "interruptible" or self.startup.entry_mode != "wait_user" ): raise ValueError("Prompt Realtime 模式暂不支持启动阶段") diff --git a/backend/services/brains/base.py b/backend/services/brains/base.py index ae170b6..6587e01 100644 --- a/backend/services/brains/base.py +++ b/backend/services/brains/base.py @@ -132,6 +132,9 @@ class BaseBrain: async def on_greeting_finished(self) -> None: """Continue startup after the shared greeting has actually played.""" + async def on_interruption_processed(self) -> None: + """Observe that an output interruption reached assistant aggregation.""" + def prepare_greeting_context( self, greeting: str, @@ -214,6 +217,8 @@ class Brain(Protocol): async def on_greeting_finished(self) -> None: ... + async def on_interruption_processed(self) -> None: ... + def prepare_greeting_context( self, greeting: str, diff --git a/backend/services/brains/prompt_brain.py b/backend/services/brains/prompt_brain.py index 0e5784e..b5b18cd 100644 --- a/backend/services/brains/prompt_brain.py +++ b/backend/services/brains/prompt_brain.py @@ -81,7 +81,7 @@ class PromptBrain(BaseBrain): async def greeting(self, cfg: AssistantConfig) -> str: # The built-in opening Message owns the greeting so speech and the # client dialog can start as one atomic stage. - if self._opening_message() is not None: + if self._opening_mode() == "confirmation": return "" return self._render_greeting(cfg) @@ -155,8 +155,19 @@ class PromptBrain(BaseBrain): async def on_connected(self, *, greeting_pending: bool = False) -> None: self._greeting_pending = greeting_pending + should_block_input = ( + self._has_opening_stage() + or ( + greeting_pending + and self._opening_mode() == "playback" + ) + or ( + not greeting_pending + and self._entry_mode() == "generate" + ) + ) if ( - (self._has_opening_stage() or self._entry_mode() == "generate") + should_block_input and self._runtime is not None and self._runtime.set_input_enabled is not None ): @@ -168,6 +179,16 @@ class PromptBrain(BaseBrain): self._greeting_pending = False await self._enter_prompt_if_ready() + async def on_interruption_processed(self) -> None: + if ( + self._greeting_pending + and self._opening_mode() == "interruptible" + ): + # The user's input now owns the first Agent turn. Do not also queue + # an empty-context entry reply when the interrupted greeting stops. + self._entry_dispatched = True + logger.debug("Prompt 可打断开场白已被用户输入接管") + async def on_client_ready(self) -> None: if self._output is not None: await self._output.mark_client_ready() @@ -180,7 +201,11 @@ class PromptBrain(BaseBrain): runtime = self._runtime if runtime is None: raise RuntimeError("PromptBrain 尚未初始化") - opening_message = self._opening_message() + opening_message = ( + self._opening_message() + if self._opening_mode() == "confirmation" + else None + ) opening_actions = self._startup_actions("opening") speech = ( self._render_greeting(self._cfg).strip() @@ -238,6 +263,19 @@ class PromptBrain(BaseBrain): value = startup.get("opening_message", startup.get("openingMessage")) return value if isinstance(value, dict) else None + def _opening_mode(self) -> str: + startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {} + value = str(startup.get("opening_mode", startup.get("openingMode")) or "") + if value in {"interruptible", "playback", "confirmation"}: + return value + # Before openingMode existed, an opening message meant confirmation; + # ordinary Prompt greetings were interruptible. + return ( + "confirmation" + if self._opening_message() is not None + else "interruptible" + ) + def _entry_mode(self) -> str: startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {} value = str(startup.get("entry_mode", startup.get("entryMode")) or "") @@ -245,7 +283,7 @@ class PromptBrain(BaseBrain): return value # Raw runtime configs saved before entryMode existed generated a first # reply after their opening confirmation. Keep that legacy behavior. - return "generate" if self._opening_message() is not None else "wait_user" + return "generate" if self._opening_mode() == "confirmation" else "wait_user" async def _enter_prompt_if_ready(self) -> None: if ( @@ -269,7 +307,7 @@ class PromptBrain(BaseBrain): self._opening_input_blocked = False def _has_opening_stage(self) -> bool: - return self._opening_message() is not None or bool( + return self._opening_mode() == "confirmation" or bool( self._startup_actions("opening") ) @@ -322,12 +360,6 @@ class PromptBrain(BaseBrain): ).strip(), ), completion_policy=MESSAGE_CONFIRMATION, - skip_speech_on_confirmation=bool( - config.get( - "skip_speech_on_confirm", - config.get("skipSpeechOnConfirm", True), - ) - ), ) async def _publish_opening_outcome( diff --git a/backend/services/message_stage.py b/backend/services/message_stage.py index 0791379..923c302 100644 --- a/backend/services/message_stage.py +++ b/backend/services/message_stage.py @@ -38,7 +38,6 @@ class MessageStageSpec: speech: str = "" display: MessageDisplaySpec | None = None completion_policy: MessageCompletionPolicy = MESSAGE_PLAYBACK - skip_speech_on_confirmation: bool = True @dataclass(frozen=True) @@ -55,7 +54,7 @@ class MessageStageRunner: """Run one atomic user-visible message stage. Speech is queued before the client message is dispatched. A confirmation - may interrupt the remaining audio or keep playback as the final gate. A + stage completes when the user confirms, even if audio is still playing. A speech-only stage completes at the real transport playback boundary. """ @@ -118,14 +117,11 @@ class MessageStageRunner: return result action = result.action - # A confirmation may either interrupt the remaining speech or - # acknowledge the dialog while keeping playback as the final gate. + # Confirmation is the gate. It deliberately does not wait for the + # audio completion future, so the user can continue immediately. if ( playback_completion is not None - and ( - not require_confirmation - or not spec.skip_speech_on_confirmation - ) + and not require_confirmation ): await playback_completion @@ -191,10 +187,7 @@ class MessageStageRunner: response_wait_mode=( "session" if require_confirmation else "timeout" ), - interrupt_on_result=( - require_confirmation - and spec.skip_speech_on_confirmation - ), + interrupt_on_result=require_confirmation, ) except ClientToolError as exc: return MessageStageResult( diff --git a/backend/services/pipecat/pipeline_events.py b/backend/services/pipecat/pipeline_events.py index 70220c8..9fd1b89 100644 --- a/backend/services/pipecat/pipeline_events.py +++ b/backend/services/pipecat/pipeline_events.py @@ -161,6 +161,9 @@ def bind_cascade_pipeline_events( @assistant_aggregator.event_handler("on_interruption_processed") async def on_interruption_processed(_aggregator): + brain_handler = getattr(brain, "on_interruption_processed", None) + if callable(brain_handler): + await brain_handler() if client_tools is not None: client_tools.on_interruption_processed() if not pending_user_inputs: diff --git a/backend/tests/test_brains.py b/backend/tests/test_brains.py index a959bd2..0a36eff 100644 --- a/backend/tests/test_brains.py +++ b/backend/tests/test_brains.py @@ -366,6 +366,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): tools=[tool], startup={ "execution_mode": "sequential", + "opening_mode": "confirmation", "entry_mode": "generate", "opening_message": { "title": "重要提示", @@ -485,6 +486,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): type="prompt", greeting="请先确认", startup={ + "opening_mode": "confirmation", "entry_mode": "wait_user", "opening_message": { "title": "重要提示", @@ -527,7 +529,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): cfg = AssistantConfig( type="prompt", greeting="固定开场白", - startup={"entry_mode": "generate"}, + startup={ + "opening_mode": "playback", + "entry_mode": "generate", + }, ) brain = build_brain(cfg) queued = [] @@ -561,45 +566,22 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): 1, ) - async def test_opening_confirmation_can_keep_speech_playing_before_entry(self): + async def test_interruptible_opening_user_input_suppresses_entry_reply(self): cfg = AssistantConfig( type="prompt", - greeting="必须完整播放的开场白", + greeting="可以打断的开场白", startup={ + "opening_mode": "interruptible", "entry_mode": "generate", - "opening_message": { - "title": "重要提示", - "message": "请确认已阅读。", - "skip_speech_on_confirm": False, - }, }, ) brain = build_brain(cfg) queued = [] input_states = [] - message_displayed = asyncio.Event() - - class TrackedCallEnd(FakeCallEnd): - def __init__(self): - super().__init__() - self.playback_completion = None - - def track_speech(self): - self.tracked_speeches += 1 - self.playback_completion = asyncio.get_running_loop().create_future() - return self.playback_completion - - class FakeClientTools: - async def call(self, *_args, **options): - self.options = options - message_displayed.set() - return {"status": "ok", "data": {"action": "confirmed"}} async def queue_frame(frame): queued.append(frame) - call_end = TrackedCallEnd() - client_tools = FakeClientTools() await brain.setup( cfg, BrainRuntime( @@ -608,24 +590,52 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): queue_frame=queue_frame, set_system_prompt=lambda _prompt: None, set_tools=lambda _tools: None, - call_end=call_end, - client_tools=client_tools, + call_end=FakeCallEnd(), set_input_enabled=input_states.append, ), ) - await brain.on_connected(greeting_pending=False) - opening_task = asyncio.create_task(brain.on_client_ready()) - await message_displayed.wait() - await asyncio.sleep(0) + await brain.on_connected(greeting_pending=True) + self.assertEqual(input_states, []) + + await brain.on_interruption_processed() + await brain.on_greeting_finished() - self.assertFalse(client_tools.options["interrupt_on_result"]) - self.assertFalse(opening_task.done()) self.assertFalse(any(isinstance(frame, LLMRunFrame) for frame in queued)) - call_end.playback_completion.set_result(None) - await opening_task - self.assertEqual(input_states, [False, True]) + async def test_interruptible_opening_natural_finish_runs_entry_behavior(self): + cfg = AssistantConfig( + type="prompt", + greeting="可以打断的开场白", + startup={ + "opening_mode": "interruptible", + "entry_mode": "generate", + }, + ) + brain = build_brain(cfg) + queued = [] + input_states = [] + + async def queue_frame(frame): + queued.append(frame) + + await brain.setup( + cfg, + BrainRuntime( + context=LLMContext(messages=[]), + llm=FakeLLM(), + queue_frame=queue_frame, + set_system_prompt=lambda _prompt: None, + set_tools=lambda _tools: None, + call_end=FakeCallEnd(), + set_input_enabled=input_states.append, + ), + ) + + await brain.on_connected(greeting_pending=True) + await brain.on_greeting_finished() + + self.assertEqual(input_states, []) self.assertEqual( sum(isinstance(frame, LLMRunFrame) for frame in queued), 1, diff --git a/backend/tests/test_pipeline_events.py b/backend/tests/test_pipeline_events.py index 1c39be0..2230d2f 100644 --- a/backend/tests/test_pipeline_events.py +++ b/backend/tests/test_pipeline_events.py @@ -52,6 +52,7 @@ class _Brain: self.prepared_greeting = "" self.greeting_pending = False self.greeting_finished = 0 + self.interruptions_processed = 0 def prepare_greeting_context(self, greeting, _context): self.prepared_greeting = greeting @@ -62,6 +63,9 @@ class _Brain: async def on_greeting_finished(self): self.greeting_finished += 1 + async def on_interruption_processed(self): + self.interruptions_processed += 1 + async def on_client_ready(self): for content, timestamp in ( ("Message 节点播报", "2026-07-14T10:00:00.200+00:00"), @@ -111,6 +115,7 @@ class PipelineEventTest(unittest.IsolatedAsyncioTestCase): ) self.assertEqual(acknowledgements, [True]) + self.assertEqual(brain.interruptions_processed, 1) async def test_greeting_keeps_playback_timestamp_until_client_ready(self): transport = _EventSource() diff --git a/backend/tests/test_startup_actions.py b/backend/tests/test_startup_actions.py index edf41cb..728c71a 100644 --- a/backend/tests/test_startup_actions.py +++ b/backend/tests/test_startup_actions.py @@ -109,7 +109,7 @@ class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(body.startup.actions, []) self.assertEqual(body.startup.opening_message.confirm_label, "确认") - self.assertTrue(body.startup.opening_message.skip_speech_on_confirm) + self.assertEqual(body.startup.opening_mode, "confirmation") self.assertEqual(body.startup.entry_mode, "generate") async def test_startup_tool_does_not_need_conversation_binding(self): diff --git a/frontend/src/components/assistant-editor/prompt-editor.tsx b/frontend/src/components/assistant-editor/prompt-editor.tsx index bfeccdc..ab21d8f 100644 --- a/frontend/src/components/assistant-editor/prompt-editor.tsx +++ b/frontend/src/components/assistant-editor/prompt-editor.tsx @@ -26,7 +26,6 @@ import { ResourceSelectField, RuntimeModeSelector, TextAreaField, - ToggleRow, ToolPicker, } from "@/components/assistant-editor/editor-controls"; import type { AssistantForm } from "@/components/assistant-editor/types"; @@ -35,9 +34,52 @@ import { VisionConfigSection } from "@/components/editor/vision-config-section"; import { TurnConfigEditor } from "@/components/turn-config-editor"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; -import type { DynamicVariableDefinition, Tool } from "@/lib/api"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { + DynamicVariableDefinition, + StartupConfig, + Tool, +} from "@/lib/api"; type ResourceOption = { value: string; label: string }; +type PromptOpeningMode = StartupConfig["openingMode"]; +type PromptEntryMode = StartupConfig["entryMode"]; + +const OPENING_MODE_OPTIONS: Array<{ + value: PromptOpeningMode; + label: string; + description: string; +}> = [ + { + value: "interruptible", + label: "可打断", + description: "用户说话、发文字或图片时立即结束开场白，并保留这次输入。", + }, + { + value: "playback", + label: "播完继续", + description: "开场白播放期间关闭输入，实际播放完成后进入 Agent。", + }, + { + value: "confirmation", + label: "需要弹窗确认继续", + description: "关闭对话输入并显示弹窗，用户确认后立即进入 Agent。", + }, +]; + +const ENTRY_MODE_OPTIONS: Array<{ + value: PromptEntryMode; + label: string; +}> = [ + { value: "wait_user", label: "等待下一轮用户输入" }, + { value: "generate", label: "进入后立即回复" }, +]; const promptSections = [ { id: "conversation", label: "对话内容" }, @@ -94,6 +136,10 @@ export function PromptEditor({ handlePromptModelChange, }: PromptEditorProps) { const openingMessage = form.startup.openingMessage; + const openingMode = form.startup.openingMode; + const openingModeDescription = OPENING_MODE_OPTIONS.find( + (option) => option.value === openingMode, + )?.description; const scrollContainerRef = useRef(null); const sectionRefs = useRef>({ conversation: null, @@ -192,15 +238,15 @@ export function PromptEditor({ }); } - function setOpeningMessage(enabled: boolean) { + function setOpeningMode(nextMode: PromptOpeningMode) { updateForm("startup", { ...form.startup, - openingMessage: enabled + openingMode: nextMode, + openingMessage: nextMode === "confirmation" ? { - title: "重要提示", - message: "请确认已阅读以上信息。", - confirmLabel: "确认", - skipSpeechOnConfirm: true, + title: openingMessage?.title ?? "重要提示", + message: openingMessage?.message ?? "请确认已阅读以上信息。", + confirmLabel: openingMessage?.confirmLabel ?? "确认", } : null, }); @@ -322,14 +368,33 @@ export function PromptEditor({ /> {form.runtimeMode === "pipeline" && (
- +
+
+ 开场白继续方式 +
+ +

+ {openingModeDescription} +

+
- {openingMessage && ( + {openingMode === "confirmation" && openingMessage && (
- - updateOpeningMessage({ skipSpeechOnConfirm }) - } - />

- 弹窗不可跳过；失败时将结束本次通话。这是平台内置开场阶段， - 不会作为工具暴露给模型。 + 确认前关闭语音、文字和图片输入；确认后立即停止剩余开场白并进入 Agent。

)}
- +
+ Agent 进入行为 +
+ +

+ 该设置仅控制开场阶段正常完成后的 Agent 首轮行为。 +

)} @@ -423,10 +494,13 @@ export function PromptEditor({ if ( runtimeMode === "realtime" && (form.startup.actions.length || + form.startup.openingMode !== "interruptible" || + form.startup.entryMode !== "wait_user" || form.startup.openingMessage) ) { updateForm("startup", { executionMode: "sequential", + openingMode: "interruptible", entryMode: "wait_user", actions: [], openingMessage: null, diff --git a/frontend/src/components/pages/AssistantPage.tsx b/frontend/src/components/pages/AssistantPage.tsx index 9a29605..c64900d 100644 --- a/frontend/src/components/pages/AssistantPage.tsx +++ b/frontend/src/components/pages/AssistantPage.tsx @@ -179,6 +179,7 @@ function blankPromptForm(name: string): AssistantForm { turnConfig: defaultTurnConfig(), startup: { executionMode: "sequential", + openingMode: "interruptible", entryMode: "wait_user", actions: [], openingMessage: null, @@ -473,6 +474,7 @@ export function AssistantPage(props: AssistantPageProps) { turnConfig: normalizeTurnConfig(a.turnConfig), startup: { executionMode: "sequential", + openingMode: a.startup?.openingMode ?? "interruptible", entryMode: a.startup?.entryMode ?? "wait_user", actions: a.startup?.actions ?? [], openingMessage: a.startup?.openingMessage ?? null, @@ -548,6 +550,7 @@ export function AssistantPage(props: AssistantPageProps) { turnConfig: defaultTurnConfig(), startup: { executionMode: "sequential", + openingMode: "interruptible", entryMode: "wait_user", actions: [], openingMessage: null, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ba5ea64..31ee8c1 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -206,13 +206,13 @@ export type StartupAction = { export type StartupConfig = { executionMode: "sequential"; + openingMode: "interruptible" | "playback" | "confirmation"; entryMode: "wait_user" | "generate"; actions: StartupAction[]; openingMessage: { title: string; message: string; confirmLabel: string; - skipSpeechOnConfirm: boolean; } | null; };