F4: verify task recovery and document Prompt background jobs

2026-10-23 16:27:00 +08:00
parent 9a4d48a86e
commit 8ac75a1873
14 changed files with 654 additions and 46 deletions
@@ -31,7 +31,7 @@ class RemoteAgentWorker(BaseWorker):
        self._execute = execute
        self._cancel = cancel

-    @job(name="execute")
+    @job(name="execute", sequential=True)
    async def execute(self, message):
        try:
            result = await self._execute(message, self)
@@ -25,7 +25,12 @@ class TaskService:
        self._closing = False
        self.runtime = NativeTaskRuntime(self._execute, self._cancel_remote, self._on_update, self._on_result)
        await self.runtime.start()
-        for record in await self.repository.list(active=True):
+        try:
+            records = await self.repository.list(active=True)
+        except Exception:
+            await self.close()
+            raise
+        for record in records:
            if record.remote_handle:
                await self._dispatch(record)
            else:
@@ -37,10 +42,13 @@ class TaskService:
            await self.runtime.close()
        self.runtime = None
        self._listeners.clear()
+        self._jobs.clear()

    async def submit(self, *, assistant_id, conversation_id, tool, instruction, invocation_id):
        if not self.runtime:
            raise ValueError("后台任务服务尚未启动")
+        if not assistant_id or not conversation_id:
+            raise ValueError("后台任务需要已保存的助手和会话标识")
        text = instruction.strip()
        if not text or len(text) > 12000:
            raise ValueError("任务指令必须为 1-12000 个字符")
@@ -77,13 +85,16 @@ class TaskService:
                self._listeners.pop(conversation_id, None)
        return detach

-    async def _save(self, task_id, **changes):
-        record = await self.repository.patch(task_id, **changes)
+    async def _publish(self, record):
        for listener in tuple(self._listeners.get(record.conversation_id, ())):
            try:
                await listener(record.public())
            except Exception:
-                logger.warning("任务通知未送达，结果仍可查询: {}", task_id)
+                logger.warning("任务通知未送达，结果仍可查询: {}", record.id)
+
+    async def _save(self, task_id, **changes):
+        record = await self.repository.patch(task_id, **changes)
+        await self._publish(record)
        return record

    async def _execute(self, message, worker):
@@ -96,7 +107,12 @@ class TaskService:
            if not resume:
                handle = await executor.prepare()
                record = await self.repository.patch(task_id, remote_handle=handle)
+            last_event = None
            async def emit(event):
+                nonlocal last_event
+                if event == last_event:
+                    return
+                last_event = deepcopy(event)
                await self.repository.patch(task_id, **event)
                await worker.send_job_update(message.job_id, {"task_id": task_id})
            # Reconnect only observations; never repeat a mutating prompt.
@@ -123,7 +139,7 @@ class TaskService:
        task_id = (message.update or {}).get("task_id")
        if task_id:
            record = await self.get(task_id)
-            await self._save(task_id, summary=record.summary)
+            await self._publish(record)

    async def _on_result(self, message):
        if self._closing:
@@ -135,7 +151,7 @@ class TaskService:
        if record.status not in TERMINAL and str(message.status) in {"error", "failed"}:
            await self._save(task_id, status="unknown", summary="任务运行中断，请核查远端状态")
        else:
-            await self._save(task_id, summary=record.summary)
+            await self._publish(record)

    async def cancel(self, task_id, assistant_id=None):
        record = await self.get(task_id, assistant_id)
@@ -157,7 +173,7 @@ class TaskService:
        if record.status in TERMINAL:
            return
        if not record.remote_handle:
-            await self._save(record.id, status="unknown", summary="尚无远端句柄，无法确认是否已停止")
+            await self._save(record.id, status="cancelled", summary="已取消，尚未向远端提交指令")
            return
        try:
            stopped = await self.executor_factory(record.connection, record.secrets).cancel(record.remote_handle)
@@ -165,6 +181,24 @@ class TaskService:
            stopped = False
        await self._save(record.id, status="cancelled" if stopped else "cancel_requested", summary="远端已停止" if stopped else "停止尚未确认，请检查远端服务")

+    async def reconcile(self, task_id):
+        record = await self.get(task_id)
+        if not record.remote_handle or record.status in TERMINAL:
+            return record.public()
+        executor = self.executor_factory(record.connection, record.secrets)
+        if record.status == "cancel_requested":
+            await self._stop_record(record)
+        else:
+            snapshot = await executor.snapshot(record.remote_handle)
+            # Session status does not describe pending questions/permissions.
+            # Keep the saved request visible until a reply or terminal result.
+            if snapshot["status"] == "running" and record.input_request:
+                snapshot.update(status="waiting_input", summary="远端等待确认或补充信息")
+            await self._save(task_id, **snapshot)
+            if snapshot["status"] not in TERMINAL and self.runtime and record.job_id not in self.runtime.coordinator.job_groups:
+                await self._dispatch(await self.get(task_id))
+        return (await self.get(task_id)).public()
+
    async def send_input(self, task_id, value, assistant_id=None):
        record = await self.get(task_id, assistant_id)
        if record.status != "waiting_input" or not record.input_request:
@@ -847,40 +847,43 @@ async def run_pipeline(
            messages.insert(0, {"role": "system", "content": content})

    set_visible_tools([])
-    await brain.setup(
-        cfg,
-        BrainRuntime(
-            context=context,
-            llm=llm,
-            queue_frame=worker.queue_frame,
-            set_system_prompt=set_system_prompt,
-            set_tools=set_visible_tools,
-            call_end=call_end,
-            session_id=cfg.conversation_id,
-            client_tools=client_tools,
-            worker=worker,
-            context_aggregator=WorkflowAggregatorPair(
-                user_aggregator,
-                assistant_aggregator,
-            ),
-            transport=transport,
-            switch_services=service_controller.switch,
-            set_knowledge_scope=knowledge_retrieval.set_scope,
-            set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
-            vision_function=workflow_vision_function,
-            set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
-            apply_turn_config=apply_workflow_turn_config,
-            flow_global_functions=flow_global_functions,
-        ),
-    )
    try:
+        await brain.setup(
+            cfg,
+            BrainRuntime(
+                context=context,
+                llm=llm,
+                queue_frame=worker.queue_frame,
+                set_system_prompt=set_system_prompt,
+                set_tools=set_visible_tools,
+                call_end=call_end,
+                session_id=cfg.conversation_id,
+                client_tools=client_tools,
+                worker=worker,
+                context_aggregator=WorkflowAggregatorPair(
+                    user_aggregator,
+                    assistant_aggregator,
+                ),
+                transport=transport,
+                switch_services=service_controller.switch,
+                set_knowledge_scope=knowledge_retrieval.set_scope,
+                set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
+                vision_function=workflow_vision_function,
+                set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
+                apply_turn_config=apply_workflow_turn_config,
+                flow_global_functions=flow_global_functions,
+            ),
+        )
        await brain.run_preflight()
-    except Exception:
+        # Preflight tools may assign variables used by the opening speech.
+        greeting = await brain.greeting(cfg)
+    except BaseException:
+        close = getattr(brain, "close", None)
+        if close:
+            await close()
        if recorder:
            await recorder.finish(status="failed")
        raise
-    # Preflight tools may assign variables used by the opening speech.
-    greeting = await brain.greeting(cfg)

    async def submit_user_input(value: UserInput) -> None:
        if not value.has_image: