q792602257/agentic-mobile-control
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: surface task execution progress across Host Agent and Cloud

Browse Source 
Host Agent now persists step-level execution detail locally (via a real
TaskMetadataStore/Timeline wired into TaskRunner) and reports a bounded
in-progress snapshot piggybacked on lease renewal. Cloud persists that
snapshot per active assignment and exposes it through the existing task
list/detail query path; Cloud Console renders it as a live badge. Host
Agent's local console gains authenticated, read-only task list and
detail/timeline pages (same-origin, server-rendered) with inlined
screenshots.

Also fixes a pre-existing gap in the shared Timeline: the actual
per-step LLM prompt is now recorded instead of the task goal, benefiting
both Runtime and Host Agent consoles. When a host uses the cloud planner
transport, each decide call's prompt and resulting tool decision are
durably logged in a new planner_decision_log table (with bounded
retention) and browsable from Cloud Console; direct-transport hosts
explicitly surface a "not reported" state.

Includes Alembic migrations 0008 (progress columns on scheduled_tasks)
and 0009 (planner_decision_log), bounded Host-Agent-local retention,
dual-backend repository parity, and Vitest + pytest coverage. Task 6.5
(manual end-to-end device verification) remains.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
q792602257
2026-07-14 12:47:49 +08:00
co-authored by Claude Opus 4.6
parent c049c3c1b1
commit ec261d57c2
59 changed files with 3801 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/ai_planner.py
+7 -5
View File
@@ -36,13 +36,14 @@ class AIPlanner(Planner):
world: "WorldState | None" = None,
screenshot: bytes | None = None,
) -> list[PlannedStep]:
user_prompt = planner_user_prompt(
goal=goal,
scene_json=scene.to_dict(),
history_summary=_history_summary(world),
)
decision = self.client.decide(
system_prompt=PLANNER_SYSTEM_PROMPT,
user_prompt=planner_user_prompt(
goal=goal,
scene_json=scene.to_dict(),
history_summary=_history_summary(world),
),
user_prompt=user_prompt,
screenshot=screenshot,
tools=ALL_TOOL_SPECS,
timeout=self.config.timeout,
@@ -58,6 +59,7 @@ class AIPlanner(Planner):
action=decision.tool_name,
description=f"AI planner: {decision.tool_name}({decision.arguments})",
args=dict(decision.arguments),
prompt=decision.user_prompt or user_prompt,
)
]
runtime/planner.py
+3
View File
@@ -16,6 +16,9 @@ class PlannedStep:
description: str
args: dict[str, Any] = field(default_factory=dict)
expected_text: str | None = None
# The actual prompt sent to the LLM for this step (AI planners only).
# ``None`` for non-LLM planners; TaskRunner falls back to the task goal.
prompt: str | None = None
class Planner:
runtime/task.py
+32 -1
View File
@@ -40,6 +40,7 @@ Observer = Callable[[str], Scene]
ScreenshotProvider = Callable[[str], bytes]
TaskSucceededHook = Callable[[str, str, Timeline], None]
StopRequested = Callable[[], bool]
StepProgressCallback = Callable[[int, str, str], None]
class TaskRunner:
@@ -60,6 +61,7 @@ class TaskRunner:
skill_store: SkillStore | None = None,
skill_embedding_client: EmbeddingClient | None = None,
planner_config: PlannerConfig | None = None,
on_step_progress: StepProgressCallback | None = None,
) -> None:
self.planner_config = planner_config or load_planner_config()
self.planner = planner or self._default_planner()
@@ -89,6 +91,7 @@ class TaskRunner:
self.on_task_succeeded = self._default_task_succeeded_hook
else:
self.on_task_succeeded = None
self.on_step_progress = on_step_progress
def run(
self,
@@ -114,6 +117,7 @@ class TaskRunner:
reason = (
f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
)
self._emit_step_progress(len(context.step_results), "failed", reason)
self._update_task(
task,
status="failed",
@@ -140,7 +144,17 @@ class TaskRunner:
self._record_step_result(
world_handle, context, task, scene, step, result
)
self._emit_step_progress(
len(context.step_results),
"running",
f"{step.action}: {step.description}",
)
if not result.success:
self._emit_step_progress(
len(context.step_results),
"failed",
result.error or "step failed",
)
self._update_task(
task,
status="failed",
@@ -149,6 +163,11 @@ class TaskRunner:
)
return task
self._emit_step_progress(
len(context.step_results),
"failed",
f"max steps exceeded: {self.config.max_steps}",
)
self._update_task(
task,
status="failed",
@@ -158,6 +177,7 @@ class TaskRunner:
return task
def _interrupt_task(self, task: Task) -> Task:
self._emit_step_progress(-1, "failed", "execution interrupted")
self._update_task(
task,
status="failed",
@@ -194,7 +214,18 @@ class TaskRunner:
self._update_world(world_handle, context, scene, step, result)
self._append_timeline(task, scene, step, result)
def _emit_step_progress(
self, step_index: int, step_status: str, summary: str
) -> None:
if self.on_step_progress is None:
return
try:
self.on_step_progress(max(step_index, 0), step_status, summary[:200])
except Exception as exc:
logger.debug("step progress callback failed: %s", exc)
def _complete_task(self, task: Task) -> Task:
self._emit_step_progress(-1, "completed", "task completed")
self._update_task(task, status="completed", completed=True)
self._notify_task_succeeded(task)
return task
@@ -306,7 +337,7 @@ class TaskRunner:
self.timeline.append(
task_id=task.id,
scene=scene.to_dict(),
prompt=task.goal,
prompt=step.prompt or task.goal,
tool_call={
"action": step.action,
"description": step.description,
runtime/tool_calling_client.py
+31 -4
View File
@@ -25,6 +25,11 @@ class ToolCallDecision:
tool_name: str
arguments: dict[str, Any]
usage: ToolCallUsage | None = None
# The actual prompts sent to the LLM for this call. Populated by the
# built-in Anthropic/OpenAI clients; empty for clients (e.g.
# CloudProxyToolCallingClient) that don't surface them.
system_prompt: str = ""
user_prompt: str = ""
class ToolCallingClient(Protocol):
@@ -72,7 +77,11 @@ class AnthropicToolCallingClient:
tools,
timeout=timeout,
)
return _decision_from_anthropic_response(response)
return _decision_from_anthropic_response(
response,
system_prompt=system_prompt,
user_prompt=user_prompt,
)
except ToolCallUnavailable:
raise
except Exception as exc:
@@ -164,7 +173,11 @@ class OpenAIToolCallingClient:
tools,
timeout=timeout,
)
return _decision_from_openai_response(response)
return _decision_from_openai_response(
response,
system_prompt=system_prompt,
user_prompt=user_prompt,
)
except ToolCallUnavailable:
raise
except Exception as exc:
@@ -250,7 +263,12 @@ def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
}
def _decision_from_anthropic_response(response: Any) -> ToolCallDecision:
def _decision_from_anthropic_response(
response: Any,
*,
system_prompt: str = "",
user_prompt: str = "",
) -> ToolCallDecision:
content = _value(response, "content")
if not isinstance(content, list):
raise ValueError("anthropic tool-call response missing content list")
@@ -264,6 +282,8 @@ def _decision_from_anthropic_response(response: Any) -> ToolCallDecision:
tool_name=name,
arguments=arguments,
usage=_anthropic_usage(response),
system_prompt=system_prompt,
user_prompt=user_prompt,
)
raise ValueError("anthropic response did not include a tool_use block")
@@ -295,7 +315,12 @@ def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
}
def _decision_from_openai_response(response: Any) -> ToolCallDecision:
def _decision_from_openai_response(
response: Any,
*,
system_prompt: str = "",
user_prompt: str = "",
) -> ToolCallDecision:
choices = _value(response, "choices")
if not isinstance(choices, list) or not choices:
raise ValueError("openai tool-call response missing choices")
@@ -312,6 +337,8 @@ def _decision_from_openai_response(response: Any) -> ToolCallDecision:
tool_name=name,
arguments=arguments,
usage=_openai_usage(response),
system_prompt=system_prompt,
user_prompt=user_prompt,
)