feat: surface task execution progress across Host Agent and Cloud

Host Agent now persists step-level execution detail locally (via a real
TaskMetadataStore/Timeline wired into TaskRunner) and reports a bounded
in-progress snapshot piggybacked on lease renewal. Cloud persists that
snapshot per active assignment and exposes it through the existing task
list/detail query path; Cloud Console renders it as a live badge. Host
Agent's local console gains authenticated, read-only task list and
detail/timeline pages (same-origin, server-rendered) with inlined
screenshots.

Also fixes a pre-existing gap in the shared Timeline: the actual
per-step LLM prompt is now recorded instead of the task goal, benefiting
both Runtime and Host Agent consoles. When a host uses the cloud planner
transport, each decide call's prompt and resulting tool decision are
durably logged in a new planner_decision_log table (with bounded
retention) and browsable from Cloud Console; direct-transport hosts
explicitly surface a "not reported" state.

Includes Alembic migrations 0008 (progress columns on scheduled_tasks)
and 0009 (planner_decision_log), bounded Host-Agent-local retention,
dual-backend repository parity, and Vitest + pytest coverage. Task 6.5
(manual end-to-end device verification) remains.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-07-14 12:47:49 +08:00
co-authored by Claude Opus 4.6
parent c049c3c1b1
commit ec261d57c2
59 changed files with 3801 additions and 122 deletions
+38
View File
@@ -187,6 +187,19 @@ def create_app(
),
name="cloud-lease-reaper",
),
asyncio.create_task(
_run_planner_decision_log_pruner_loop(
services,
stop_workers,
interval_seconds=(
control_config.planner_decision_log_prune_interval_seconds
),
retention_days=(
control_config.planner_decision_log_retention_days
),
),
name="cloud-planner-decision-log-pruner",
),
]
app.state.worker_tasks = tuple(worker_tasks)
app.state.startup_complete = True
@@ -442,3 +455,28 @@ async def _wait_for_stop(stop: asyncio.Event, interval_seconds: float) -> bool:
except TimeoutError:
return False
return True
async def _run_planner_decision_log_pruner_loop(
services: CloudApplicationServices,
stop: asyncio.Event,
*,
interval_seconds: float,
retention_days: int,
) -> None:
while not stop.is_set():
correlation_token = bind_correlation_id(new_correlation_id())
try:
services.repository.prune_planner_decision_log(
now=utc_now(),
prune_after_terminal_seconds=retention_days * 86_400,
)
except Exception:
logger.exception(
"planner decision log prune failed",
extra={"worker": "planner_decision_log_pruner"},
)
finally:
reset_correlation_id(correlation_token)
if await _wait_for_stop(stop, interval_seconds):
return
+47 -8
View File
@@ -22,10 +22,14 @@ from host_agent.lease import ActiveAssignmentRunner
from host_agent.local_account import LocalAccountStore
from host_agent.policy_cache import HostPolicyCacheStore
from host_agent.processor import AssignmentProcessingResult, AssignmentProcessor
from host_agent.retention import prune_task_history
from host_agent.status import AgentStatusTracker
from host_agent.web.app import create_console_app
from host_agent.web.auth import SessionManager
from storage.artifact_store import ArtifactStore
from storage.device_config import DeviceConfigStore
from storage.task_metadata import TaskMetadataStore
from storage.timeline import Timeline
@dataclass
@@ -178,6 +182,16 @@ def create_application(
console_enrollment_client: HostAgentEnrollmentClient | None = None
if resolved_config.enrollment_managed:
console_enrollment_client = HostAgentEnrollmentClient(resolved_config)
metadata_store = TaskMetadataStore(db_path=resolved_config.task_progress_db_path)
timeline = Timeline(ArtifactStore(root=resolved_config.task_artifact_dir))
executor = AssignmentExecutor(
create_execution_factories(
resolved_manager,
metadata_store=metadata_store,
timeline=timeline,
host_agent_config=resolved_config,
)
)
console_app = create_console_app(
config=resolved_config,
manager=resolved_manager,
@@ -190,6 +204,9 @@ def create_application(
ttl_seconds=resolved_config.console_session_ttl_seconds
),
enrollment_client=console_enrollment_client,
metadata_store=metadata_store,
timeline=timeline,
executor=executor,
)
console_server = _EmbeddedConsoleServer(
uvicorn.Config(
@@ -215,19 +232,18 @@ def create_application(
revision=revision
),
)
executor = AssignmentExecutor(
create_execution_factories(
resolved_manager,
host_agent_config=resolved_config,
)
)
active_runner = ActiveAssignmentRunner(client, executor)
processor = AssignmentProcessor(
client,
active_runner,
status_tracker=status_tracker,
on_result=lambda assignment, result: _record_assignment_history(
history_store, assignment, result
on_result=lambda assignment, result: _on_assignment_finished(
history_store,
metadata_store,
timeline,
resolved_config,
assignment,
result,
),
)
dependency_supervisor: DependencySupervisor | None = None
@@ -245,6 +261,18 @@ def create_application(
)
def _on_assignment_finished(
history_store: ConsoleHistoryStore,
metadata_store: TaskMetadataStore,
timeline: Timeline,
config: HostAgentConfig,
assignment: AssignmentModel,
result: AssignmentProcessingResult,
) -> None:
_record_assignment_history(history_store, assignment, result)
_prune_task_history(metadata_store, timeline, config)
def _record_assignment_history(
history_store: ConsoleHistoryStore,
assignment: AssignmentModel,
@@ -260,6 +288,17 @@ def _record_assignment_history(
)
def _prune_task_history(
metadata_store: TaskMetadataStore,
timeline: Timeline,
config: HostAgentConfig,
) -> None:
try:
prune_task_history(metadata_store, timeline, config=config)
except Exception:
pass
def _configured_device_manager(
config_store: DeviceConfigStore,
*,
@@ -8,6 +8,7 @@ from cloud.internal_api.models import AssignmentModel
from core.models import Task
from host_agent.execution import ExecutionFactories
from host_agent.planner_context import bind_planner_execution_context
from host_agent.progress import TaskProgressHolder, TaskProgressSnapshot
@dataclass(frozen=True)
@@ -20,6 +21,11 @@ class AssignmentExecutionResult:
class AssignmentExecutor:
def __init__(self, factories: ExecutionFactories) -> None:
self.factories = factories
self._progress = TaskProgressHolder()
def latest_progress(self) -> TaskProgressSnapshot | None:
"""Latest step progress reported by the currently-running assignment."""
return self._progress.snapshot()
def execute(
self,
@@ -27,6 +33,7 @@ class AssignmentExecutor:
*,
should_stop: Callable[[], bool] | None = None,
) -> AssignmentExecutionResult:
self._progress.clear()
with bind_planner_execution_context(assignment):
if should_stop is not None and should_stop():
return AssignmentExecutionResult(
@@ -50,6 +57,7 @@ class AssignmentExecutor:
) -> AssignmentExecutionResult:
task = Task(goal=assignment.goal or "", device_id=assignment.device_id)
runner = self.factories.task_runner_factory()
runner.on_step_progress = self._progress.update
if should_stop is None:
completed = runner.run(task)
else:
+24 -6
View File
@@ -16,9 +16,13 @@ from cloud.internal_api.models import (
HostEnrollmentResponse,
HostTaskSubmissionResponse,
LeaseRenewalResponse,
TaskProgressModel,
TerminalResultResponse,
)
from host_agent.config import HostAgentConfig
from host_agent.progress import TaskProgressSnapshot
_VALID_STEP_STATUSES = frozenset({"running", "completed", "failed"})
class HostAgentAPIError(RuntimeError):
@@ -194,19 +198,33 @@ class HostAgentClient:
async def renew(
self,
assignment: AssignmentModel,
*,
progress: TaskProgressSnapshot | None = None,
) -> LeaseRenewalResponse:
payload: dict[str, Any] = {
"host_id": self.config.host_id,
"task_id": assignment.task_id,
"attempt": assignment.attempt,
"lease_id": assignment.lease_id,
}
if progress is not None:
step_status = (
progress.step_status
if progress.step_status in _VALID_STEP_STATUSES
else "running"
)
payload["progress"] = TaskProgressModel(
step_index=max(progress.step_index, 0),
step_status=step_status,
summary=progress.summary[:500],
).model_dump(mode="json")
response = await self._request(
"POST",
(
f"/internal/v1/hosts/{self.config.host_id}/assignments/"
f"{assignment.task_id}/renew"
),
json={
"host_id": self.config.host_id,
"task_id": assignment.task_id,
"attempt": assignment.attempt,
"lease_id": assignment.lease_id,
},
json=payload,
)
return LeaseRenewalResponse.model_validate(response.json())
@@ -43,6 +43,10 @@ class HostAgentConfig:
runtime_host: str = "127.0.0.1"
runtime_port: int = 8000
dependency_restart_max_attempts: int = 5
task_progress_db_path: Path = Path("host_agent_data/task_progress.sqlite3")
task_artifact_dir: Path = Path("host_agent_data/history")
task_retention_max_count: int = 50
task_retention_max_age_days: int = 7
def load_host_agent_config(
@@ -135,6 +139,23 @@ def load_host_agent_config(
dependency_restart_max_attempts=_positive_int(
values, "HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS", 5
),
task_progress_db_path=Path(
values.get(
"HOST_AGENT_TASK_PROGRESS_DB_PATH",
"host_agent_data/task_progress.sqlite3",
).strip()
),
task_artifact_dir=Path(
values.get(
"HOST_AGENT_TASK_ARTIFACT_DIR", "host_agent_data/history"
).strip()
),
task_retention_max_count=_positive_int(
values, "HOST_AGENT_TASK_RETENTION_MAX_COUNT", 50
),
task_retention_max_age_days=_positive_int(
values, "HOST_AGENT_TASK_RETENTION_MAX_AGE_DAYS", 7
),
)
if config.max_retry_backoff_seconds < config.retry_backoff_seconds:
raise HostAgentConfigurationError(
+6 -1
View File
@@ -11,6 +11,7 @@ import httpx
from cloud.internal_api.models import AssignmentModel
from host_agent.assignment import AssignmentExecutionResult
from host_agent.client import HostAgentAPIError, HostAgentClient, StaleLeaseError
from host_agent.progress import TaskProgressSnapshot
class InterruptibleAssignmentExecutor(Protocol):
@@ -21,6 +22,8 @@ class InterruptibleAssignmentExecutor(Protocol):
should_stop: Callable[[], bool] | None = None,
) -> AssignmentExecutionResult: ...
def latest_progress(self) -> TaskProgressSnapshot | None: ...
class LeaseGuard:
def __init__(self) -> None:
@@ -92,7 +95,9 @@ class ActiveAssignmentRunner:
if done:
return
try:
response = await self.client.renew(assignment)
response = await self.client.renew(
assignment, progress=self.executor.latest_progress()
)
except StaleLeaseError:
guard.mark_lost("lease rejected by control plane")
return
@@ -0,0 +1,52 @@
from __future__ import annotations
import threading
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Callable
_MAX_SUMMARY_LENGTH = 500
@dataclass(frozen=True)
class TaskProgressSnapshot:
step_index: int
step_status: str
summary: str
updated_at: datetime
class TaskProgressHolder:
"""Thread-safe latest-step-progress holder for one in-flight assignment.
Written by the execution thread (via ``update``, wired as the
``TaskRunner.on_step_progress`` callback) and read by the asyncio
lease-renewal loop (via ``snapshot``) just before each renewal call.
"""
def __init__(self, *, now: Callable[[], datetime] | None = None) -> None:
self._now = now or (lambda: datetime.now(UTC))
self._lock = threading.Lock()
self._snapshot: TaskProgressSnapshot | None = None
def update(self, step_index: int, step_status: str, summary: str) -> None:
if len(summary) > _MAX_SUMMARY_LENGTH:
summary = summary[:_MAX_SUMMARY_LENGTH]
with self._lock:
self._snapshot = TaskProgressSnapshot(
step_index=step_index,
step_status=step_status,
summary=summary,
updated_at=self._now(),
)
def clear(self) -> None:
with self._lock:
self._snapshot = None
def snapshot(self) -> TaskProgressSnapshot | None:
with self._lock:
return self._snapshot
@@ -0,0 +1,63 @@
from __future__ import annotations
import logging
from datetime import UTC, datetime, timedelta
from host_agent.config import HostAgentConfig
from storage.task_metadata import TaskMetadataStore
from storage.timeline import Timeline
logger = logging.getLogger(__name__)
def prune_task_history(
metadata_store: TaskMetadataStore,
timeline: Timeline,
*,
config: HostAgentConfig,
now: datetime | None = None,
) -> int:
"""Delete tasks beyond the configured retention window/count.
Computes two candidate retained sets -- the last ``max_count`` tasks
and tasks younger than ``max_age_days`` -- and keeps whichever set
is smaller (i.e. the more restrictive bound always wins).
Returns the number of tasks pruned.
"""
reference = now or datetime.now(UTC)
tasks = metadata_store.list_tasks()
keep_by_count = {task["id"] for task in tasks[: config.task_retention_max_count]}
cutoff = reference - timedelta(days=config.task_retention_max_age_days)
keep_by_age = {
task["id"] for task in tasks if _parse_timestamp(task["created_at"]) >= cutoff
}
keep_ids = keep_by_count if len(keep_by_count) <= len(keep_by_age) else keep_by_age
pruned = 0
for task in tasks:
if task["id"] not in keep_ids:
_safe_delete(timeline, metadata_store, task["id"])
pruned += 1
return pruned
def _safe_delete(
timeline: Timeline, metadata_store: TaskMetadataStore, task_id: str
) -> None:
try:
timeline.delete_task(task_id)
except Exception:
logger.debug("timeline delete failed for task %s", task_id, exc_info=True)
try:
metadata_store.delete_task(task_id)
except Exception:
logger.debug("metadata delete failed for task %s", task_id, exc_info=True)
def _parse_timestamp(value: str) -> datetime:
parsed = datetime.fromisoformat(value)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=UTC)
return parsed
@@ -6,6 +6,7 @@ from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from cloud.internal_api.models import AssignmentModel
from host_agent.progress import TaskProgressSnapshot
if TYPE_CHECKING:
from collections.abc import Callable
@@ -34,6 +35,7 @@ class AgentStatusTracker:
self._current_assignment: _CurrentAssignment | None = None
self._last_heartbeat: _LastHeartbeat | None = None
self._host_policy: dict[str, Any] | None = None
self._latest_progress: TaskProgressSnapshot | None = None
def mark_assignment_started(self, assignment: AssignmentModel) -> None:
with self._lock:
@@ -48,6 +50,11 @@ class AgentStatusTracker:
def mark_assignment_finished(self) -> None:
with self._lock:
self._current_assignment = None
self._latest_progress = None
def set_latest_progress(self, snapshot: TaskProgressSnapshot | None) -> None:
with self._lock:
self._latest_progress = snapshot
def mark_heartbeat(self, *, ok: bool, device_count: int) -> None:
with self._lock:
@@ -74,6 +81,7 @@ class AgentStatusTracker:
with self._lock:
current_assignment = self._current_assignment
last_heartbeat = self._last_heartbeat
progress = self._latest_progress
return {
"current_assignment": (
{
@@ -96,4 +104,14 @@ class AgentStatusTracker:
else None
),
"host_policy": self._host_policy.copy() if self._host_policy else None,
"progress": (
{
"step_index": progress.step_index,
"step_status": progress.step_status,
"summary": progress.summary,
"updated_at": progress.updated_at.isoformat(),
}
if progress is not None
else None
),
}
+145 -1
View File
@@ -1,14 +1,17 @@
from __future__ import annotations
import asyncio
import base64
import json
from html import escape as _escape
from pathlib import Path
from typing import Any
from fastapi import Depends, FastAPI, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
from device.manager import DeviceManager
from host_agent.assignment import AssignmentExecutor
from host_agent.client import HostAgentEnrollmentClient
from host_agent.config import HostAgentConfig
from host_agent.devices import register_local_device, unregister_local_device
@@ -23,6 +26,8 @@ from host_agent.web.auth import (
change_password,
)
from storage.device_config import DeviceConfigStore
from storage.task_metadata import TaskMetadataStore
from storage.timeline import Timeline
SESSION_COOKIE_NAME = "host_console_session"
CSRF_HEADER_NAME = "X-CSRF-Token"
@@ -58,6 +63,7 @@ def _chrome(title: str, body_html: str, *, session: SessionState | None) -> str:
<nav>
<a href="/">Status</a>
<a href="/devices">Devices</a>
<a href="/tasks">Tasks</a>
<a href="/account">Account</a>
<a href="/history">History</a>
<form class="inline" method="post" action="/logout">
@@ -142,11 +148,19 @@ def _dashboard_body(
if assignment
else "none"
)
progress = snapshot.get("progress")
progress_text = (
f"step {progress['step_index']}{progress['step_status']}: {progress['summary']}"
if progress
else ""
)
policy_text = (
"revision {revision}; self-submission {self_submission}; "
"max active tasks {max_active}; daily token budget {daily_budget}".format(
revision=policy["revision"],
self_submission="enabled" if policy["self_submission_enabled"] else "disabled",
self_submission="enabled"
if policy["self_submission_enabled"]
else "disabled",
max_active=policy["max_active_tasks"] or "unlimited",
daily_budget=policy["daily_token_budget"] or "unmetered",
)
@@ -178,6 +192,7 @@ def _dashboard_body(
<section>
<h2>Current assignment</h2>
<p id="current-assignment">{escape(assignment_text)}</p>
<p id="current-progress">{escape(progress_text)}</p>
</section>
<section>
<h2>Devices</h2>
@@ -197,6 +212,10 @@ def _dashboard_body(
document.getElementById("current-assignment").textContent = current
? current.task_id + " on " + current.device_id + " (started " + current.started_at + ")"
: "none";
var progress = data.status.progress;
document.getElementById("current-progress").textContent = progress
? "step " + progress.step_index + " \\u2014 " + progress.step_status + ": " + progress.summary
: "";
var policy = data.status.host_policy;
document.getElementById("host-policy").textContent = policy
? "revision " + policy.revision + "; self-submission "
@@ -314,6 +333,88 @@ def _history_body(entries: list[dict[str, Any]]) -> str:
"""
def _inline_screenshot(record: dict[str, Any]) -> str:
"""Return a ``<img>`` tag with base64-encoded screenshot, or empty string."""
screenshot_path = record.get("screenshot_path")
if not screenshot_path:
return ""
path = Path(str(screenshot_path))
if not path.exists():
return ""
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f'<img src="data:image/png;base64,{encoded}" alt="screenshot" style="max-width:100%;border:1px solid #ccc;margin-top:0.5rem;">'
def _tasks_body(tasks: list[dict[str, Any]]) -> str:
if not tasks:
return """
<h1>Tasks</h1>
<p>No tasks recorded.</p>
"""
rows = "".join(
f"<tr>"
f'<td><a href="/tasks/{escape(task["id"])}">{escape(task["id"])}</a></td>'
f"<td>{escape(task.get('status') or '')}</td>"
f"<td>{escape(task.get('device_id') or '')}</td>"
f"<td>{escape(task.get('created_at') or '')}</td>"
f"<td>{escape(task.get('updated_at') or '')}</td>"
f"</tr>"
for task in tasks
)
return f"""
<h1>Tasks</h1>
<table>
<thead><tr><th>Task ID</th><th>Status</th><th>Device</th><th>Created</th><th>Updated</th></tr></thead>
<tbody>{rows}</tbody>
</table>
"""
def _task_detail_body(
task: dict[str, Any], timeline_records: list[dict[str, Any]]
) -> str:
task_rows = "".join(
f"<tr><td>{escape(key)}</td><td>{escape(task[key])}</td></tr>"
for key in ("id", "goal", "device_id", "status", "created_at", "updated_at")
if task.get(key) is not None
)
metadata_html = f"""
<h1>Task {escape(task.get("id") or "")}</h1>
<table>
<thead><tr><th>Field</th><th>Value</th></tr></thead>
<tbody>{task_rows}</tbody>
</table>
"""
if not timeline_records:
timeline_html = "<h2>Timeline</h2><p>No timeline records.</p>"
else:
step_blocks = "".join(
_timeline_step_html(record) for record in timeline_records
)
timeline_html = f"<h2>Timeline</h2>{step_blocks}"
return metadata_html + timeline_html
def _timeline_step_html(record: dict[str, Any]) -> str:
index = record.get("index", "")
timestamp = record.get("timestamp", "")
tool_call = record.get("tool_call")
result = record.get("result")
prompt = record.get("prompt") or ""
tool_call_text = json.dumps(tool_call, ensure_ascii=False) if tool_call else ""
result_text = json.dumps(result, ensure_ascii=False) if result else ""
screenshot_html = _inline_screenshot(record)
return f"""
<div style="border:1px solid #ccc;background:#fff;padding:0.75rem;margin-bottom:0.75rem;">
<p><strong>Step {escape(index)}</strong> {escape(timestamp)}</p>
<p>Prompt: {escape(prompt)}</p>
<p>Tool call: <code>{escape(tool_call_text)}</code></p>
<p>Result: <code>{escape(result_text)}</code></p>
{screenshot_html}
</div>
"""
def create_console_app(
*,
config: HostAgentConfig,
@@ -325,6 +426,9 @@ def create_console_app(
status_tracker: AgentStatusTracker,
session_manager: SessionManager,
enrollment_client: HostAgentEnrollmentClient | None,
metadata_store: TaskMetadataStore | None = None,
timeline: Timeline | None = None,
executor: AssignmentExecutor | None = None,
) -> FastAPI:
app = FastAPI(title="Host Agent Console")
cookie_secure = config.console_bind_host not in _LOOPBACK_BIND_HOSTS
@@ -416,6 +520,16 @@ def create_console_app(
session: SessionState = Depends(require_session),
) -> JSONResponse:
snapshot = status_tracker.snapshot()
# Pull live progress from the executor if the tracker has no data yet.
if executor is not None and snapshot.get("progress") is None:
live = executor.latest_progress()
if live is not None:
snapshot["progress"] = {
"step_index": live.step_index,
"step_status": live.step_status,
"summary": live.summary,
"updated_at": live.updated_at.isoformat(),
}
current_assignment = snapshot.get("current_assignment")
busy_device_id = current_assignment["device_id"] if current_assignment else None
devices = [
@@ -570,4 +684,34 @@ def create_console_app(
body = _history_body(entries)
return HTMLResponse(_chrome("History", body, session=session))
@app.get("/tasks", response_class=HTMLResponse)
async def tasks_page(
session: SessionState = Depends(require_session),
) -> HTMLResponse:
if metadata_store is None:
raise HTTPException(
status_code=503, detail="task metadata store not configured"
)
tasks_list = await asyncio.to_thread(metadata_store.list_tasks)
body = _tasks_body(tasks_list)
return HTMLResponse(_chrome("Tasks", body, session=session))
@app.get("/tasks/{task_id}", response_class=HTMLResponse)
async def task_detail_page(
task_id: str,
session: SessionState = Depends(require_session),
) -> HTMLResponse:
if metadata_store is None:
raise HTTPException(
status_code=503, detail="task metadata store not configured"
)
task = await asyncio.to_thread(metadata_store.get_task, task_id)
if task is None:
raise HTTPException(status_code=404, detail="task not found")
timeline_records: list[dict[str, Any]] = []
if timeline is not None:
timeline_records = await asyncio.to_thread(timeline.read, task_id)
body = _task_detail_body(task, timeline_records)
return HTMLResponse(_chrome(f"Task {task_id}", body, session=session))
return app