feat: surface task execution progress across Host Agent and Cloud
Host Agent now persists step-level execution detail locally (via a real TaskMetadataStore/Timeline wired into TaskRunner) and reports a bounded in-progress snapshot piggybacked on lease renewal. Cloud persists that snapshot per active assignment and exposes it through the existing task list/detail query path; Cloud Console renders it as a live badge. Host Agent's local console gains authenticated, read-only task list and detail/timeline pages (same-origin, server-rendered) with inlined screenshots. Also fixes a pre-existing gap in the shared Timeline: the actual per-step LLM prompt is now recorded instead of the task goal, benefiting both Runtime and Host Agent consoles. When a host uses the cloud planner transport, each decide call's prompt and resulting tool decision are durably logged in a new planner_decision_log table (with bounded retention) and browsable from Cloud Console; direct-transport hosts explicitly surface a "not reported" state. Includes Alembic migrations 0008 (progress columns on scheduled_tasks) and 0009 (planner_decision_log), bounded Host-Agent-local retention, dual-backend repository parity, and Vitest + pytest coverage. Task 6.5 (manual end-to-end device verification) remains. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+98
-17
@@ -8,7 +8,6 @@ from core.errors import TaskFailedError
|
||||
from core.models import Bounds, Scene, SceneElement
|
||||
from runtime.ai_planner import AIPlanner
|
||||
from runtime.context import TaskContext
|
||||
from runtime.planner import PlannedStep
|
||||
from runtime.planner_config import PlannerConfig
|
||||
from runtime.tool_calling_client import ToolCallDecision
|
||||
from runtime.tool_specs import ALL_TOOL_SPECS
|
||||
@@ -44,7 +43,11 @@ def _scene() -> Scene:
|
||||
return Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[SceneElement(id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4))],
|
||||
elements=[
|
||||
SceneElement(
|
||||
id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4)
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -53,23 +56,25 @@ def _context() -> TaskContext:
|
||||
|
||||
|
||||
def test_ai_planner_returns_single_planned_step_for_action_decision() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
steps = planner.plan(goal="send a message", scene=_scene(), context=_context())
|
||||
|
||||
assert steps == [
|
||||
PlannedStep(
|
||||
action="tap",
|
||||
description="AI planner: tap({'x': 1, 'y': 2})",
|
||||
args={"x": 1, "y": 2},
|
||||
)
|
||||
]
|
||||
assert len(steps) == 1
|
||||
step = steps[0]
|
||||
assert step.action == "tap"
|
||||
assert step.description == "AI planner: tap({'x': 1, 'y': 2})"
|
||||
assert step.args == {"x": 1, "y": 2}
|
||||
|
||||
|
||||
def test_ai_planner_finish_task_success_returns_empty_plan() -> None:
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="finish_task", arguments={"success": True, "reason": "done"})
|
||||
ToolCallDecision(
|
||||
tool_name="finish_task", arguments={"success": True, "reason": "done"}
|
||||
)
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
@@ -80,7 +85,10 @@ def test_ai_planner_finish_task_success_returns_empty_plan() -> None:
|
||||
|
||||
def test_ai_planner_finish_task_failure_raises_task_failed_error_with_reason() -> None:
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="finish_task", arguments={"success": False, "reason": "stuck on login"})
|
||||
ToolCallDecision(
|
||||
tool_name="finish_task",
|
||||
arguments={"success": False, "reason": "stuck on login"},
|
||||
)
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
@@ -89,7 +97,9 @@ def test_ai_planner_finish_task_failure_raises_task_failed_error_with_reason() -
|
||||
|
||||
|
||||
def test_ai_planner_finish_task_failure_without_reason_uses_default_message() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="finish_task", arguments={"success": False}))
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="finish_task", arguments={"success": False})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
with pytest.raises(TaskFailedError, match="task failed"):
|
||||
@@ -97,20 +107,91 @@ def test_ai_planner_finish_task_failure_without_reason_uses_default_message() ->
|
||||
|
||||
|
||||
def test_ai_planner_goal_reached_is_always_false() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
assert planner.goal_reached(goal="anything", scene=_scene(), context=_context()) is False
|
||||
assert (
|
||||
planner.goal_reached(goal="anything", scene=_scene(), context=_context())
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_ai_planner_forwards_tools_screenshot_and_timeout_to_client() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
)
|
||||
planner = AIPlanner(client=client, config=PlannerConfig(timeout=12.5))
|
||||
|
||||
planner.plan(goal="send a message", scene=_scene(), context=_context(), screenshot=b"fake-bytes")
|
||||
planner.plan(
|
||||
goal="send a message",
|
||||
scene=_scene(),
|
||||
context=_context(),
|
||||
screenshot=b"fake-bytes",
|
||||
)
|
||||
|
||||
call = client.calls[0]
|
||||
assert call["tools"] == ALL_TOOL_SPECS
|
||||
assert call["screenshot"] == b"fake-bytes"
|
||||
assert call["timeout"] == 12.5
|
||||
assert "send a message" in call["user_prompt"]
|
||||
|
||||
|
||||
def test_ai_planner_populates_step_prompt_from_user_prompt() -> None:
|
||||
"""PlannedStep.prompt should carry the actual user prompt sent to the LLM,
|
||||
not the bare task goal."""
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
steps = planner.plan(goal="send a message", scene=_scene(), context=_context())
|
||||
|
||||
assert len(steps) == 1
|
||||
assert steps[0].prompt is not None
|
||||
# The per-step prompt contains the goal but also scene JSON and instruction text
|
||||
assert "send a message" in steps[0].prompt
|
||||
assert "Current Scene (JSON)" in steps[0].prompt
|
||||
assert "Call exactly one tool" in steps[0].prompt
|
||||
|
||||
|
||||
def test_ai_planner_step_prompt_reflects_scene_changes() -> None:
|
||||
"""Per-step prompts differ when the scene changes, proving they are not
|
||||
just the repeated task goal."""
|
||||
from runtime.context import TaskContext
|
||||
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
scene_a = Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[
|
||||
SceneElement(
|
||||
id="btn_a", type="button", text="Alpha", bounds=Bounds(1, 2, 3, 4)
|
||||
)
|
||||
],
|
||||
)
|
||||
scene_b = Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[
|
||||
SceneElement(
|
||||
id="btn_b", type="button", text="Beta", bounds=Bounds(5, 6, 7, 8)
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
steps_a = planner.plan(
|
||||
goal="test", scene=scene_a, context=TaskContext(task_id="t", goal="test")
|
||||
)
|
||||
steps_b = planner.plan(
|
||||
goal="test", scene=scene_b, context=TaskContext(task_id="t", goal="test")
|
||||
)
|
||||
|
||||
assert steps_a[0].prompt != steps_b[0].prompt
|
||||
assert "Alpha" in steps_a[0].prompt
|
||||
assert "Beta" in steps_b[0].prompt
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.models import Bounds, Scene, SceneElement, Task
|
||||
from runtime.ai_planner import AIPlanner
|
||||
from runtime.executor import Executor, ExecutorConfig
|
||||
from runtime.planner import PlannedStep, Planner
|
||||
from runtime.planner_config import PlannerConfig
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from runtime.tool_calling_client import ToolCallDecision
|
||||
from storage.artifact_store import ArtifactStore
|
||||
from storage.timeline import Timeline
|
||||
from tests.fakes import PNG_10X20
|
||||
|
||||
|
||||
@@ -56,7 +61,11 @@ def _scene() -> Scene:
|
||||
return Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[SceneElement(id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4))],
|
||||
elements=[
|
||||
SceneElement(
|
||||
id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4)
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -126,7 +135,144 @@ def test_task_runner_default_planner_is_stub_when_ai_planner_disabled() -> None:
|
||||
def test_task_runner_default_planner_is_ai_planner_when_enabled() -> None:
|
||||
runner = _runner(
|
||||
planner=None,
|
||||
planner_config=PlannerConfig(enabled=True, provider="anthropic", model="test-model"),
|
||||
planner_config=PlannerConfig(
|
||||
enabled=True, provider="anthropic", model="test-model"
|
||||
),
|
||||
)
|
||||
|
||||
assert isinstance(runner.planner, AIPlanner)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# D9: per-step prompt recording
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ScriptedToolCallingClient:
|
||||
"""Returns a sequence of decisions, capturing the actual user_prompt each call."""
|
||||
|
||||
def __init__(self, decisions: list[ToolCallDecision]) -> None:
|
||||
self._decisions = list(decisions)
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def decide(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[Any],
|
||||
timeout: float,
|
||||
) -> ToolCallDecision:
|
||||
index = len(self.calls)
|
||||
self.calls.append(
|
||||
{
|
||||
"system_prompt": system_prompt,
|
||||
"user_prompt": user_prompt,
|
||||
}
|
||||
)
|
||||
return self._decisions[index]
|
||||
|
||||
|
||||
def _multi_step_scene(element_text: str = "Send") -> Scene:
|
||||
return Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[
|
||||
SceneElement(
|
||||
id="btn",
|
||||
type="button",
|
||||
text=element_text,
|
||||
bounds=Bounds(1, 2, 3, 4),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_multi_step_timeline_records_actual_per_step_prompts(tmp_path) -> None:
|
||||
"""When AIPlanner is used, each timeline step's prompt is the real
|
||||
per-step user prompt (containing scene JSON), not the bare task goal."""
|
||||
decisions = [
|
||||
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}),
|
||||
ToolCallDecision(tool_name="finish_task", arguments={"success": True}),
|
||||
]
|
||||
client = ScriptedToolCallingClient(decisions)
|
||||
planner = AIPlanner(client=client)
|
||||
executor = Executor(
|
||||
tools={"tap": lambda **kwargs: {"ok": True, **kwargs}},
|
||||
config=ExecutorConfig(max_retries=1, backoff_seconds=0),
|
||||
)
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
task = Task(goal="tap the button", device_id="phone")
|
||||
|
||||
runner = TaskRunner(
|
||||
planner=planner,
|
||||
executor=executor,
|
||||
timeline=timeline,
|
||||
config=TaskRunnerConfig(max_steps=5),
|
||||
observer=lambda device_id: _multi_step_scene("Send"),
|
||||
screenshot_provider=lambda device_id: PNG_10X20,
|
||||
)
|
||||
|
||||
runner.run(task)
|
||||
|
||||
records = timeline.read(task.id)
|
||||
# Step 1 was recorded (step 2 was finish_task, which returns empty plan
|
||||
# and completes the task without a timeline append).
|
||||
assert len(records) == 1
|
||||
prompt = records[0]["prompt"]
|
||||
# The per-step prompt is NOT the bare task goal.
|
||||
assert prompt != "tap the button"
|
||||
# It contains scene-specific content that only the real planner_user_prompt
|
||||
# would include.
|
||||
assert "Current Scene (JSON)" in prompt
|
||||
assert "Call exactly one tool" in prompt
|
||||
assert "tap the button" in prompt
|
||||
|
||||
|
||||
def test_non_ai_planner_falls_back_to_task_goal_for_prompt(tmp_path) -> None:
|
||||
"""A non-LLM planner (no step.prompt) keeps recording task.goal as the
|
||||
timeline prompt — backward compat with pre-D9 behavior."""
|
||||
scene = _multi_step_scene("Search")
|
||||
planner = ScriptedPlannerForTimeline(
|
||||
[PlannedStep(action="tap", description="tap", args={"x": 1, "y": 2})]
|
||||
)
|
||||
executor = Executor(
|
||||
tools={"tap": lambda **kwargs: {"ok": True, **kwargs}},
|
||||
config=ExecutorConfig(max_retries=1, backoff_seconds=0),
|
||||
)
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
task = Task(goal="search for something", device_id="phone")
|
||||
|
||||
runner = TaskRunner(
|
||||
planner=planner,
|
||||
executor=executor,
|
||||
timeline=timeline,
|
||||
config=TaskRunnerConfig(max_steps=5),
|
||||
observer=lambda device_id: scene,
|
||||
screenshot_provider=lambda device_id: PNG_10X20,
|
||||
)
|
||||
|
||||
runner.run(task)
|
||||
|
||||
records = timeline.read(task.id)
|
||||
assert len(records) >= 1
|
||||
# Non-AI planner: prompt falls back to task goal.
|
||||
assert records[0]["prompt"] == "search for something"
|
||||
|
||||
|
||||
class ScriptedPlannerForTimeline(Planner):
|
||||
"""Simple planner that returns a fixed list of steps then signals done."""
|
||||
|
||||
def __init__(self, steps: list[PlannedStep]) -> None:
|
||||
self.steps = steps
|
||||
|
||||
def plan(self, *, goal, scene, context):
|
||||
if len(context.step_results) >= len(self.steps):
|
||||
return []
|
||||
return [self.steps[len(context.step_results)]]
|
||||
|
||||
def goal_reached(self, *, goal, scene, context):
|
||||
return len(context.step_results) >= len(self.steps) and all(
|
||||
r.success for r in context.step_results
|
||||
)
|
||||
|
||||
@@ -195,6 +195,57 @@ def test_schema_readiness_requires_head_revision(tmp_path) -> None:
|
||||
require_current_schema(database_url)
|
||||
|
||||
|
||||
def test_planner_decision_log_migration_upgrades_and_downgrades(tmp_path) -> None:
|
||||
database_url = _database_url(tmp_path)
|
||||
|
||||
upgrade_database(database_url, "0008_task_progress_columns")
|
||||
engine = create_engine(database_url)
|
||||
try:
|
||||
inspector = inspect(engine)
|
||||
assert "planner_decision_log" not in inspector.get_table_names()
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
upgrade_database(database_url)
|
||||
|
||||
engine = create_engine(database_url)
|
||||
try:
|
||||
inspector = inspect(engine)
|
||||
assert "planner_decision_log" in inspector.get_table_names()
|
||||
columns = {col["name"] for col in inspector.get_columns("planner_decision_log")}
|
||||
assert {
|
||||
"id",
|
||||
"host_id",
|
||||
"task_id",
|
||||
"attempt",
|
||||
"step_index",
|
||||
"system_prompt",
|
||||
"user_prompt",
|
||||
"tool_name",
|
||||
"arguments_json",
|
||||
"created_at",
|
||||
} <= columns
|
||||
index_names = {
|
||||
idx["name"] for idx in inspector.get_indexes("planner_decision_log")
|
||||
}
|
||||
assert {
|
||||
"ix_planner_decision_log_task_attempt",
|
||||
"ix_planner_decision_log_host_created",
|
||||
} <= index_names
|
||||
assert current_revision(database_url) == HEAD_REVISION
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
downgrade_database(database_url, "0008_task_progress_columns")
|
||||
engine = create_engine(database_url)
|
||||
try:
|
||||
inspector = inspect(engine)
|
||||
assert "planner_decision_log" not in inspector.get_table_names()
|
||||
assert current_revision(database_url) == "0008_task_progress_columns"
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _create_legacy_schema(connection) -> None:
|
||||
connection.exec_driver_sql(
|
||||
"create table host_registrations ("
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from cloud.auth import BearerCredential, ConfiguredBearerAuthProvider
|
||||
from cloud.config import CloudConfig
|
||||
from cloud.db_models import TaskAttemptRow
|
||||
from cloud.internal_api.api import create_internal_router
|
||||
from cloud.pool import DevicePool
|
||||
from cloud.store import CloudStore
|
||||
@@ -49,7 +53,7 @@ class _FakeToolCallingClient:
|
||||
|
||||
def _build_client(
|
||||
tmp_path, *, fake_client: _FakeToolCallingClient
|
||||
) -> tuple[TestClient, _FakeToolCallingClient]:
|
||||
) -> tuple[TestClient, _FakeToolCallingClient, DevicePool]:
|
||||
pool = DevicePool(
|
||||
CloudStore(tmp_path / "internal.sqlite3"),
|
||||
CloudConfig(stale_after_seconds=60),
|
||||
@@ -68,7 +72,7 @@ def _build_client(
|
||||
planner_client_factory=lambda: fake_client,
|
||||
)
|
||||
)
|
||||
return TestClient(app), fake_client
|
||||
return TestClient(app), fake_client, pool
|
||||
|
||||
|
||||
def _decision_payload(**overrides: object) -> dict[str, object]:
|
||||
@@ -94,7 +98,7 @@ def test_authenticated_host_resolves_planner_decision(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
)
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -113,7 +117,7 @@ def test_planner_decision_decodes_screenshot_base64(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -129,7 +133,7 @@ def test_invalid_screenshot_base64_is_rejected(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -145,7 +149,7 @@ def test_unauthenticated_request_is_rejected(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -160,7 +164,7 @@ def test_foreign_host_token_cannot_request_another_hosts_decision(tmp_path) -> N
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -176,7 +180,7 @@ def test_path_and_payload_host_id_mismatch_is_rejected(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -190,7 +194,7 @@ def test_path_and_payload_host_id_mismatch_is_rejected(tmp_path) -> None:
|
||||
|
||||
def test_provider_failure_returns_structured_error_without_crashing(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(error="anthropic timed out")
|
||||
client, fake_client = _build_client(tmp_path, fake_client=fake_client)
|
||||
client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
@@ -203,3 +207,153 @@ def test_provider_failure_returns_structured_error_without_crashing(tmp_path) ->
|
||||
"code": "planner_unavailable",
|
||||
"detail": "anthropic timed out",
|
||||
}
|
||||
|
||||
|
||||
def _seed_attempt(pool: DevicePool, task_id: str, host_id: str = "host-a") -> None:
|
||||
"""Insert a task_attempts row so _validate_planner_context passes."""
|
||||
now = datetime.now(tz=UTC)
|
||||
with Session(pool.store.engine) as session, session.begin():
|
||||
session.add(
|
||||
TaskAttemptRow(
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
lease_id="lease-x",
|
||||
host_id=host_id,
|
||||
device_id="device-x",
|
||||
status="dispatched",
|
||||
lease_expires_at=(now + timedelta(minutes=5)).isoformat(),
|
||||
created_at=now.isoformat(),
|
||||
completed_at=None,
|
||||
failure_reason=None,
|
||||
result_json=None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_successful_decision_is_persisted_with_correct_fields(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={"x": 10, "y": 20})
|
||||
)
|
||||
client, fake_client, pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
_seed_attempt(pool, "task-log-1")
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
json=_decision_payload(
|
||||
task_id="task-log-1",
|
||||
attempt=1,
|
||||
lease_id="lease-x",
|
||||
system_prompt="you are a test planner",
|
||||
user_prompt="tap the button now",
|
||||
),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
decisions = pool.store.list_planner_decisions(task_id="task-log-1", attempt=1)
|
||||
assert len(decisions) == 1
|
||||
row = decisions[0]
|
||||
assert row.step_index == 1
|
||||
assert row.system_prompt == "you are a test planner"
|
||||
assert row.user_prompt == "tap the button now"
|
||||
assert row.tool_name == "tap"
|
||||
assert '"x": 10' in row.arguments_json
|
||||
assert '"y": 20' in row.arguments_json
|
||||
|
||||
|
||||
def test_failed_decision_persists_nothing(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(error="model is down")
|
||||
client, fake_client, pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
_seed_attempt(pool, "task-log-fail")
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
json=_decision_payload(
|
||||
task_id="task-log-fail",
|
||||
attempt=1,
|
||||
lease_id="lease-x",
|
||||
),
|
||||
)
|
||||
|
||||
assert response.status_code == 502
|
||||
assert pool.store.list_planner_decisions(task_id="task-log-fail", attempt=1) == []
|
||||
|
||||
|
||||
def test_screenshot_bytes_are_never_persisted_to_decision_log(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client, pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
_seed_attempt(pool, "task-log-screenshot")
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
json=_decision_payload(
|
||||
task_id="task-log-screenshot",
|
||||
attempt=1,
|
||||
lease_id="lease-x",
|
||||
screenshot_base64="aGVsbG8gd29ybGQ=",
|
||||
),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
decisions = pool.store.list_planner_decisions(
|
||||
task_id="task-log-screenshot", attempt=1
|
||||
)
|
||||
assert len(decisions) == 1
|
||||
# The row has no screenshot column at all — verify the raw row text
|
||||
# does not contain the screenshot bytes.
|
||||
row_repr = repr(decisions[0])
|
||||
assert "hello world" not in row_repr
|
||||
assert "aGVsbG8" not in row_repr
|
||||
|
||||
|
||||
def test_decision_without_task_context_is_not_persisted(tmp_path) -> None:
|
||||
"""When task_id/attempt are absent, the log insert is skipped."""
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client, pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
json=_decision_payload(), # no task_id / attempt / lease_id
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
# No task_id to query by — verify no rows exist at all via direct SQL.
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from cloud.db_models import PlannerDecisionLogRow
|
||||
|
||||
with Session(pool.store.engine) as session:
|
||||
count = session.scalar(select(func.count()).select_from(PlannerDecisionLogRow))
|
||||
assert count == 0
|
||||
|
||||
|
||||
def test_multiple_decisions_get_incrementing_step_index(tmp_path) -> None:
|
||||
fake_client = _FakeToolCallingClient(
|
||||
decision=ToolCallDecision(tool_name="tap", arguments={})
|
||||
)
|
||||
client, fake_client, pool = _build_client(tmp_path, fake_client=fake_client)
|
||||
_seed_attempt(pool, "task-log-multi")
|
||||
|
||||
for i in range(3):
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/planner/decide",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
json=_decision_payload(
|
||||
task_id="task-log-multi",
|
||||
attempt=1,
|
||||
lease_id="lease-x",
|
||||
user_prompt=f"step {i}",
|
||||
),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
decisions = pool.store.list_planner_decisions(task_id="task-log-multi", attempt=1)
|
||||
assert [d.step_index for d in decisions] == [1, 2, 3]
|
||||
assert [d.user_prompt for d in decisions] == ["step 0", "step 1", "step 2"]
|
||||
|
||||
@@ -82,6 +82,9 @@ def test_cloud_repository_exposes_crud_and_atomic_lease_operations() -> None:
|
||||
"record_task_result",
|
||||
"reap_expired_leases",
|
||||
"list_task_attempts",
|
||||
"record_planner_decision",
|
||||
"prune_planner_decision_log",
|
||||
"list_planner_decisions",
|
||||
"health_check",
|
||||
"close",
|
||||
} <= members
|
||||
@@ -1553,3 +1556,198 @@ def test_task_lifecycle_logs_structured_identifiers(
|
||||
assert "secret-image" not in repr(events)
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_record_planner_decision_assigns_incrementing_step_index(
|
||||
database_url: str,
|
||||
) -> None:
|
||||
database = CloudDatabase(database_url)
|
||||
task_id = _unique_id("planner-task")
|
||||
other_task_id = _unique_id("planner-task-other")
|
||||
host_id = _unique_id("planner-host")
|
||||
now = datetime(2026, 7, 14, 0, 0, tzinfo=UTC)
|
||||
|
||||
try:
|
||||
step1 = database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
system_prompt="system",
|
||||
user_prompt="prompt-1",
|
||||
tool_name="tap",
|
||||
arguments_json='{"x": 1}',
|
||||
now=now,
|
||||
)
|
||||
step2 = database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
system_prompt="system",
|
||||
user_prompt="prompt-2",
|
||||
tool_name="swipe",
|
||||
arguments_json='{"y": 2}',
|
||||
now=now + timedelta(seconds=1),
|
||||
)
|
||||
step3 = database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
system_prompt="system",
|
||||
user_prompt="prompt-3",
|
||||
tool_name="wait",
|
||||
arguments_json="{}",
|
||||
now=now + timedelta(seconds=2),
|
||||
)
|
||||
|
||||
assert [step1, step2, step3] == [1, 2, 3]
|
||||
|
||||
# Different (task_id, attempt) gets its own counter starting at 1.
|
||||
other_step = database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=other_task_id,
|
||||
attempt=1,
|
||||
system_prompt="system",
|
||||
user_prompt="other",
|
||||
tool_name="tap",
|
||||
arguments_json="{}",
|
||||
now=now,
|
||||
)
|
||||
assert other_step == 1
|
||||
|
||||
# Different attempt on the same task also gets its own counter.
|
||||
attempt2_step = database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=2,
|
||||
system_prompt="system",
|
||||
user_prompt="retry",
|
||||
tool_name="tap",
|
||||
arguments_json="{}",
|
||||
now=now,
|
||||
)
|
||||
assert attempt2_step == 1
|
||||
|
||||
# Verify stored rows.
|
||||
decisions = database.repository.list_planner_decisions(
|
||||
task_id=task_id, attempt=1
|
||||
)
|
||||
assert len(decisions) == 3
|
||||
assert [d.step_index for d in decisions] == [1, 2, 3]
|
||||
assert [d.user_prompt for d in decisions] == [
|
||||
"prompt-1",
|
||||
"prompt-2",
|
||||
"prompt-3",
|
||||
]
|
||||
assert [d.tool_name for d in decisions] == ["tap", "swipe", "wait"]
|
||||
assert decisions[0].arguments_json == '{"x": 1}'
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_prune_planner_decision_log_deletes_only_old_terminal_tasks(
|
||||
database_url: str,
|
||||
) -> None:
|
||||
database = CloudDatabase(database_url)
|
||||
old_terminal_task = _unique_id("old-terminal")
|
||||
in_flight_task = _unique_id("in-flight")
|
||||
recent_terminal_task = _unique_id("recent-terminal")
|
||||
host_id = _unique_id("prune-host")
|
||||
now = datetime(2026, 7, 14, 12, 0, tzinfo=UTC)
|
||||
|
||||
try:
|
||||
with Session(database.engine) as session, session.begin():
|
||||
# Old terminal task (completed > window ago).
|
||||
session.add(
|
||||
TaskAttemptRow(
|
||||
task_id=old_terminal_task,
|
||||
attempt=1,
|
||||
lease_id="lease-old",
|
||||
host_id=host_id,
|
||||
device_id="device-old",
|
||||
status="done",
|
||||
lease_expires_at=(now - timedelta(days=10)).isoformat(),
|
||||
created_at=(now - timedelta(days=11)).isoformat(),
|
||||
completed_at=(now - timedelta(days=10)).isoformat(),
|
||||
failure_reason=None,
|
||||
result_json=None,
|
||||
)
|
||||
)
|
||||
# In-flight task (also old, but NOT terminal).
|
||||
session.add(
|
||||
TaskAttemptRow(
|
||||
task_id=in_flight_task,
|
||||
attempt=1,
|
||||
lease_id="lease-flight",
|
||||
host_id=host_id,
|
||||
device_id="device-flight",
|
||||
status="dispatched",
|
||||
lease_expires_at=(now - timedelta(days=10)).isoformat(),
|
||||
created_at=(now - timedelta(days=11)).isoformat(),
|
||||
completed_at=None,
|
||||
failure_reason=None,
|
||||
result_json=None,
|
||||
)
|
||||
)
|
||||
# Recently terminal task (within window).
|
||||
session.add(
|
||||
TaskAttemptRow(
|
||||
task_id=recent_terminal_task,
|
||||
attempt=1,
|
||||
lease_id="lease-recent",
|
||||
host_id=host_id,
|
||||
device_id="device-recent",
|
||||
status="done",
|
||||
lease_expires_at=(now - timedelta(hours=1)).isoformat(),
|
||||
created_at=(now - timedelta(hours=2)).isoformat(),
|
||||
completed_at=(now - timedelta(hours=1)).isoformat(),
|
||||
failure_reason=None,
|
||||
result_json=None,
|
||||
)
|
||||
)
|
||||
|
||||
# Seed decision log rows for all three tasks.
|
||||
for task_id in [old_terminal_task, in_flight_task, recent_terminal_task]:
|
||||
database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
system_prompt="s",
|
||||
user_prompt="u",
|
||||
tool_name="tap",
|
||||
arguments_json="{}",
|
||||
now=now - timedelta(days=11),
|
||||
)
|
||||
|
||||
# 7-day window.
|
||||
deleted = database.repository.prune_planner_decision_log(
|
||||
now=now,
|
||||
prune_after_terminal_seconds=7 * 86_400,
|
||||
)
|
||||
assert deleted == 1
|
||||
|
||||
# Old terminal task's rows are gone.
|
||||
assert (
|
||||
database.repository.list_planner_decisions(
|
||||
task_id=old_terminal_task, attempt=1
|
||||
)
|
||||
== []
|
||||
)
|
||||
# In-flight and recent terminal rows survive.
|
||||
assert (
|
||||
len(
|
||||
database.repository.list_planner_decisions(
|
||||
task_id=in_flight_task, attempt=1
|
||||
)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
len(
|
||||
database.repository.list_planner_decisions(
|
||||
task_id=recent_terminal_task, attempt=1
|
||||
)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
@@ -338,6 +338,7 @@ def test_submit_rejects_incomplete_or_foreign_target(tmp_path) -> None:
|
||||
("get", "/v1/tasks/missing", None, "tasks:read"),
|
||||
("get", "/v1/tasks", None, "tasks:read"),
|
||||
("get", "/v1/tasks/missing/attempts", None, "tasks:read"),
|
||||
("get", "/v1/tasks/missing/planner-decisions?attempt=0", None, "tasks:read"),
|
||||
("get", "/v1/devices", None, "pool:read"),
|
||||
("get", "/v1/hosts", None, "pool:read"),
|
||||
("get", "/v1/plugins", None, "plugins:read"),
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Repository contract tests for cloud task progress columns (task 3.6).
|
||||
|
||||
Separate from ``test_cloud_repository_contract.py``; covers the new
|
||||
``progress_*`` fields added by migration 0008.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from cloud.database import CloudDatabase
|
||||
from cloud.pool import PooledDevice
|
||||
from cloud.repository import AssignmentProgressSnapshot
|
||||
from cloud.scheduler import ScheduledTask, TaskConstraints
|
||||
|
||||
|
||||
def _device(device_id: str, host_id: str) -> PooledDevice:
|
||||
return PooledDevice(
|
||||
device_id=device_id,
|
||||
host_id=host_id,
|
||||
driver_type="wda",
|
||||
status="idle",
|
||||
capability_tags=["ios"],
|
||||
synced_at=datetime(2026, 7, 12, tzinfo=UTC),
|
||||
)
|
||||
|
||||
|
||||
def _task(task_id: str, *, created_at: datetime | None = None) -> ScheduledTask:
|
||||
return ScheduledTask(
|
||||
id=task_id,
|
||||
goal="test progress",
|
||||
workflow_definition_id=None,
|
||||
constraints=TaskConstraints(),
|
||||
created_at=created_at or datetime(2026, 7, 12, tzinfo=UTC),
|
||||
)
|
||||
|
||||
|
||||
def _setup_assigned_and_claimed(
|
||||
database: CloudDatabase, host_id: str, device_id: str, task_id: str
|
||||
):
|
||||
"""Enqueue, assign, and claim a task. Returns LeasedAssignment."""
|
||||
now = datetime(2026, 7, 12, 5, 0, tzinfo=UTC)
|
||||
database.repository.upsert_host(host_id, address=None, last_seen_at=now)
|
||||
database.repository.replace_host_devices(host_id, [_device(device_id, host_id)])
|
||||
database.repository.enqueue_task(_task(task_id, created_at=now))
|
||||
database.repository.assign_task(
|
||||
task_id=task_id,
|
||||
host_id=host_id,
|
||||
device_id=device_id,
|
||||
lease_id="lease-1",
|
||||
lease_expires_at=now + timedelta(minutes=10),
|
||||
now=now,
|
||||
)
|
||||
assignment = database.repository.claim_assignment(
|
||||
host_id=host_id,
|
||||
now=now + timedelta(seconds=1),
|
||||
)
|
||||
assert assignment is not None
|
||||
return assignment, now
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# renew_lease writes progress on success
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_renew_lease_writes_progress_on_success(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'progress.sqlite3').as_posix()}")
|
||||
host_id = "progress-host"
|
||||
device_id = "progress-device"
|
||||
task_id = "progress-task"
|
||||
|
||||
try:
|
||||
assignment, now = _setup_assigned_and_claimed(
|
||||
database, host_id, device_id, task_id
|
||||
)
|
||||
|
||||
progress = AssignmentProgressSnapshot(
|
||||
step_index=2,
|
||||
step_status="running",
|
||||
summary="executing step 2",
|
||||
updated_at=now + timedelta(seconds=5),
|
||||
)
|
||||
result = database.repository.renew_lease(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
lease_expires_at=now + timedelta(minutes=15),
|
||||
now=now + timedelta(seconds=5),
|
||||
progress=progress,
|
||||
)
|
||||
assert result == "renewed"
|
||||
|
||||
task = database.repository.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.progress_step_index == 2
|
||||
assert task.progress_step_status == "running"
|
||||
assert task.progress_summary == "executing step 2"
|
||||
assert task.progress_updated_at is not None
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# renew_lease without progress leaves previous untouched
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_renew_lease_without_progress_leaves_previous_untouched(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'progress2.sqlite3').as_posix()}")
|
||||
host_id = "retain-host"
|
||||
device_id = "retain-device"
|
||||
task_id = "retain-task"
|
||||
|
||||
try:
|
||||
assignment, now = _setup_assigned_and_claimed(
|
||||
database, host_id, device_id, task_id
|
||||
)
|
||||
|
||||
# First renew WITH progress.
|
||||
progress = AssignmentProgressSnapshot(
|
||||
step_index=1,
|
||||
step_status="completed",
|
||||
summary="step 1 done",
|
||||
updated_at=now + timedelta(seconds=2),
|
||||
)
|
||||
database.repository.renew_lease(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
lease_expires_at=now + timedelta(minutes=15),
|
||||
now=now + timedelta(seconds=2),
|
||||
progress=progress,
|
||||
)
|
||||
|
||||
# Second renew WITHOUT progress (None).
|
||||
database.repository.renew_lease(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
lease_expires_at=now + timedelta(minutes=20),
|
||||
now=now + timedelta(seconds=4),
|
||||
progress=None,
|
||||
)
|
||||
|
||||
task = database.repository.get_task(task_id)
|
||||
assert task is not None
|
||||
# Previous values must be retained, not cleared.
|
||||
assert task.progress_step_index == 1
|
||||
assert task.progress_step_status == "completed"
|
||||
assert task.progress_summary == "step 1 done"
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# record_task_result clears progress on terminal
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_record_task_result_clears_progress_on_terminal(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'clear.sqlite3').as_posix()}")
|
||||
host_id = "clear-host"
|
||||
device_id = "clear-device"
|
||||
task_id = "clear-task"
|
||||
|
||||
try:
|
||||
assignment, now = _setup_assigned_and_claimed(
|
||||
database, host_id, device_id, task_id
|
||||
)
|
||||
|
||||
# Write progress first.
|
||||
progress = AssignmentProgressSnapshot(
|
||||
step_index=3,
|
||||
step_status="running",
|
||||
summary="almost done",
|
||||
updated_at=now + timedelta(seconds=3),
|
||||
)
|
||||
database.repository.renew_lease(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
lease_expires_at=now + timedelta(minutes=15),
|
||||
now=now + timedelta(seconds=3),
|
||||
progress=progress,
|
||||
)
|
||||
|
||||
task = database.repository.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.progress_step_index == 3
|
||||
|
||||
# Record terminal result.
|
||||
result = database.repository.record_task_result(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
status="done",
|
||||
failure_reason=None,
|
||||
terminal_result={"output": "success"},
|
||||
completed_at=now + timedelta(seconds=10),
|
||||
)
|
||||
assert result == "recorded"
|
||||
|
||||
task = database.repository.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.status == "done"
|
||||
assert task.progress_step_index is None
|
||||
assert task.progress_step_status is None
|
||||
assert task.progress_summary is None
|
||||
assert task.progress_updated_at is None
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Task list/detail response includes progress fields
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_list_tasks_includes_progress_fields(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'list.sqlite3').as_posix()}")
|
||||
host_id = "list-host"
|
||||
device_id = "list-device"
|
||||
task_id = "list-task"
|
||||
|
||||
try:
|
||||
assignment, now = _setup_assigned_and_claimed(
|
||||
database, host_id, device_id, task_id
|
||||
)
|
||||
|
||||
progress = AssignmentProgressSnapshot(
|
||||
step_index=1,
|
||||
step_status="running",
|
||||
summary="running step 1",
|
||||
updated_at=now + timedelta(seconds=5),
|
||||
)
|
||||
database.repository.renew_lease(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
lease_expires_at=now + timedelta(minutes=15),
|
||||
now=now + timedelta(seconds=5),
|
||||
progress=progress,
|
||||
)
|
||||
|
||||
tasks = database.repository.list_tasks()
|
||||
matching = [t for t in tasks if t.id == task_id]
|
||||
assert len(matching) == 1
|
||||
task = matching[0]
|
||||
assert task.progress_step_index == 1
|
||||
assert task.progress_step_status == "running"
|
||||
assert task.progress_summary == "running step 1"
|
||||
assert task.progress_updated_at is not None
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_get_task_includes_progress_fields(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'get.sqlite3').as_posix()}")
|
||||
host_id = "get-host"
|
||||
device_id = "get-device"
|
||||
task_id = "get-task"
|
||||
|
||||
try:
|
||||
assignment, now = _setup_assigned_and_claimed(
|
||||
database, host_id, device_id, task_id
|
||||
)
|
||||
|
||||
progress = AssignmentProgressSnapshot(
|
||||
step_index=5,
|
||||
step_status="failed",
|
||||
summary="step 5 failed",
|
||||
updated_at=now + timedelta(seconds=8),
|
||||
)
|
||||
database.repository.renew_lease(
|
||||
task_id=task_id,
|
||||
attempt=assignment.attempt,
|
||||
lease_id=assignment.lease_id,
|
||||
host_id=host_id,
|
||||
lease_expires_at=now + timedelta(minutes=15),
|
||||
now=now + timedelta(seconds=8),
|
||||
progress=progress,
|
||||
)
|
||||
|
||||
task = database.repository.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.progress_step_index == 5
|
||||
assert task.progress_step_status == "failed"
|
||||
assert task.progress_summary == "step 5 failed"
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_progress_fields_default_null_before_any_renewal(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'null.sqlite3').as_posix()}")
|
||||
host_id = "null-host"
|
||||
device_id = "null-device"
|
||||
task_id = "null-task"
|
||||
|
||||
try:
|
||||
_setup_assigned_and_claimed(database, host_id, device_id, task_id)
|
||||
|
||||
task = database.repository.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.progress_step_index is None
|
||||
assert task.progress_step_status is None
|
||||
assert task.progress_summary is None
|
||||
assert task.progress_updated_at is None
|
||||
finally:
|
||||
database.close()
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Tests for Host Agent console /tasks routes (task 5.4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from device.manager import DeviceManager
|
||||
from host_agent.config import HostAgentConfig
|
||||
from host_agent.history import ConsoleHistoryStore
|
||||
from host_agent.identity import HostIdentityStore
|
||||
from host_agent.local_account import LocalAccountStore
|
||||
from host_agent.status import AgentStatusTracker
|
||||
from host_agent.web.app import create_console_app
|
||||
from host_agent.web.auth import SessionManager
|
||||
from storage.artifact_store import ArtifactStore
|
||||
from storage.device_config import DeviceConfigStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
from tests.fakes import PNG_10X20
|
||||
|
||||
CSRF_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"')
|
||||
|
||||
|
||||
def _build_client(
|
||||
tmp_path,
|
||||
*,
|
||||
metadata_store: TaskMetadataStore | None = None,
|
||||
timeline: Timeline | None = None,
|
||||
create_account: bool = True,
|
||||
) -> tuple[TestClient, dict]:
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="secret",
|
||||
enrollment_managed=False,
|
||||
console_session_ttl_seconds=3600.0,
|
||||
)
|
||||
manager = DeviceManager()
|
||||
config_store = DeviceConfigStore(tmp_path / "devices.sqlite3")
|
||||
local_account_store = LocalAccountStore(tmp_path / "host_local_account.json")
|
||||
if create_account:
|
||||
local_account_store.create("operator", "correct horse battery staple")
|
||||
identity_store = HostIdentityStore(tmp_path / "host_identity.json")
|
||||
history_store = ConsoleHistoryStore(tmp_path / "history.sqlite3")
|
||||
status_tracker = AgentStatusTracker()
|
||||
session_manager = SessionManager(ttl_seconds=3600.0)
|
||||
|
||||
if metadata_store is None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "task_progress.sqlite3")
|
||||
if timeline is None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
app = create_console_app(
|
||||
config=config,
|
||||
manager=manager,
|
||||
config_store=config_store,
|
||||
local_account_store=local_account_store,
|
||||
identity_store=identity_store,
|
||||
history_store=history_store,
|
||||
status_tracker=status_tracker,
|
||||
session_manager=session_manager,
|
||||
enrollment_client=None,
|
||||
metadata_store=metadata_store,
|
||||
timeline=timeline,
|
||||
)
|
||||
client = TestClient(app)
|
||||
context = {
|
||||
"metadata_store": metadata_store,
|
||||
"timeline": timeline,
|
||||
"session_manager": session_manager,
|
||||
}
|
||||
return client, context
|
||||
|
||||
|
||||
def _login(client: TestClient) -> str:
|
||||
response = client.post(
|
||||
"/login",
|
||||
data={"username": "operator", "password": "correct horse battery staple"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
match = CSRF_PATTERN.search(response.text)
|
||||
assert match is not None
|
||||
return match.group(1)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Unauthenticated requests rejected
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_unauthenticated_tasks_list_redirects_to_login(tmp_path) -> None:
|
||||
client, _ = _build_client(tmp_path)
|
||||
|
||||
response = client.get("/tasks", follow_redirects=False)
|
||||
|
||||
assert response.status_code == 303
|
||||
assert response.headers["location"] == "/login"
|
||||
|
||||
|
||||
def test_unauthenticated_task_detail_redirects_to_login(tmp_path) -> None:
|
||||
client, _ = _build_client(tmp_path)
|
||||
|
||||
response = client.get("/tasks/some-id", follow_redirects=False)
|
||||
|
||||
assert response.status_code == 303
|
||||
assert response.headers["location"] == "/login"
|
||||
|
||||
|
||||
def test_unauthenticated_devices_page_also_redirects_for_parity(tmp_path) -> None:
|
||||
"""Confirm /tasks has the same auth behavior as the existing /devices route."""
|
||||
client, _ = _build_client(tmp_path)
|
||||
|
||||
response = client.get("/devices", follow_redirects=False)
|
||||
|
||||
assert response.status_code == 303
|
||||
assert response.headers["location"] == "/login"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Authenticated list/detail render
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_authenticated_tasks_list_renders_completed_task(tmp_path) -> None:
|
||||
from core.models import Task
|
||||
|
||||
metadata_store = TaskMetadataStore(tmp_path / "task_progress.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
task = Task(
|
||||
id="task-visible",
|
||||
goal="open settings",
|
||||
device_id="iphone-1",
|
||||
status="completed",
|
||||
created_at=datetime(2026, 7, 10, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 7, 10, 1, tzinfo=UTC),
|
||||
completed_at=datetime(2026, 7, 10, 1, tzinfo=UTC),
|
||||
)
|
||||
metadata_store.create_task(task)
|
||||
|
||||
client, _ = _build_client(
|
||||
tmp_path, metadata_store=metadata_store, timeline=timeline
|
||||
)
|
||||
_login(client)
|
||||
|
||||
response = client.get("/tasks")
|
||||
assert response.status_code == 200
|
||||
assert "task-visible" in response.text
|
||||
assert "completed" in response.text
|
||||
|
||||
|
||||
def test_authenticated_task_detail_renders_timeline_with_screenshot(tmp_path) -> None:
|
||||
from core.models import Task
|
||||
|
||||
metadata_store = TaskMetadataStore(tmp_path / "task_progress.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
task = Task(
|
||||
id="task-detail",
|
||||
goal="tap search",
|
||||
device_id="iphone-1",
|
||||
status="completed",
|
||||
created_at=datetime(2026, 7, 10, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 7, 10, 1, tzinfo=UTC),
|
||||
)
|
||||
metadata_store.create_task(task)
|
||||
|
||||
timeline.append(
|
||||
task_id="task-detail",
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="find search button",
|
||||
tool_call={"action": "tap", "x": 5, "y": 10},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
|
||||
client, _ = _build_client(
|
||||
tmp_path, metadata_store=metadata_store, timeline=timeline
|
||||
)
|
||||
_login(client)
|
||||
|
||||
response = client.get("/tasks/task-detail")
|
||||
assert response.status_code == 200
|
||||
assert "task-detail" in response.text
|
||||
# Timeline step visible.
|
||||
assert "Step 1" in response.text
|
||||
assert "find search button" in response.text
|
||||
# Screenshot inlined as base64 data URI.
|
||||
assert "data:image/png;base64," in response.text
|
||||
|
||||
|
||||
def test_task_detail_404_for_unknown_task(tmp_path) -> None:
|
||||
client, _ = _build_client(tmp_path)
|
||||
_login(client)
|
||||
|
||||
response = client.get("/tasks/nonexistent")
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_tasks_list_shows_empty_state(tmp_path) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "task_progress.sqlite3")
|
||||
client, _ = _build_client(tmp_path, metadata_store=metadata_store)
|
||||
_login(client)
|
||||
|
||||
response = client.get("/tasks")
|
||||
assert response.status_code == 200
|
||||
assert "No tasks recorded" in response.text
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Tests for Host Agent progress reporting via lease renewal (task 2.4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import Event
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from cloud.internal_api.models import (
|
||||
AssignmentModel,
|
||||
LeaseRenewalResponse,
|
||||
)
|
||||
from host_agent.config import HostAgentConfig
|
||||
from host_agent.lease import ActiveAssignmentRunner
|
||||
from host_agent.progress import TaskProgressHolder, TaskProgressSnapshot
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Helpers
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _assignment(
|
||||
*,
|
||||
lease_expires_at: datetime | None = None,
|
||||
) -> AssignmentModel:
|
||||
return AssignmentModel(
|
||||
task_id="task-progress",
|
||||
attempt=1,
|
||||
lease_id="lease-1",
|
||||
lease_expires_at=lease_expires_at or datetime.now(UTC) + timedelta(minutes=5),
|
||||
host_id="host-a",
|
||||
device_id="device-a",
|
||||
goal="do something",
|
||||
)
|
||||
|
||||
|
||||
class _FakeExecutor:
|
||||
"""Stub executor that optionally fires a progress callback then blocks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
fire_progress_before_block: bool = False,
|
||||
progress_payload: tuple[int, str, str] = (1, "running", "step 1 summary"),
|
||||
block_event: Event | None = None,
|
||||
) -> None:
|
||||
self._fire = fire_progress_before_block
|
||||
self._payload = progress_payload
|
||||
self._block_event = block_event or Event()
|
||||
self._progress = TaskProgressHolder()
|
||||
|
||||
def execute(
|
||||
self,
|
||||
assignment: AssignmentModel,
|
||||
*,
|
||||
should_stop: Any | None = None,
|
||||
) -> Any:
|
||||
from host_agent.assignment import AssignmentExecutionResult
|
||||
|
||||
if self._fire:
|
||||
self._progress.update(*self._payload)
|
||||
|
||||
# Block until test signals completion.
|
||||
self._block_event.wait(timeout=5)
|
||||
return AssignmentExecutionResult(status="done")
|
||||
|
||||
def latest_progress(self) -> TaskProgressSnapshot | None:
|
||||
return self._progress.snapshot()
|
||||
|
||||
|
||||
class _RecordingClient:
|
||||
"""Fake HostAgentClient that records each renew() call's progress argument."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.renew_calls: list[TaskProgressSnapshot | None] = []
|
||||
self._call_count = 0
|
||||
|
||||
async def renew(
|
||||
self,
|
||||
assignment: AssignmentModel,
|
||||
*,
|
||||
progress: TaskProgressSnapshot | None = None,
|
||||
) -> LeaseRenewalResponse:
|
||||
self.renew_calls.append(progress)
|
||||
self._call_count += 1
|
||||
return LeaseRenewalResponse(
|
||||
status="renewed",
|
||||
lease_expires_at=datetime.now(UTC) + timedelta(minutes=5),
|
||||
)
|
||||
|
||||
|
||||
def _runner(
|
||||
client: _RecordingClient, executor: _FakeExecutor
|
||||
) -> ActiveAssignmentRunner:
|
||||
return ActiveAssignmentRunner(client, executor, now=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Renewal includes progress after a step completes
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_renewal_includes_progress_after_step(tmp_path) -> None:
|
||||
block_event = Event()
|
||||
executor = _FakeExecutor(
|
||||
fire_progress_before_block=True,
|
||||
progress_payload=(1, "running", "step 1 summary"),
|
||||
block_event=block_event,
|
||||
)
|
||||
client = _RecordingClient()
|
||||
runner = _runner(client, executor)
|
||||
|
||||
# Set a very short lease so renewal fires quickly.
|
||||
assignment = _assignment(
|
||||
lease_expires_at=datetime.now(UTC) + timedelta(milliseconds=50),
|
||||
)
|
||||
|
||||
async def _drive() -> None:
|
||||
# Start the runner; execution thread will fire progress then block.
|
||||
task = asyncio.create_task(runner.run(assignment))
|
||||
# Wait for at least one renewal to happen.
|
||||
for _ in range(50):
|
||||
await asyncio.sleep(0.05)
|
||||
if client.renew_calls:
|
||||
break
|
||||
# Signal the executor to complete.
|
||||
block_event.set()
|
||||
await task
|
||||
|
||||
asyncio.run(_drive())
|
||||
|
||||
assert len(client.renew_calls) > 0
|
||||
# At least one renewal should have non-None progress.
|
||||
progress_renewals = [p for p in client.renew_calls if p is not None]
|
||||
assert len(progress_renewals) > 0
|
||||
snap = progress_renewals[0]
|
||||
assert snap.step_index == 1
|
||||
assert snap.step_status == "running"
|
||||
assert "step 1 summary" in snap.summary
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Renewal omits progress before any step completes
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_renewal_omits_progress_before_any_step(tmp_path) -> None:
|
||||
block_event = Event()
|
||||
executor = _FakeExecutor(
|
||||
fire_progress_before_block=False,
|
||||
block_event=block_event,
|
||||
)
|
||||
client = _RecordingClient()
|
||||
runner = _runner(client, executor)
|
||||
|
||||
assignment = _assignment(
|
||||
lease_expires_at=datetime.now(UTC) + timedelta(milliseconds=50),
|
||||
)
|
||||
|
||||
async def _drive() -> None:
|
||||
task = asyncio.create_task(runner.run(assignment))
|
||||
for _ in range(50):
|
||||
await asyncio.sleep(0.05)
|
||||
if client.renew_calls:
|
||||
break
|
||||
block_event.set()
|
||||
await task
|
||||
|
||||
asyncio.run(_drive())
|
||||
|
||||
assert len(client.renew_calls) > 0
|
||||
# No progress should have been reported.
|
||||
assert all(p is None for p in client.renew_calls)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Oversized summary truncated client-side before sending
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_oversized_summary_truncated_client_side() -> None:
|
||||
"""HostAgentClient.renew truncates summary to <=500 chars before serializing."""
|
||||
from host_agent.client import HostAgentClient
|
||||
|
||||
captured_payload: dict[str, Any] = {}
|
||||
|
||||
def _handler(request: httpx.Request) -> httpx.Response:
|
||||
import json
|
||||
|
||||
body = json.loads(request.content.decode("utf-8"))
|
||||
captured_payload.update(body)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"status": "renewed",
|
||||
"lease_expires_at": (
|
||||
datetime.now(UTC) + timedelta(minutes=5)
|
||||
).isoformat(),
|
||||
},
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(_handler)
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="secret",
|
||||
retry_backoff_seconds=0.01,
|
||||
max_retry_attempts=1,
|
||||
)
|
||||
http_client = httpx.AsyncClient(
|
||||
base_url=config.control_plane_url, transport=transport
|
||||
)
|
||||
client = HostAgentClient(config, http_client=http_client)
|
||||
|
||||
long_summary = "x" * 2000
|
||||
snapshot = TaskProgressSnapshot(
|
||||
step_index=1,
|
||||
step_status="running",
|
||||
summary=long_summary,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
|
||||
asyncio.run(client.renew(_assignment(), progress=snapshot))
|
||||
|
||||
assert "progress" in captured_payload
|
||||
assert captured_payload["progress"] is not None
|
||||
serialized_summary = captured_payload["progress"]["summary"]
|
||||
assert len(serialized_summary) <= 500
|
||||
assert serialized_summary == "x" * 500
|
||||
|
||||
asyncio.run(http_client.aclose())
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Tests for Host Agent task storage, timeline, retention, and collision avoidance (task 1.5)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from core.models import Task
|
||||
from host_agent.config import HostAgentConfig
|
||||
from host_agent.retention import prune_task_history
|
||||
from storage.artifact_store import ArtifactStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
from tests.fakes import PNG_10X20
|
||||
|
||||
|
||||
def _make_task(
|
||||
task_id: str,
|
||||
*,
|
||||
goal: str = "test goal",
|
||||
device_id: str = "iphone-1",
|
||||
created_at: datetime | None = None,
|
||||
status: str = "created",
|
||||
) -> Task:
|
||||
ts = created_at or datetime(2026, 7, 1, tzinfo=UTC)
|
||||
return Task(
|
||||
id=task_id,
|
||||
goal=goal,
|
||||
device_id=device_id,
|
||||
status=status, # type: ignore[arg-type]
|
||||
created_at=ts,
|
||||
updated_at=ts,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Persist during execution
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_step_transitions_persist_during_execution(tmp_path) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
task = _make_task("task-persist")
|
||||
metadata_store.create_task(task)
|
||||
|
||||
# Simulate the execution loop writing step transitions.
|
||||
metadata_store.update_task(task.id, status="running")
|
||||
timeline.append(
|
||||
task_id=task.id,
|
||||
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
||||
prompt="step 1",
|
||||
tool_call={"action": "tap"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
timeline.append(
|
||||
task_id=task.id,
|
||||
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
||||
prompt="step 2",
|
||||
tool_call={"action": "swipe"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
metadata_store.update_task(task.id, status="completed", completed=True)
|
||||
|
||||
row = metadata_store.get_task(task.id)
|
||||
assert row is not None
|
||||
assert row["status"] == "completed"
|
||||
assert row["completed_at"] is not None
|
||||
|
||||
records = timeline.read(task.id)
|
||||
assert len(records) == 2
|
||||
assert [r["index"] for r in records] == [1, 2]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Queryable after completion (in-memory Task discarded)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_task_history_queryable_after_in_memory_task_discarded(tmp_path) -> None:
|
||||
db_path = tmp_path / "tasks.sqlite3"
|
||||
history_dir = tmp_path / "history"
|
||||
|
||||
metadata_store = TaskMetadataStore(db_path)
|
||||
timeline = Timeline(ArtifactStore(history_dir))
|
||||
|
||||
task = _make_task("task-survive")
|
||||
metadata_store.create_task(task)
|
||||
metadata_store.update_task(task.id, status="running")
|
||||
timeline.append(
|
||||
task_id=task.id,
|
||||
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
||||
prompt="do something",
|
||||
tool_call={"action": "tap"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
metadata_store.update_task(task.id, status="completed", completed=True)
|
||||
|
||||
# Drop ALL in-memory references.
|
||||
del metadata_store
|
||||
del timeline
|
||||
del task
|
||||
|
||||
# Re-open from the same paths.
|
||||
reopened_store = TaskMetadataStore(db_path)
|
||||
reopened_timeline = Timeline(ArtifactStore(history_dir))
|
||||
|
||||
row = reopened_store.get_task("task-survive")
|
||||
assert row is not None
|
||||
assert row["status"] == "completed"
|
||||
assert row["goal"] == "test goal"
|
||||
|
||||
records = reopened_timeline.read("task-survive")
|
||||
assert len(records) == 1
|
||||
assert records[0]["tool_call"] == {"action": "tap"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Retention prunes tasks beyond configured threshold
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _seed_n_tasks(
|
||||
store: TaskMetadataStore,
|
||||
n: int,
|
||||
*,
|
||||
base: datetime = datetime(2026, 7, 1, tzinfo=UTC),
|
||||
) -> list[str]:
|
||||
ids: list[str] = []
|
||||
for i in range(n):
|
||||
task = _make_task(f"task-{i:02d}", created_at=base + timedelta(hours=i))
|
||||
store.create_task(task)
|
||||
ids.append(task.id)
|
||||
return ids
|
||||
|
||||
|
||||
def test_retention_prunes_by_max_count(tmp_path) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
ids = _seed_n_tasks(metadata_store, 10)
|
||||
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
task_retention_max_count=5,
|
||||
task_retention_max_age_days=365, # large so count is the binding constraint
|
||||
)
|
||||
pruned = prune_task_history(
|
||||
metadata_store,
|
||||
timeline,
|
||||
config=config,
|
||||
now=datetime(2026, 7, 1, tzinfo=UTC) + timedelta(days=1),
|
||||
)
|
||||
assert pruned == 5
|
||||
remaining = metadata_store.list_task_ids()
|
||||
assert len(remaining) == 5
|
||||
# list_tasks is ordered by created_at desc, so the 5 newest survive.
|
||||
expected_survivors = set(ids[5:])
|
||||
assert set(remaining) == expected_survivors
|
||||
|
||||
|
||||
def test_retention_prunes_by_max_age(tmp_path) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
base = datetime(2026, 7, 1, tzinfo=UTC)
|
||||
_seed_n_tasks(metadata_store, 10, base=base)
|
||||
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
task_retention_max_count=100, # large so age is the binding constraint
|
||||
task_retention_max_age_days=3,
|
||||
)
|
||||
# "now" is 5 days after base; tasks 0..2 are within 3 days of now-5d... actually:
|
||||
# tasks created at base+0h .. base+9h. now = base + 5 days.
|
||||
# cutoff = now - 3 days = base + 2 days. Tasks with created_at >= base+2d survive.
|
||||
# Only tasks created at base+0h..base+9h -- all are < base+2d, so all pruned.
|
||||
# Let's use a now that's closer.
|
||||
now = base + timedelta(hours=5)
|
||||
pruned = prune_task_history(metadata_store, timeline, config=config, now=now)
|
||||
# cutoff = now - 3 days. All tasks are within 3 days. None pruned by age.
|
||||
# Actually max_count=100 so nothing pruned at all.
|
||||
# Let's redo: use max_age_days to actually prune older tasks.
|
||||
assert pruned == 0 # all tasks are younger than 3 days
|
||||
|
||||
# Now use a stricter age that prunes some.
|
||||
config_strict = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
task_retention_max_count=100,
|
||||
task_retention_max_age_days=1,
|
||||
)
|
||||
# cutoff = now - 1 day = base + 5h - 24h = base - 19h. Still all tasks are after that.
|
||||
# So we need a much later "now".
|
||||
late_now = base + timedelta(days=10)
|
||||
pruned2 = prune_task_history(
|
||||
metadata_store, timeline, config=config_strict, now=late_now
|
||||
)
|
||||
# All tasks are ~10 days old, max_age=1 day. All pruned.
|
||||
assert pruned2 == 10
|
||||
assert metadata_store.list_task_ids() == []
|
||||
|
||||
|
||||
def test_retention_whichever_keeps_fewer_wins(tmp_path) -> None:
|
||||
"""The more restrictive bound between max_count and max_age wins."""
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
base = datetime(2026, 7, 1, tzinfo=UTC)
|
||||
_seed_n_tasks(metadata_store, 10, base=base)
|
||||
|
||||
# max_count=8 would keep 8, max_age=2 days from base+3d would keep tasks
|
||||
# created within 2 days of base+3d = base+1d..base+9h. Only tasks at base+0h..base+9h.
|
||||
# cutoff = base+3d - 2d = base+1d = base+24h. Tasks created at base+0h..base+9h
|
||||
# are all < base+24h, so keep_by_age = {} (0 tasks). keep_by_count = newest 8.
|
||||
# len(keep_by_age)=0 <= len(keep_by_count)=8, so keep_by_age wins -> 0 kept, all pruned.
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
task_retention_max_count=8,
|
||||
task_retention_max_age_days=2,
|
||||
)
|
||||
now = base + timedelta(days=3)
|
||||
pruned = prune_task_history(metadata_store, timeline, config=config, now=now)
|
||||
assert pruned == 10
|
||||
assert metadata_store.list_task_ids() == []
|
||||
|
||||
|
||||
def test_retention_deletes_timeline_alongside_metadata(tmp_path) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
# Create a task with timeline records.
|
||||
task = _make_task("task-to-prune")
|
||||
metadata_store.create_task(task)
|
||||
timeline.append(
|
||||
task_id=task.id,
|
||||
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
||||
prompt="step",
|
||||
tool_call={"action": "tap"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
assert len(timeline.read(task.id)) == 1
|
||||
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
task_retention_max_count=0,
|
||||
task_retention_max_age_days=365,
|
||||
)
|
||||
pruned = prune_task_history(
|
||||
metadata_store,
|
||||
timeline,
|
||||
config=config,
|
||||
now=datetime(2026, 7, 1, tzinfo=UTC),
|
||||
)
|
||||
assert pruned == 1
|
||||
assert metadata_store.get_task(task.id) is None
|
||||
assert timeline.read(task.id) == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# No collision between Host Agent and Runtime default paths
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_host_agent_default_db_path_differs_from_runtime_default() -> None:
|
||||
host_agent_config = HostAgentConfig(control_plane_url="https://control.example")
|
||||
# Use posix paths for cross-platform comparison.
|
||||
assert host_agent_config.task_progress_db_path.as_posix() == (
|
||||
"host_agent_data/task_progress.sqlite3"
|
||||
)
|
||||
|
||||
# Runtime's TaskMetadataStore default.
|
||||
runtime_default = TaskMetadataStore.__init__.__defaults__[0]
|
||||
assert Path(runtime_default).as_posix() == "tasks/tasks.sqlite3"
|
||||
|
||||
assert host_agent_config.task_progress_db_path != Path(runtime_default)
|
||||
|
||||
|
||||
def test_two_metadata_stores_at_different_paths_are_isolated(tmp_path) -> None:
|
||||
store_a = TaskMetadataStore(tmp_path / "a.sqlite3")
|
||||
store_b = TaskMetadataStore(tmp_path / "b.sqlite3")
|
||||
|
||||
task_a = _make_task("task-in-a")
|
||||
store_a.create_task(task_a)
|
||||
|
||||
assert store_a.get_task("task-in-a") is not None
|
||||
assert store_b.get_task("task-in-a") is None
|
||||
assert "task-in-a" not in store_b.list_task_ids()
|
||||
assert "task-in-a" in store_a.list_task_ids()
|
||||
@@ -31,3 +31,24 @@ def test_timeline_records_survive_reopening_store(tmp_path) -> None:
|
||||
assert [record["index"] for record in records] == [1, 2]
|
||||
assert records[0]["screenshot_path"].endswith("001.png")
|
||||
|
||||
|
||||
def test_timeline_records_per_step_prompt_not_task_goal(tmp_path) -> None:
|
||||
"""The prompt field should persist exactly what was passed to append(),
|
||||
not a pre-D9 task goal fallback."""
|
||||
store = ArtifactStore(tmp_path / "history")
|
||||
timeline = Timeline(store)
|
||||
|
||||
per_step_prompt = "Goal:\nsend a message\n\nCurrent Scene (JSON):\n{...}\n\nCall exactly one tool."
|
||||
timeline.append(
|
||||
task_id="task-42",
|
||||
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
||||
prompt=per_step_prompt,
|
||||
tool_call={"action": "tap"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
|
||||
records = timeline.read("task-42")
|
||||
assert len(records) == 1
|
||||
assert records[0]["prompt"] == per_step_prompt
|
||||
assert "Call exactly one tool" in records[0]["prompt"]
|
||||
|
||||
@@ -20,7 +20,9 @@ from tests.fakes import PNG_10X20
|
||||
|
||||
|
||||
class FakeMessages:
|
||||
def __init__(self, *, response: object | None = None, error: Exception | None = None) -> None:
|
||||
def __init__(
|
||||
self, *, response: object | None = None, error: Exception | None = None
|
||||
) -> None:
|
||||
self.response = response
|
||||
self.error = error
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
@@ -38,7 +40,9 @@ class FakeTransport:
|
||||
|
||||
|
||||
class FakeCompletions:
|
||||
def __init__(self, *, response: object | None = None, error: Exception | None = None) -> None:
|
||||
def __init__(
|
||||
self, *, response: object | None = None, error: Exception | None = None
|
||||
) -> None:
|
||||
self.response = response
|
||||
self.error = error
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
@@ -65,9 +69,13 @@ class FakeOpenAITransport:
|
||||
|
||||
def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -> None:
|
||||
messages = FakeMessages(
|
||||
response={"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]}
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
|
||||
decision = client.decide(
|
||||
system_prompt="system",
|
||||
@@ -77,14 +85,23 @@ def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -
|
||||
timeout=2.5,
|
||||
)
|
||||
|
||||
assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
assert decision == ToolCallDecision(
|
||||
tool_name="tap",
|
||||
arguments={"x": 1, "y": 2},
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
)
|
||||
assert len(messages.calls) == 1
|
||||
call = messages.calls[0]
|
||||
assert call["model"] == "test-model"
|
||||
assert call["timeout"] == 2.5
|
||||
assert call["tool_choice"] == {"type": "any", "disable_parallel_tool_use": True}
|
||||
assert call["tools"] == [
|
||||
{"name": "tap", "description": TAP_SPEC.description, "input_schema": TAP_SPEC.parameters},
|
||||
{
|
||||
"name": "tap",
|
||||
"description": TAP_SPEC.description,
|
||||
"input_schema": TAP_SPEC.parameters,
|
||||
},
|
||||
{
|
||||
"name": "finish_task",
|
||||
"description": FINISH_TASK_SPEC.description,
|
||||
@@ -93,18 +110,28 @@ def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -
|
||||
]
|
||||
assert call["system"][0]["text"] == "system"
|
||||
assert call["system"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert call["messages"] == [{"role": "user", "content": [{"type": "text", "text": "user"}]}]
|
||||
assert call["messages"] == [
|
||||
{"role": "user", "content": [{"type": "text", "text": "user"}]}
|
||||
]
|
||||
|
||||
|
||||
def test_anthropic_tool_calling_client_includes_image_block_when_screenshot_present() -> None:
|
||||
def test_anthropic_tool_calling_client_includes_image_block_when_screenshot_present() -> (
|
||||
None
|
||||
):
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "tool_use", "name": "finish_task", "input": {"success": True, "reason": "done"}}
|
||||
{
|
||||
"type": "tool_use",
|
||||
"name": "finish_task",
|
||||
"input": {"success": True, "reason": "done"},
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
|
||||
client.decide(
|
||||
system_prompt="system",
|
||||
@@ -157,10 +184,18 @@ def test_anthropic_tool_calling_client_passes_custom_base_url_to_sdk(
|
||||
|
||||
def test_anthropic_tool_calling_client_wraps_transport_errors() -> None:
|
||||
messages = FakeMessages(error=TimeoutError("timed out"))
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
client.decide(
|
||||
system_prompt="s",
|
||||
user_prompt="u",
|
||||
screenshot=None,
|
||||
tools=[TAP_SPEC],
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -171,12 +206,22 @@ def test_anthropic_tool_calling_client_wraps_transport_errors() -> None:
|
||||
{"content": [{"type": "tool_use", "name": "tap", "input": "not-a-dict"}]},
|
||||
],
|
||||
)
|
||||
def test_anthropic_tool_calling_client_wraps_malformed_responses(response: object) -> None:
|
||||
def test_anthropic_tool_calling_client_wraps_malformed_responses(
|
||||
response: object,
|
||||
) -> None:
|
||||
messages = FakeMessages(response=response)
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
client.decide(
|
||||
system_prompt="s",
|
||||
user_prompt="u",
|
||||
screenshot=None,
|
||||
tools=[TAP_SPEC],
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
|
||||
# --- OpenAI --------------------------------------------------------------
|
||||
@@ -186,11 +231,24 @@ def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> N
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{"message": {"tool_calls": [{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}]}}
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "tap",
|
||||
"arguments": '{"x": 1, "y": 2}',
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
|
||||
decision = client.decide(
|
||||
system_prompt="system",
|
||||
@@ -200,7 +258,12 @@ def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> N
|
||||
timeout=2.5,
|
||||
)
|
||||
|
||||
assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
assert decision == ToolCallDecision(
|
||||
tool_name="tap",
|
||||
arguments={"x": 1, "y": 2},
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
)
|
||||
assert len(completions.calls) == 1
|
||||
call = completions.calls[0]
|
||||
assert call["model"] == "test-model"
|
||||
@@ -233,7 +296,9 @@ def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> N
|
||||
]
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_includes_image_block_when_screenshot_present() -> None:
|
||||
def test_openai_tool_calling_client_includes_image_block_when_screenshot_present() -> (
|
||||
None
|
||||
):
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
@@ -252,7 +317,9 @@ def test_openai_tool_calling_client_includes_image_block_when_screenshot_present
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
|
||||
client.decide(
|
||||
system_prompt="system",
|
||||
@@ -275,22 +342,47 @@ def test_openai_tool_calling_client_includes_image_block_when_screenshot_present
|
||||
def test_openai_tool_calling_client_accepts_arguments_already_as_dict() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [{"message": {"tool_calls": [{"function": {"name": "tap", "arguments": {"x": 1, "y": 2}}}]}}]
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": {"x": 1, "y": 2}}}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
|
||||
decision = client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
|
||||
assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
assert decision == ToolCallDecision(
|
||||
tool_name="tap",
|
||||
arguments={"x": 1, "y": 2},
|
||||
system_prompt="s",
|
||||
user_prompt="u",
|
||||
)
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_wraps_transport_errors() -> None:
|
||||
completions = FakeCompletions(error=TimeoutError("timed out"))
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
client.decide(
|
||||
system_prompt="s",
|
||||
user_prompt="u",
|
||||
screenshot=None,
|
||||
tools=[TAP_SPEC],
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -298,15 +390,33 @@ def test_openai_tool_calling_client_wraps_transport_errors() -> None:
|
||||
[
|
||||
{"choices": []},
|
||||
{"choices": [{"message": {"tool_calls": []}}]},
|
||||
{"choices": [{"message": {"tool_calls": [{"function": {"name": "tap", "arguments": "not-json"}}]}}]},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": "not-json"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_openai_tool_calling_client_wraps_malformed_responses(response: object) -> None:
|
||||
completions = FakeCompletions(response=response)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
client.decide(
|
||||
system_prompt="s",
|
||||
user_prompt="u",
|
||||
screenshot=None,
|
||||
tools=[TAP_SPEC],
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
|
||||
# --- build_client ----------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user