Add Host Agent local console task cancellation (task-cancellation 7.1-7.3)
- New internal API route POST /internal/v1/hosts/{host_id}/tasks/{task_id}/cancel,
authenticated via the host's own bearer credential (authorize_host) with an
ownership check, since host tokens carry no scopes and cannot reach the
public SDK's tasks:submit-scoped cancel endpoint.
- HostAgentClient.cancel_task() calls the new internal route directly.
- create_console_app() gains a cancel_task callable with automatic default
wiring from host_client, so production app.py needs no changes.
- Local console: POST /tasks/{task_id}/cancel route resolves the local
execution id to its Cloud source_task_id before cancelling, and the task
detail page/template show a Cancel button plus notice/error banners.
- Tests across all three layers: internal API route, Jinja2 template
rendering, and FastAPI console route behavior.
This commit is contained in:
@@ -296,6 +296,192 @@ def test_host_policy_converges_and_disables_self_submission(tmp_path) -> None:
|
||||
assert disabled.status_code == 403
|
||||
|
||||
|
||||
def _enqueue_task_for_host(
|
||||
pool: DevicePool,
|
||||
*,
|
||||
task_id: str,
|
||||
host_id: str,
|
||||
goal: str = "cancel me",
|
||||
) -> None:
|
||||
pool.store.enqueue_task(
|
||||
ScheduledTask(
|
||||
id=task_id,
|
||||
goal=goal,
|
||||
workflow_definition_id=None,
|
||||
constraints=TaskConstraints(target_host_id=host_id),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _register_idle_device(
|
||||
pool: DevicePool, *, host_id: str, device_id: str, now: datetime
|
||||
) -> None:
|
||||
pool.store.upsert_host(host_id, address=None, last_seen_at=now)
|
||||
pool.store.replace_host_devices(
|
||||
host_id,
|
||||
[
|
||||
PooledDevice(
|
||||
device_id=device_id,
|
||||
host_id=host_id,
|
||||
driver_type="wda",
|
||||
status="idle",
|
||||
synced_at=now,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_cancel_host_task_transitions_queued_task_to_cancelled_immediately(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
_enqueue_task_for_host(pool, task_id="task-1", host_id="host-a")
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/task-1/cancel",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"task_id": "task-1", "status": "cancelled"}
|
||||
assert pool.store.get_task("task-1").status == "cancelled" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_cancel_host_task_returns_202_for_pending_assigned_task(tmp_path) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
now = datetime.now(UTC)
|
||||
_enqueue_task_for_host(pool, task_id="task-2", host_id="host-a")
|
||||
_register_idle_device(pool, host_id="host-a", device_id="device-a", now=now)
|
||||
pool.store.assign_task(
|
||||
task_id="task-2",
|
||||
host_id="host-a",
|
||||
device_id="device-a",
|
||||
lease_id="lease-2",
|
||||
lease_expires_at=now + timedelta(minutes=1),
|
||||
now=now,
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/task-2/cancel",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
)
|
||||
|
||||
assert response.status_code == 202, response.text
|
||||
body = response.json()
|
||||
assert body["task_id"] == "task-2"
|
||||
assert body["status"] == "assigned"
|
||||
task = pool.store.get_task("task-2")
|
||||
assert task is not None
|
||||
assert task.cancel_requested_at is not None
|
||||
|
||||
|
||||
def test_cancel_host_task_repeat_call_on_pending_request_is_idempotent(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
now = datetime.now(UTC)
|
||||
_enqueue_task_for_host(pool, task_id="task-3", host_id="host-a")
|
||||
_register_idle_device(pool, host_id="host-a", device_id="device-a", now=now)
|
||||
pool.store.assign_task(
|
||||
task_id="task-3",
|
||||
host_id="host-a",
|
||||
device_id="device-a",
|
||||
lease_id="lease-3",
|
||||
lease_expires_at=now + timedelta(minutes=1),
|
||||
now=now,
|
||||
)
|
||||
headers = {"Authorization": "Bearer token-a"}
|
||||
|
||||
first = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/task-3/cancel", headers=headers
|
||||
)
|
||||
second = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/task-3/cancel", headers=headers
|
||||
)
|
||||
|
||||
assert first.status_code == 202, first.text
|
||||
assert second.status_code == 200, second.text
|
||||
assert second.json() == {"task_id": "task-3", "status": "assigned"}
|
||||
|
||||
|
||||
def test_cancel_host_task_rejects_terminal_task(tmp_path) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
now = datetime.now(UTC)
|
||||
_enqueue_task_for_host(pool, task_id="task-4", host_id="host-a")
|
||||
_register_idle_device(pool, host_id="host-a", device_id="device-a", now=now)
|
||||
pool.store.assign_task(
|
||||
task_id="task-4",
|
||||
host_id="host-a",
|
||||
device_id="device-a",
|
||||
lease_id="lease-4",
|
||||
lease_expires_at=now + timedelta(minutes=1),
|
||||
now=now,
|
||||
)
|
||||
pool.store.record_task_result(
|
||||
task_id="task-4",
|
||||
attempt=1,
|
||||
lease_id="lease-4",
|
||||
host_id="host-a",
|
||||
status="done",
|
||||
failure_reason=None,
|
||||
terminal_result=None,
|
||||
completed_at=now,
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/task-4/cancel",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
)
|
||||
|
||||
assert response.status_code == 409, response.text
|
||||
|
||||
|
||||
def test_cancel_host_task_rejects_unknown_task_id(tmp_path) -> None:
|
||||
client, _ = _build_client(tmp_path)
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/does-not-exist/cancel",
|
||||
headers={"Authorization": "Bearer token-a"},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_cancel_host_task_rejects_task_owned_by_other_host(tmp_path) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
_enqueue_task_for_host(pool, task_id="task-5", host_id="host-a")
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-b/tasks/task-5/cancel",
|
||||
headers={"Authorization": "Bearer token-b"},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert pool.store.get_task("task-5").status == "queued" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_cancel_host_task_rejects_mismatched_host_identity(tmp_path) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
_enqueue_task_for_host(pool, task_id="task-6", host_id="host-a")
|
||||
|
||||
response = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks/task-6/cancel",
|
||||
headers={"Authorization": "Bearer token-b"},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert pool.store.get_task("task-6").status == "queued" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_cancel_host_task_requires_authentication(tmp_path) -> None:
|
||||
client, _ = _build_client(tmp_path)
|
||||
|
||||
response = client.post("/internal/v1/hosts/host-a/tasks/task-7/cancel")
|
||||
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_planner_proxy_reserves_and_enforces_host_daily_token_budget(tmp_path) -> None:
|
||||
class FakePlannerClient:
|
||||
calls = 0
|
||||
|
||||
Reference in New Issue
Block a user