This commit is contained in:
@@ -16,10 +16,10 @@ contracts.
|
||||
- `driver/`: the `Driver` contract, concrete driver adapters, and driver-type
|
||||
registry.
|
||||
- `device/`: device lifecycle and active driver management.
|
||||
- `tools/`: device capabilities exposed to runtime and API layers.
|
||||
- `tools/`: device capabilities exposed to Runtime and adapter layers.
|
||||
- `perception/`: screen-to-`Scene` perception behind `PerceptionProvider`.
|
||||
- `runtime/`: planning and execution orchestration.
|
||||
- `api/`: REST/MCP transport adapters.
|
||||
- `api/`: MCP and supporting integration adapters.
|
||||
- `storage/`: timeline, task, and device configuration persistence.
|
||||
- `packages/cloud-platform/`: cloud scheduling, device pooling, plugins, and
|
||||
the Python cloud SDK as the `device-cloud-platform` workspace member.
|
||||
@@ -54,12 +54,11 @@ uv build --package device-agent-runtime
|
||||
uv build --package device-cloud-platform
|
||||
```
|
||||
|
||||
The Runtime API ships a same-origin operator console at `/ui/`, rendered through
|
||||
Jinja2 templates packaged with `device-agent-runtime`. Start the API
|
||||
(`uvicorn api.rest:create_app --factory`) and open `/` (it redirects to `/ui/`).
|
||||
The `/console/*` JSON endpoints remain available for programmatic clients. The
|
||||
console assumes a trusted local network; it has no authentication, authorization,
|
||||
or CSRF protection.
|
||||
Runtime is an in-process execution library, not a standalone HTTP service. The
|
||||
Host Agent console at `http://127.0.0.1:8765/tasks` is the authenticated
|
||||
operator view for the tasks that actually execute on that Host, including
|
||||
per-step screenshots, OCR observations, and UI-tree results. The Cloud Console
|
||||
remains the fleet-level view for dispatch status and Cloud-proxy planner history.
|
||||
|
||||
## Project Direction
|
||||
|
||||
@@ -73,5 +72,5 @@ invariants future changes must preserve are in
|
||||
or deployed PostgreSQL, configure credentials and Runtime AI planning, and
|
||||
perform orderly shutdown or rollback.
|
||||
- [macOS migration and real iPhone setup](docs/MACOS_IPHONE_SETUP.md): install
|
||||
Xcode, Appium/XCUITest, sign WebDriverAgent, verify a real device, and start a
|
||||
connected Runtime API.
|
||||
Xcode, Appium/XCUITest, sign WebDriverAgent, verify a real device, and run a
|
||||
connected Host Agent.
|
||||
|
||||
-225
@@ -1,225 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from device.manager import DeviceManager
|
||||
from driver.registry import build_driver_factory
|
||||
from pydantic import BaseModel, Field
|
||||
from runtime.task import TaskRunner
|
||||
from storage.device_config import DeviceConfigStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
|
||||
|
||||
class RegisterDeviceRequest(BaseModel):
|
||||
driver_type: str
|
||||
connection_info: dict[str, Any] = Field(default_factory=dict)
|
||||
name: str | None = None
|
||||
|
||||
|
||||
class RuntimeConfigRequest(BaseModel):
|
||||
max_steps: int
|
||||
|
||||
|
||||
class ConsoleService:
|
||||
"""API-local typed operations shared by JSON and HTML console routers.
|
||||
|
||||
Stays in the ``api`` layer: it only depends on the existing injected
|
||||
Runtime state (device manager, metadata store, timeline, config store,
|
||||
task runner) and does not introduce HTTP/UI concerns below this layer.
|
||||
Page and JSON handlers receive the same instance so validation, driver
|
||||
allow-listing, persistence, and error mapping cannot drift apart.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
device_manager: DeviceManager,
|
||||
metadata_store: TaskMetadataStore,
|
||||
timeline: Timeline,
|
||||
config_store: DeviceConfigStore,
|
||||
task_runner: TaskRunner,
|
||||
) -> None:
|
||||
self._device_manager = device_manager
|
||||
self._metadata_store = metadata_store
|
||||
self._timeline = timeline
|
||||
self._config_store = config_store
|
||||
self._task_runner = task_runner
|
||||
|
||||
def list_devices(self) -> list[dict[str, Any]]:
|
||||
return [device.to_dict() for device in self._device_manager.list_devices()]
|
||||
|
||||
def register_device(
|
||||
self,
|
||||
*,
|
||||
driver_type: str,
|
||||
connection_info: dict[str, Any],
|
||||
name: str | None,
|
||||
) -> dict[str, Any]:
|
||||
driver_factory = build_driver_factory(driver_type, connection_info)
|
||||
device_id = uuid4().hex
|
||||
device = self._device_manager.register_device(
|
||||
device_id,
|
||||
driver_factory,
|
||||
name=name,
|
||||
driver_type=driver_type,
|
||||
connection_info=connection_info,
|
||||
)
|
||||
self._config_store.add(
|
||||
device_id=device_id,
|
||||
name=name,
|
||||
driver_type=driver_type,
|
||||
connection_info=connection_info,
|
||||
)
|
||||
return device.to_dict()
|
||||
|
||||
def unregister_device(self, device_id: str) -> None:
|
||||
if not self._has_device(device_id):
|
||||
raise KeyError("device not found")
|
||||
self._device_manager.unregister_device(device_id)
|
||||
self._config_store.remove(device_id)
|
||||
|
||||
def list_tasks(
|
||||
self,
|
||||
*,
|
||||
device_id: str | None = None,
|
||||
status: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = self._metadata_store.list_tasks()
|
||||
if device_id is not None:
|
||||
rows = [row for row in rows if row["device_id"] == device_id]
|
||||
if status is not None:
|
||||
rows = [row for row in rows if row["status"] == status]
|
||||
return rows
|
||||
|
||||
def get_task(self, task_id: str) -> dict[str, Any]:
|
||||
task = self._metadata_store.get_task(task_id)
|
||||
if task is None:
|
||||
raise KeyError("task not found")
|
||||
return task
|
||||
|
||||
def get_task_timeline(self, task_id: str) -> list[dict[str, Any]]:
|
||||
if self._metadata_store.get_task(task_id) is None:
|
||||
raise KeyError("task not found")
|
||||
return [
|
||||
self._inline_screenshots(record) for record in self._timeline.read(task_id)
|
||||
]
|
||||
|
||||
def get_runtime_config(self) -> dict[str, int]:
|
||||
return {"max_steps": self._runner_max_steps()}
|
||||
|
||||
def update_runtime_config(self, *, max_steps: int) -> dict[str, int]:
|
||||
if max_steps <= 0:
|
||||
raise ValueError("max_steps must be positive")
|
||||
self._set_runner_max_steps(max_steps)
|
||||
self._config_store.set_setting("max_steps", max_steps)
|
||||
return {"max_steps": max_steps}
|
||||
|
||||
def _has_device(self, device_id: str) -> bool:
|
||||
return any(
|
||||
device.id == device_id for device in self._device_manager.list_devices()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _inline_screenshots(record: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(record)
|
||||
for path_key, image_key in (
|
||||
("before_screenshot_path", "before_image_base64"),
|
||||
("after_screenshot_path", "after_image_base64"),
|
||||
("screenshot_path", "image_base64"),
|
||||
):
|
||||
screenshot_path = payload.get(path_key)
|
||||
if not screenshot_path:
|
||||
continue
|
||||
path = Path(str(screenshot_path))
|
||||
if path.exists():
|
||||
payload[image_key] = base64.b64encode(path.read_bytes()).decode("ascii")
|
||||
if "after_image_base64" not in payload and "image_base64" in payload:
|
||||
payload["after_image_base64"] = payload["image_base64"]
|
||||
elif "after_image_base64" in payload:
|
||||
payload["image_base64"] = payload["after_image_base64"]
|
||||
return payload
|
||||
|
||||
def _runner_max_steps(self) -> int:
|
||||
config = getattr(self._task_runner, "config", None)
|
||||
if config is None or not hasattr(config, "max_steps"):
|
||||
raise RuntimeError("task runner config unavailable")
|
||||
return int(config.max_steps)
|
||||
|
||||
def _set_runner_max_steps(self, max_steps: int) -> None:
|
||||
config = getattr(self._task_runner, "config", None)
|
||||
if config is None or not hasattr(config, "max_steps"):
|
||||
raise RuntimeError("task runner config unavailable")
|
||||
config.max_steps = max_steps
|
||||
|
||||
|
||||
def create_console_router(service: ConsoleService) -> Any:
|
||||
from fastapi import APIRouter, HTTPException, Response, status
|
||||
|
||||
router = APIRouter(prefix="/console", tags=["console"])
|
||||
|
||||
@router.get("/devices")
|
||||
def devices() -> list[dict[str, Any]]:
|
||||
return service.list_devices()
|
||||
|
||||
@router.post("/devices", status_code=status.HTTP_201_CREATED)
|
||||
def register_device(request: RegisterDeviceRequest) -> dict[str, Any]:
|
||||
try:
|
||||
return service.register_device(
|
||||
driver_type=request.driver_type,
|
||||
connection_info=request.connection_info,
|
||||
name=request.name,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@router.delete(
|
||||
"/devices/{device_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
response_model=None,
|
||||
)
|
||||
def unregister_device(device_id: str) -> Response:
|
||||
try:
|
||||
service.unregister_device(device_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
@router.get("/tasks")
|
||||
def tasks(
|
||||
device_id: str | None = None,
|
||||
status: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
return service.list_tasks(device_id=device_id, status=status)
|
||||
|
||||
@router.get("/tasks/{task_id}")
|
||||
def task_detail(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return service.get_task(task_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@router.get("/tasks/{task_id}/timeline")
|
||||
def task_timeline(task_id: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
return service.get_task_timeline(task_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@router.get("/config")
|
||||
def runtime_config() -> dict[str, int]:
|
||||
return service.get_runtime_config()
|
||||
|
||||
@router.put("/config")
|
||||
def update_runtime_config(
|
||||
request: RuntimeConfigRequest,
|
||||
) -> dict[str, int]:
|
||||
try:
|
||||
return service.update_runtime_config(max_steps=request.max_steps)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
return router
|
||||
@@ -1,360 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from api.console import ConsoleService
|
||||
from driver.registry import SUPPORTED_DRIVER_TYPES
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
|
||||
_TEMPLATE_DIR = Path(__file__).parent / "templates" / "runtime_console"
|
||||
_STATIC_DIR = Path(__file__).parent / "static" / "runtime_console"
|
||||
|
||||
_ENV = Environment(
|
||||
loader=FileSystemLoader(str(_TEMPLATE_DIR)),
|
||||
autoescape=select_autoescape(["html", "xml"]),
|
||||
auto_reload=False,
|
||||
)
|
||||
|
||||
_TASK_STATUSES = ["created", "running", "completed", "failed", "cancelled"]
|
||||
_DEFAULT_FORM_VALUES: dict[str, str] = {
|
||||
"name": "",
|
||||
"driver_type": "",
|
||||
"server_url": "",
|
||||
"udid": "",
|
||||
"wda_local_port": "",
|
||||
}
|
||||
|
||||
|
||||
def _render(request: Request, template_name: str, **context: Any) -> str:
|
||||
return _ENV.get_template(template_name).render(
|
||||
url_for=request.url_for,
|
||||
**context,
|
||||
)
|
||||
|
||||
|
||||
def _device_names(service: ConsoleService) -> dict[str, str]:
|
||||
return {
|
||||
device["id"]: device.get("name") or device["id"]
|
||||
for device in service.list_devices()
|
||||
}
|
||||
|
||||
|
||||
def _default_driver_type() -> str:
|
||||
if not _DEFAULT_FORM_VALUES["driver_type"] and SUPPORTED_DRIVER_TYPES:
|
||||
_DEFAULT_FORM_VALUES["driver_type"] = next(iter(SUPPORTED_DRIVER_TYPES))
|
||||
return _DEFAULT_FORM_VALUES["driver_type"]
|
||||
|
||||
|
||||
def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw_results = record.get("ocr_results")
|
||||
if not isinstance(raw_results, list):
|
||||
return []
|
||||
return [result for result in raw_results if isinstance(result, dict)]
|
||||
|
||||
|
||||
def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
tool_call = record.get("tool_call")
|
||||
if not isinstance(tool_call, dict):
|
||||
return []
|
||||
if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}:
|
||||
return []
|
||||
step_result = record.get("result")
|
||||
if not isinstance(step_result, dict):
|
||||
return []
|
||||
raw_nodes = step_result.get("result")
|
||||
if not isinstance(raw_nodes, list):
|
||||
return []
|
||||
return [node for node in raw_nodes if isinstance(node, dict)]
|
||||
|
||||
|
||||
def _config_context(
|
||||
request: Request,
|
||||
service: ConsoleService,
|
||||
*,
|
||||
form_values: dict[str, str] | None = None,
|
||||
device_error: str = "",
|
||||
device_added: str = "",
|
||||
config_error: str = "",
|
||||
config_saved: bool = False,
|
||||
max_steps_override: int | None = None,
|
||||
) -> str:
|
||||
form = dict(_DEFAULT_FORM_VALUES)
|
||||
form["driver_type"] = _default_driver_type()
|
||||
if form_values:
|
||||
form.update(form_values)
|
||||
config = service.get_runtime_config()
|
||||
return _render(
|
||||
request,
|
||||
"config.html",
|
||||
active_section="config",
|
||||
devices=service.list_devices(),
|
||||
max_steps=max_steps_override
|
||||
if max_steps_override is not None
|
||||
else config["max_steps"],
|
||||
supported_driver_types=list(SUPPORTED_DRIVER_TYPES),
|
||||
form_values=form,
|
||||
device_error=device_error,
|
||||
device_added=device_added,
|
||||
config_error=config_error,
|
||||
config_saved=config_saved,
|
||||
)
|
||||
|
||||
|
||||
def create_console_web_router(service: ConsoleService) -> APIRouter:
|
||||
router = APIRouter(prefix="/ui", tags=["console-ui"])
|
||||
|
||||
@router.get("/", name="runtime_console_dashboard", include_in_schema=False)
|
||||
def dashboard(request: Request) -> HTMLResponse:
|
||||
devices = service.list_devices()
|
||||
tasks = service.list_tasks()
|
||||
running = sum(1 for task in tasks if task.get("status") == "running")
|
||||
failed = sum(1 for task in tasks if task.get("status") == "failed")
|
||||
return HTMLResponse(
|
||||
_render(
|
||||
request,
|
||||
"dashboard.html",
|
||||
active_section="dashboard",
|
||||
devices=devices,
|
||||
device_count=len(devices),
|
||||
task_count=len(tasks),
|
||||
running_tasks=running,
|
||||
failed_tasks=failed,
|
||||
)
|
||||
)
|
||||
|
||||
@router.get(
|
||||
"/_status_fragment",
|
||||
name="runtime_console_status_fragment",
|
||||
include_in_schema=False,
|
||||
)
|
||||
def status_fragment(request: Request) -> HTMLResponse:
|
||||
devices = service.list_devices()
|
||||
tasks = service.list_tasks()
|
||||
running = sum(1 for task in tasks if task.get("status") == "running")
|
||||
failed = sum(1 for task in tasks if task.get("status") == "failed")
|
||||
return HTMLResponse(
|
||||
_render(
|
||||
request,
|
||||
"_status_fragment.html",
|
||||
devices=devices,
|
||||
device_count=len(devices),
|
||||
running_tasks=running,
|
||||
failed_tasks=failed,
|
||||
)
|
||||
)
|
||||
|
||||
@router.get("/tasks", name="runtime_console_tasks", include_in_schema=False)
|
||||
def tasks(
|
||||
request: Request,
|
||||
device_id: str | None = None,
|
||||
status: str | None = None,
|
||||
) -> HTMLResponse:
|
||||
service_tasks = service.list_tasks(device_id=device_id, status=status)
|
||||
return HTMLResponse(
|
||||
_render(
|
||||
request,
|
||||
"tasks.html",
|
||||
active_section="tasks",
|
||||
tasks=service_tasks,
|
||||
devices=service.list_devices(),
|
||||
device_names=_device_names(service),
|
||||
task_statuses=_TASK_STATUSES,
|
||||
selected_device_id=device_id or "",
|
||||
selected_status=status or "",
|
||||
)
|
||||
)
|
||||
|
||||
@router.get(
|
||||
"/tasks/{task_id}",
|
||||
name="runtime_console_task_detail",
|
||||
include_in_schema=False,
|
||||
)
|
||||
def task_detail(
|
||||
request: Request,
|
||||
task_id: str,
|
||||
step: int | None = None,
|
||||
) -> HTMLResponse:
|
||||
try:
|
||||
task = service.get_task(task_id)
|
||||
timeline = service.get_task_timeline(task_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
if timeline:
|
||||
index = 0
|
||||
if step is not None:
|
||||
index = max(0, min(step, len(timeline) - 1))
|
||||
current_step = timeline[index]
|
||||
else:
|
||||
index = 0
|
||||
current_step = None
|
||||
device_name = _device_names(service).get(
|
||||
task.get("device_id", ""),
|
||||
task.get("device_id", ""),
|
||||
)
|
||||
return HTMLResponse(
|
||||
_render(
|
||||
request,
|
||||
"task_detail.html",
|
||||
active_section="tasks",
|
||||
task=task,
|
||||
device_name=device_name,
|
||||
timeline=timeline,
|
||||
current_step=current_step if current_step is not None else {},
|
||||
current_step_index=index,
|
||||
ocr_results=_ocr_results(current_step)
|
||||
if current_step is not None
|
||||
else [],
|
||||
ui_tree_nodes=_ui_tree_nodes(current_step)
|
||||
if current_step is not None
|
||||
else [],
|
||||
)
|
||||
)
|
||||
|
||||
@router.get("/config", name="runtime_console_config", include_in_schema=False)
|
||||
def config(request: Request) -> HTMLResponse:
|
||||
return HTMLResponse(_config_context(request, service))
|
||||
|
||||
@router.post(
|
||||
"/config/devices",
|
||||
name="runtime_console_register_device",
|
||||
include_in_schema=False,
|
||||
response_model=None,
|
||||
)
|
||||
async def register_device(request: Request) -> HTMLResponse | RedirectResponse:
|
||||
form = await request.form()
|
||||
name = (form.get("name") or "").strip()
|
||||
driver_type = (form.get("driver_type") or "").strip()
|
||||
server_url = (form.get("server_url") or "").strip()
|
||||
udid = (form.get("udid") or "").strip()
|
||||
wda_local_port_raw = (form.get("wda_local_port") or "").strip()
|
||||
|
||||
connection_info: dict[str, Any] = {}
|
||||
if server_url:
|
||||
connection_info["server_url"] = server_url
|
||||
if udid:
|
||||
connection_info["udid"] = udid
|
||||
if wda_local_port_raw:
|
||||
try:
|
||||
connection_info["wda_local_port"] = int(wda_local_port_raw)
|
||||
except ValueError:
|
||||
return HTMLResponse(
|
||||
_config_context(
|
||||
request,
|
||||
service,
|
||||
form_values={
|
||||
"name": name,
|
||||
"driver_type": driver_type,
|
||||
"server_url": server_url,
|
||||
"udid": udid,
|
||||
"wda_local_port": wda_local_port_raw,
|
||||
},
|
||||
device_error="wda_local_port must be a number",
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
try:
|
||||
service.register_device(
|
||||
driver_type=driver_type,
|
||||
connection_info=connection_info,
|
||||
name=name or None,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return HTMLResponse(
|
||||
_config_context(
|
||||
request,
|
||||
service,
|
||||
form_values={
|
||||
"name": name,
|
||||
"driver_type": driver_type,
|
||||
"server_url": server_url,
|
||||
"udid": udid,
|
||||
"wda_local_port": wda_local_port_raw,
|
||||
},
|
||||
device_error=str(exc),
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
return RedirectResponse(
|
||||
url=request.url_for("runtime_console_config"),
|
||||
status_code=303,
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/config/devices/{device_id}/delete",
|
||||
name="runtime_console_remove_device",
|
||||
include_in_schema=False,
|
||||
response_model=None,
|
||||
)
|
||||
async def remove_device(
|
||||
request: Request,
|
||||
device_id: str,
|
||||
) -> HTMLResponse | RedirectResponse:
|
||||
try:
|
||||
service.unregister_device(device_id)
|
||||
except KeyError as exc:
|
||||
return HTMLResponse(
|
||||
_config_context(request, service, device_error=str(exc)),
|
||||
status_code=404,
|
||||
)
|
||||
return RedirectResponse(
|
||||
url=request.url_for("runtime_console_config"),
|
||||
status_code=303,
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/config/max-steps",
|
||||
name="runtime_console_update_max_steps",
|
||||
include_in_schema=False,
|
||||
response_model=None,
|
||||
)
|
||||
async def update_max_steps(request: Request) -> HTMLResponse | RedirectResponse:
|
||||
form = await request.form()
|
||||
raw = (form.get("max_steps") or "").strip()
|
||||
try:
|
||||
max_steps = int(raw)
|
||||
except ValueError:
|
||||
return HTMLResponse(
|
||||
_config_context(
|
||||
request,
|
||||
service,
|
||||
config_error="max_steps must be an integer",
|
||||
max_steps_override=raw if raw else None,
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
try:
|
||||
service.update_runtime_config(max_steps=max_steps)
|
||||
except ValueError as exc:
|
||||
return HTMLResponse(
|
||||
_config_context(
|
||||
request,
|
||||
service,
|
||||
config_error=str(exc),
|
||||
max_steps_override=max_steps,
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
return RedirectResponse(
|
||||
url=request.url_for("runtime_console_config"),
|
||||
status_code=303,
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def mount_console_assets(app: Any) -> None:
|
||||
"""Mount the package-owned static assets under ``/ui/assets``.
|
||||
|
||||
Kept as a separate helper so ``api.rest`` can mount assets with a stable
|
||||
route name that templates reference via ``url_for``.
|
||||
"""
|
||||
app.mount(
|
||||
"/ui/assets",
|
||||
StaticFiles(directory=str(_STATIC_DIR)),
|
||||
name="runtime_console_assets",
|
||||
)
|
||||
-171
@@ -1,171 +0,0 @@
|
||||
from typing import Any
|
||||
|
||||
from api.console import ConsoleService, create_console_router
|
||||
from api.console_web import create_console_web_router, mount_console_assets
|
||||
from api.errors import semantic_error
|
||||
from core.models import Task
|
||||
from device.manager import DEFAULT_MANAGER, DeviceManager
|
||||
from driver.registry import build_driver_factory
|
||||
from runtime.executor import Executor, default_tool_registry
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from storage.device_config import DEFAULT_MAX_STEPS, DeviceConfigStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
from tools.launch_app import launch_app
|
||||
from tools.screenshot import take_screenshot
|
||||
from tools.tap import tap
|
||||
|
||||
|
||||
def create_app(
|
||||
*,
|
||||
manager: DeviceManager | None = None,
|
||||
task_runner: TaskRunner | None = None,
|
||||
metadata_store: TaskMetadataStore | None = None,
|
||||
device_config_store: DeviceConfigStore | None = None,
|
||||
timeline: Timeline | None = None,
|
||||
) -> Any:
|
||||
from fastapi import BackgroundTasks, FastAPI, HTTPException
|
||||
from fastapi.responses import RedirectResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
device_manager = manager or DEFAULT_MANAGER
|
||||
store = metadata_store or TaskMetadataStore()
|
||||
config_store = device_config_store or DeviceConfigStore()
|
||||
timeline_store = timeline or Timeline()
|
||||
max_steps = _load_max_steps(config_store)
|
||||
_reload_device_configs(device_manager, config_store)
|
||||
runner = task_runner or TaskRunner(
|
||||
metadata_store=store,
|
||||
executor=Executor(tools=default_tool_registry(manager=device_manager)),
|
||||
timeline=timeline_store,
|
||||
config=TaskRunnerConfig(max_steps=max_steps),
|
||||
)
|
||||
_apply_max_steps(runner, max_steps)
|
||||
app = FastAPI(title="Apex Agent API")
|
||||
|
||||
class TapRequest(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
|
||||
class ScreenshotResponse(BaseModel):
|
||||
image_base64: str
|
||||
mime_type: str = "image/png"
|
||||
|
||||
class LaunchRequest(BaseModel):
|
||||
app_id: str
|
||||
|
||||
class AgentTaskRequest(BaseModel):
|
||||
goal: str
|
||||
device_id: str
|
||||
|
||||
console_service = ConsoleService(
|
||||
device_manager=device_manager,
|
||||
metadata_store=store,
|
||||
timeline=timeline_store,
|
||||
config_store=config_store,
|
||||
task_runner=runner,
|
||||
)
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
def root() -> RedirectResponse:
|
||||
return RedirectResponse(url="/ui/")
|
||||
|
||||
@app.get("/devices")
|
||||
def devices() -> list[dict[str, Any]]:
|
||||
return [device.to_dict() for device in device_manager.list_devices()]
|
||||
|
||||
@app.post("/devices/{device_id}/tap")
|
||||
def tap_device(device_id: str, request: TapRequest) -> dict[str, Any]:
|
||||
return _raise_semantic(
|
||||
lambda: tap(
|
||||
request.x,
|
||||
request.y,
|
||||
device_id=device_id,
|
||||
manager=device_manager,
|
||||
)
|
||||
)
|
||||
|
||||
@app.post("/devices/{device_id}/screenshot")
|
||||
def screenshot_device(device_id: str) -> dict[str, str]:
|
||||
import base64
|
||||
|
||||
image = _raise_semantic(
|
||||
lambda: take_screenshot(device_id, manager=device_manager)
|
||||
)
|
||||
response = ScreenshotResponse(
|
||||
image_base64=base64.b64encode(image).decode("ascii")
|
||||
)
|
||||
return response.model_dump()
|
||||
|
||||
@app.post("/devices/{device_id}/launch")
|
||||
def launch_device(device_id: str, request: LaunchRequest) -> dict[str, Any]:
|
||||
return _raise_semantic(
|
||||
lambda: launch_app(
|
||||
request.app_id,
|
||||
device_id=device_id,
|
||||
manager=device_manager,
|
||||
)
|
||||
)
|
||||
|
||||
@app.post("/agent/task")
|
||||
def start_task(
|
||||
request: AgentTaskRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
) -> dict[str, str]:
|
||||
task = Task(goal=request.goal, device_id=request.device_id)
|
||||
store.create_task(task)
|
||||
background_tasks.add_task(runner.run, task)
|
||||
return {"task_id": task.id, "status": task.status}
|
||||
|
||||
@app.get("/task/{task_id}")
|
||||
def get_task(task_id: str) -> dict[str, Any]:
|
||||
task = store.get_task(task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="task not found")
|
||||
return task
|
||||
|
||||
app.include_router(create_console_router(console_service))
|
||||
app.include_router(create_console_web_router(console_service))
|
||||
mount_console_assets(app)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _raise_semantic(func: Any) -> Any:
|
||||
from fastapi import HTTPException
|
||||
|
||||
try:
|
||||
return func()
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=semantic_error(exc)) from exc
|
||||
|
||||
|
||||
def _load_max_steps(config_store: DeviceConfigStore) -> int:
|
||||
raw_value = config_store.get_setting("max_steps")
|
||||
try:
|
||||
max_steps = int(raw_value) if raw_value is not None else DEFAULT_MAX_STEPS
|
||||
except ValueError:
|
||||
return DEFAULT_MAX_STEPS
|
||||
if max_steps <= 0:
|
||||
return DEFAULT_MAX_STEPS
|
||||
return max_steps
|
||||
|
||||
|
||||
def _apply_max_steps(task_runner: Any, max_steps: int) -> None:
|
||||
config = getattr(task_runner, "config", None)
|
||||
if config is not None and hasattr(config, "max_steps"):
|
||||
config.max_steps = max_steps
|
||||
|
||||
|
||||
def _reload_device_configs(
|
||||
device_manager: DeviceManager,
|
||||
config_store: DeviceConfigStore,
|
||||
) -> None:
|
||||
for config in config_store.list():
|
||||
device_manager.register_device(
|
||||
config["device_id"],
|
||||
build_driver_factory(config["driver_type"], config["connection_info"]),
|
||||
name=config["name"],
|
||||
driver_type=config["driver_type"],
|
||||
connection_info=config["connection_info"],
|
||||
)
|
||||
@@ -1,623 +0,0 @@
|
||||
:root {
|
||||
color: #202124;
|
||||
background: #f6f7f9;
|
||||
font-family:
|
||||
Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI",
|
||||
sans-serif;
|
||||
font-synthesis: none;
|
||||
text-rendering: optimizeLegibility;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 320px;
|
||||
min-height: 100vh;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
button,
|
||||
input,
|
||||
select {
|
||||
font: inherit;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
button {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
cursor: not-allowed;
|
||||
opacity: 0.55;
|
||||
}
|
||||
|
||||
a {
|
||||
color: inherit;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.app-shell {
|
||||
display: grid;
|
||||
grid-template-columns: 248px minmax(0, 1fr);
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 24px;
|
||||
border-right: 1px solid #d9dde5;
|
||||
background: #ffffff;
|
||||
padding: 20px 16px;
|
||||
}
|
||||
|
||||
.brand {
|
||||
display: grid;
|
||||
grid-template-columns: 32px minmax(0, 1fr);
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.brand-icon {
|
||||
display: inline-grid;
|
||||
place-items: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 8px;
|
||||
background: #2f7c67;
|
||||
color: #ffffff;
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.brand strong,
|
||||
.brand span {
|
||||
display: block;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.brand strong {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.brand span {
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.nav-list {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.nav-button,
|
||||
.icon-text-button,
|
||||
.icon-button,
|
||||
.task-row {
|
||||
border: 1px solid #d4d9e2;
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
color: #202124;
|
||||
}
|
||||
|
||||
.nav-button,
|
||||
.icon-text-button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
min-height: 38px;
|
||||
padding: 8px 11px;
|
||||
}
|
||||
|
||||
.nav-button {
|
||||
width: 100%;
|
||||
justify-content: flex-start;
|
||||
}
|
||||
|
||||
.nav-button.active {
|
||||
border-color: #2f7c67;
|
||||
background: #e7f4ef;
|
||||
color: #1f5f4e;
|
||||
}
|
||||
|
||||
.workspace {
|
||||
min-width: 0;
|
||||
padding: 22px;
|
||||
}
|
||||
|
||||
.topbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
|
||||
.topbar h1 {
|
||||
margin: 0;
|
||||
font-size: 24px;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.topbar p {
|
||||
margin: 4px 0 0;
|
||||
color: #667085;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.view-grid,
|
||||
.config-layout {
|
||||
display: grid;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.metric,
|
||||
.panel {
|
||||
border: 1px solid #d9dde5;
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.metric {
|
||||
padding: 14px;
|
||||
}
|
||||
|
||||
.metric-label {
|
||||
display: block;
|
||||
margin-bottom: 8px;
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.metric strong {
|
||||
font-size: 26px;
|
||||
}
|
||||
|
||||
.panel {
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.section-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.section-title h2 {
|
||||
margin: 0;
|
||||
font-size: 16px;
|
||||
line-height: 1.3;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
display: grid;
|
||||
place-items: center;
|
||||
gap: 10px;
|
||||
min-height: 170px;
|
||||
border: 1px dashed #c8ced8;
|
||||
border-radius: 8px;
|
||||
color: #667085;
|
||||
text-align: center;
|
||||
padding: 22px;
|
||||
}
|
||||
|
||||
.empty-state.compact {
|
||||
min-height: 88px;
|
||||
}
|
||||
|
||||
.device-list {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.device-row {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
min-height: 58px;
|
||||
border: 1px solid #e2e6ec;
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px;
|
||||
}
|
||||
|
||||
.device-row strong,
|
||||
.device-row span,
|
||||
.task-goal,
|
||||
.task-meta {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.device-row span {
|
||||
display: block;
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.row-meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.driver-label,
|
||||
.status-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 24px;
|
||||
border-radius: 999px;
|
||||
padding: 3px 9px;
|
||||
font-size: 12px;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.driver-label {
|
||||
background: #eef0f4;
|
||||
color: #444b56;
|
||||
}
|
||||
|
||||
.status-pill.idle,
|
||||
.status-pill.completed {
|
||||
background: #e5f4ec;
|
||||
color: #1f6b4a;
|
||||
}
|
||||
|
||||
.status-pill.busy,
|
||||
.status-pill.running {
|
||||
background: #e8f1fb;
|
||||
color: #275b8d;
|
||||
}
|
||||
|
||||
.status-pill.created,
|
||||
.status-pill.cancelled {
|
||||
background: #f0edf8;
|
||||
color: #67508f;
|
||||
}
|
||||
|
||||
.status-pill.offline,
|
||||
.status-pill.failed,
|
||||
.status-pill.error {
|
||||
background: #fdebea;
|
||||
color: #a43c37;
|
||||
}
|
||||
|
||||
.filters,
|
||||
.form-grid,
|
||||
.settings-form,
|
||||
.detail-grid {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.filters {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
label {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
color: #475467;
|
||||
font-size: 12px;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
input,
|
||||
select {
|
||||
width: 100%;
|
||||
min-height: 38px;
|
||||
border: 1px solid #cbd2dc;
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
color: #202124;
|
||||
padding: 8px 10px;
|
||||
}
|
||||
|
||||
.task-browser {
|
||||
max-height: calc(100vh - 96px);
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.task-row {
|
||||
display: grid;
|
||||
width: 100%;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 4px 10px;
|
||||
margin-bottom: 8px;
|
||||
padding: 11px;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.task-row:hover {
|
||||
border-color: #2f7c67;
|
||||
box-shadow: 0 0 0 2px #d9efe8;
|
||||
}
|
||||
|
||||
.task-goal {
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.task-meta {
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.task-row .status-pill {
|
||||
grid-row: 1 / span 2;
|
||||
grid-column: 2;
|
||||
align-self: center;
|
||||
}
|
||||
|
||||
.detail-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
margin: 0 0 14px;
|
||||
}
|
||||
|
||||
.detail-grid div {
|
||||
border: 1px solid #e2e6ec;
|
||||
border-radius: 8px;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.detail-grid dt {
|
||||
margin-bottom: 4px;
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.detail-grid dd {
|
||||
margin: 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.timeline-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.icon-button {
|
||||
display: inline-grid;
|
||||
place-items: center;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.icon-button.danger {
|
||||
color: #a43c37;
|
||||
}
|
||||
|
||||
.timeline-stage {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.evidence-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.evidence-pane {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.evidence-pane h3 {
|
||||
margin: 0 0 6px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.screenshot-frame {
|
||||
display: grid;
|
||||
place-items: center;
|
||||
min-height: 280px;
|
||||
border: 1px solid #d9dde5;
|
||||
border-radius: 8px;
|
||||
background: #111827;
|
||||
color: #e5e7eb;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.screenshot-frame img {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
max-height: 520px;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.step-data {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.step-data h3 {
|
||||
margin: 0 0 6px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.operation-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 8px;
|
||||
margin: 0 0 8px;
|
||||
}
|
||||
|
||||
.operation-grid div {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.operation-grid dt {
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.operation-grid dd {
|
||||
margin: 2px 0 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.ocr-results {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.ocr-results li {
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
border-left: 3px solid #2f7c67;
|
||||
background: #f5faf7;
|
||||
padding: 8px 10px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.ocr-results span,
|
||||
.muted {
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.ui-tree {
|
||||
border: 1px solid #e2e6ec;
|
||||
border-radius: 8px;
|
||||
background: #f9fafb;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.ui-tree summary {
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.ui-tree-nodes {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
margin: 10px 0 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.ui-tree-nodes li {
|
||||
display: grid;
|
||||
grid-template-columns: auto minmax(0, 1fr);
|
||||
gap: 3px 8px;
|
||||
border-left: 3px solid #5f7fb0;
|
||||
background: #f1f5fb;
|
||||
padding: 8px 10px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.ui-tree-nodes code,
|
||||
.ui-tree-nodes li > span {
|
||||
grid-column: 1 / -1;
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.muted {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
pre {
|
||||
max-height: 248px;
|
||||
overflow: auto;
|
||||
margin: 0;
|
||||
border: 1px solid #e2e6ec;
|
||||
border-radius: 8px;
|
||||
background: #f9fafb;
|
||||
padding: 10px;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.form-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
align-items: end;
|
||||
}
|
||||
|
||||
.submit-button {
|
||||
align-self: end;
|
||||
}
|
||||
|
||||
.managed {
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.settings-form {
|
||||
grid-template-columns: minmax(140px, 240px) auto;
|
||||
align-items: end;
|
||||
justify-content: start;
|
||||
}
|
||||
|
||||
.alert {
|
||||
margin: 12px 0 0;
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.alert.error {
|
||||
background: #fdebea;
|
||||
color: #a43c37;
|
||||
}
|
||||
|
||||
.alert.success {
|
||||
background: #e5f4ec;
|
||||
color: #1f6b4a;
|
||||
}
|
||||
|
||||
@media (max-width: 980px) {
|
||||
.app-shell {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.nav-list {
|
||||
grid-auto-flow: column;
|
||||
}
|
||||
|
||||
.timeline-stage {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.evidence-grid,
|
||||
.operation-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.detail-grid,
|
||||
.form-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
// Narrow server-rendered live-status enhancement. Fetches an HTML fragment
|
||||
// from the same origin and replaces only the dashboard live region. Does
|
||||
// not parse JSON, rebuild DOM from client state, or depend on any framework.
|
||||
(function () {
|
||||
"use strict";
|
||||
var region = document.getElementById("live-status");
|
||||
if (!region) {
|
||||
return;
|
||||
}
|
||||
setInterval(function () {
|
||||
fetch("/ui/_status_fragment", { headers: { Accept: "text/html" } })
|
||||
.then(function (response) {
|
||||
if (!response.ok) {
|
||||
throw new Error("fragment refresh failed");
|
||||
}
|
||||
return response.text();
|
||||
})
|
||||
.then(function (html) {
|
||||
region.innerHTML = html;
|
||||
})
|
||||
.catch(function () {
|
||||
// Ignore transient network errors; the next tick will retry.
|
||||
});
|
||||
}, 10000);
|
||||
})();
|
||||
@@ -1,40 +0,0 @@
|
||||
<div class="metrics">
|
||||
<div class="metric">
|
||||
<span class="metric-label">Devices</span>
|
||||
<strong>{{ device_count }}</strong>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<span class="metric-label">Running</span>
|
||||
<strong>{{ running_tasks }}</strong>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<span class="metric-label">Failed</span>
|
||||
<strong>{{ failed_tasks }}</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section class="panel">
|
||||
<div class="section-title">
|
||||
<h2>Device Status</h2>
|
||||
</div>
|
||||
{% if not devices %}
|
||||
<div class="empty-state">
|
||||
<span>No devices registered. Add one from Config.</span>
|
||||
</div>
|
||||
{% else %}
|
||||
<ul class="device-list">
|
||||
{% for device in devices %}
|
||||
<li class="device-row">
|
||||
<div>
|
||||
<strong>{{ device.name or device.id }}</strong>
|
||||
<span>{{ device.id }}</span>
|
||||
</div>
|
||||
<div class="row-meta">
|
||||
<span class="driver-label">{{ device.driver_type }}</span>
|
||||
<span class="status-pill {{ device.status }}">{{ device.status }}</span>
|
||||
</div>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endif %}
|
||||
</section>
|
||||
@@ -1,46 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>{% block title %}Apex Console{% endblock %}</title>
|
||||
<link rel="stylesheet" href="{{ url_for('runtime_console_assets', path='console.css') }}" />
|
||||
{% block head %}{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
<div class="app-shell">
|
||||
<aside class="sidebar" aria-label="Console navigation">
|
||||
<div class="brand">
|
||||
<span class="brand-icon" aria-hidden="true">AM</span>
|
||||
<div>
|
||||
<strong>Apex Console</strong>
|
||||
<span>Runtime /ui</span>
|
||||
</div>
|
||||
</div>
|
||||
<nav class="nav-list">
|
||||
<a
|
||||
class="nav-button{% if active_section == 'dashboard' %} active{% endif %}"
|
||||
href="{{ url_for('runtime_console_dashboard') }}"
|
||||
>
|
||||
<span>Devices</span>
|
||||
</a>
|
||||
<a
|
||||
class="nav-button{% if active_section == 'tasks' %} active{% endif %}"
|
||||
href="{{ url_for('runtime_console_tasks') }}"
|
||||
>
|
||||
<span>Tasks</span>
|
||||
</a>
|
||||
<a
|
||||
class="nav-button{% if active_section == 'config' %} active{% endif %}"
|
||||
href="{{ url_for('runtime_console_config') }}"
|
||||
>
|
||||
<span>Config</span>
|
||||
</a>
|
||||
</nav>
|
||||
</aside>
|
||||
<main class="workspace">
|
||||
{% block body %}{% endblock %}
|
||||
</main>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,98 +0,0 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Config · Apex Console{% endblock %}
|
||||
{% block body %}
|
||||
<header class="topbar">
|
||||
<div>
|
||||
<h1>Config</h1>
|
||||
<p>Register devices and tune Runtime parameters.</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="config-layout">
|
||||
<section class="panel">
|
||||
<div class="section-title">
|
||||
<h2>Device Configuration</h2>
|
||||
</div>
|
||||
{% if device_error %}
|
||||
<p class="alert error">{{ device_error }}</p>
|
||||
{% endif %}
|
||||
{% if device_added %}
|
||||
<p class="alert success">Device "{{ device_added }}" registered.</p>
|
||||
{% endif %}
|
||||
<form class="form-grid" method="post" action="{{ url_for('runtime_console_register_device') }}">
|
||||
<label>
|
||||
Name
|
||||
<input type="text" name="name" value="{{ form_values.name }}" placeholder="Desk iPhone" />
|
||||
</label>
|
||||
<label>
|
||||
Driver
|
||||
<select name="driver_type">
|
||||
{% for driver_type in supported_driver_types %}
|
||||
<option
|
||||
value="{{ driver_type }}"
|
||||
{% if driver_type == form_values.driver_type %}selected{% endif %}
|
||||
>{{ driver_type }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Server URL
|
||||
<input type="url" name="server_url" value="{{ form_values.server_url }}" />
|
||||
</label>
|
||||
<label>
|
||||
UDID
|
||||
<input type="text" name="udid" value="{{ form_values.udid }}" />
|
||||
</label>
|
||||
<label>
|
||||
WDA local port
|
||||
<input
|
||||
type="number"
|
||||
min="1"
|
||||
name="wda_local_port"
|
||||
value="{{ form_values.wda_local_port }}"
|
||||
/>
|
||||
</label>
|
||||
<button class="icon-text-button submit-button" type="submit">
|
||||
<span>Add Device</span>
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<ul class="device-list managed">
|
||||
{% for device in devices %}
|
||||
<li class="device-row">
|
||||
<div>
|
||||
<strong>{{ device.name or device.id }}</strong>
|
||||
<span>{{ device.id }}</span>
|
||||
</div>
|
||||
<form method="post" action="{{ url_for('runtime_console_remove_device', device_id=device.id) }}">
|
||||
<button class="icon-button danger" type="submit" title="Remove device" aria-label="Remove device">
|
||||
<span>×</span>
|
||||
</button>
|
||||
</form>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<div class="section-title">
|
||||
<h2>Runtime Parameters</h2>
|
||||
</div>
|
||||
{% if config_error %}
|
||||
<p class="alert error">{{ config_error }}</p>
|
||||
{% endif %}
|
||||
{% if config_saved %}
|
||||
<p class="alert success">Saved.</p>
|
||||
{% endif %}
|
||||
<form class="settings-form" method="post" action="{{ url_for('runtime_console_update_max_steps') }}">
|
||||
<label>
|
||||
Max steps
|
||||
<input type="number" min="1" name="max_steps" value="{{ max_steps }}" />
|
||||
</label>
|
||||
<button class="icon-text-button" type="submit">
|
||||
<span>Save</span>
|
||||
</button>
|
||||
</form>
|
||||
</section>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -1,17 +0,0 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Devices · Apex Console{% endblock %}
|
||||
{% block body %}
|
||||
<header class="topbar">
|
||||
<div>
|
||||
<h1>Devices</h1>
|
||||
<p>{{ device_count }} device(s) / {{ task_count }} task(s)</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="view-grid">
|
||||
<div id="live-status">
|
||||
{% include "_status_fragment.html" %}
|
||||
</div>
|
||||
</section>
|
||||
<script src="{{ url_for('runtime_console_assets', path='dashboard.js') }}"></script>
|
||||
{% endblock %}
|
||||
@@ -1,156 +0,0 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Task {{ task.id }} · Apex Console{% endblock %}
|
||||
{% block body %}
|
||||
<header class="topbar">
|
||||
<div>
|
||||
<h1>Task Detail</h1>
|
||||
<p><a href="{{ url_for('runtime_console_tasks') }}">← Back to tasks</a></p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="panel timeline-panel">
|
||||
<div class="section-title">
|
||||
<h2>{{ task.goal }}</h2>
|
||||
<span class="status-pill {{ task.status }}">{{ task.status }}</span>
|
||||
</div>
|
||||
|
||||
<dl class="detail-grid">
|
||||
<div>
|
||||
<dt>Task ID</dt>
|
||||
<dd>{{ task.id }}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Device</dt>
|
||||
<dd>{{ device_name }}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Created</dt>
|
||||
<dd>{{ task.created_at or "-" }}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Updated</dt>
|
||||
<dd>{{ task.updated_at or "-" }}</dd>
|
||||
</div>
|
||||
{% if task.failure_reason %}
|
||||
<div>
|
||||
<dt>Failure</dt>
|
||||
<dd>{{ task.failure_reason }}</dd>
|
||||
</div>
|
||||
{% endif %}
|
||||
</dl>
|
||||
|
||||
<div class="timeline-controls">
|
||||
<span>Step {{ current_step_index }} of {{ timeline|length }}</span>
|
||||
</div>
|
||||
|
||||
{% if not timeline %}
|
||||
<div class="empty-state compact">
|
||||
<span>No timeline records captured.</span>
|
||||
</div>
|
||||
{% else %}
|
||||
<form class="timeline-controls" method="get" action="{{ url_for('runtime_console_task_detail', task_id=task.id) }}">
|
||||
<label>
|
||||
Step
|
||||
<select name="step" onchange="this.form.submit()">
|
||||
{% for record in timeline %}
|
||||
<option
|
||||
value="{{ loop.index0 }}"
|
||||
{% if loop.index0 == current_step_index %}selected{% endif %}
|
||||
>Step {{ loop.index }}{% if record.tool_call and record.tool_call.get('action') %} ({{ record.tool_call.get('action') }}){% endif %}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<noscript><button class="icon-text-button" type="submit"><span>Show</span></button></noscript>
|
||||
</form>
|
||||
|
||||
<div class="timeline-stage">
|
||||
<div class="evidence-grid">
|
||||
<section class="evidence-pane">
|
||||
<h3>Before action</h3>
|
||||
<div class="screenshot-frame">
|
||||
{% if current_step.before_image_base64 %}
|
||||
<img
|
||||
src="data:image/png;base64,{{ current_step.before_image_base64 }}"
|
||||
alt="Screenshot captured before the action"
|
||||
/>
|
||||
{% else %}
|
||||
<span>No screenshot</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
<section class="evidence-pane">
|
||||
<h3>After action</h3>
|
||||
<div class="screenshot-frame">
|
||||
{% if current_step.after_image_base64 %}
|
||||
<img
|
||||
src="data:image/png;base64,{{ current_step.after_image_base64 }}"
|
||||
alt="Screenshot captured after the action"
|
||||
/>
|
||||
{% else %}
|
||||
<span>No screenshot</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<div class="step-data">
|
||||
<div>
|
||||
<h3>Operation</h3>
|
||||
<dl class="operation-grid">
|
||||
<div>
|
||||
<dt>Action</dt>
|
||||
<dd>{{ current_step.tool_call.get('action', '-') }}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Description</dt>
|
||||
<dd>{{ current_step.tool_call.get('description', '-') }}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<pre>{{ current_step.tool_call | tojson(indent=2) }}</pre>
|
||||
</div>
|
||||
<div>
|
||||
<h3>Execution result</h3>
|
||||
<pre>{{ current_step.result | tojson(indent=2) }}</pre>
|
||||
</div>
|
||||
<div>
|
||||
<h3>OCR results</h3>
|
||||
{% if ocr_results %}
|
||||
<ul class="ocr-results">
|
||||
{% for ocr in ocr_results %}
|
||||
<li>
|
||||
<strong>{{ ocr.get('text') or '-' }}</strong>
|
||||
<span>{{ ocr.get('bounds') | tojson }}</span>
|
||||
{% if ocr.get('confidence') is not none %}
|
||||
<span>confidence {{ '%.3f' | format(ocr.get('confidence')) }}</span>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% else %}
|
||||
<p class="muted">No OCR output captured for this step.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if ui_tree_nodes %}
|
||||
<div>
|
||||
<h3>UI tree</h3>
|
||||
<details class="ui-tree" open>
|
||||
<summary>{{ ui_tree_nodes|length }} normalized nodes</summary>
|
||||
<ul class="ui-tree-nodes">
|
||||
{% for node in ui_tree_nodes %}
|
||||
<li>
|
||||
<strong>{{ node.get('type') or 'unknown' }}</strong>
|
||||
<span>{{ node.get('text') or node.get('id') or '-' }}</span>
|
||||
<code>{{ node.get('bounds') | tojson }}</code>
|
||||
{% if node.get('confidence') is not none %}
|
||||
<span>confidence {{ '%.3f' | format(node.get('confidence')) }}</span>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</details>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -1,62 +0,0 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Tasks · Apex Console{% endblock %}
|
||||
{% block body %}
|
||||
<header class="topbar">
|
||||
<div>
|
||||
<h1>Tasks</h1>
|
||||
<p>{{ tasks|length }} task(s) match the current filters</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="panel task-browser">
|
||||
<div class="section-title">
|
||||
<h2>Task List</h2>
|
||||
</div>
|
||||
<form class="filters" method="get" action="{{ url_for('runtime_console_tasks') }}">
|
||||
<label>
|
||||
Device
|
||||
<select name="device_id">
|
||||
<option value="">All devices</option>
|
||||
{% for device in devices %}
|
||||
<option
|
||||
value="{{ device.id }}"
|
||||
{% if device.id == selected_device_id %}selected{% endif %}
|
||||
>{{ device.name or device.id }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Status
|
||||
<select name="status">
|
||||
<option value="">All statuses</option>
|
||||
{% for status_name in task_statuses %}
|
||||
<option
|
||||
value="{{ status_name }}"
|
||||
{% if status_name == selected_status %}selected{% endif %}
|
||||
>{{ status_name }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<noscript><button class="icon-text-button" type="submit"><span>Apply</span></button></noscript>
|
||||
</form>
|
||||
|
||||
{% if not tasks %}
|
||||
<div class="empty-state compact">
|
||||
<span>No tasks match the current filters.</span>
|
||||
</div>
|
||||
{% else %}
|
||||
{% for task in tasks %}
|
||||
<a
|
||||
class="task-row"
|
||||
href="{{ url_for('runtime_console_task_detail', task_id=task.id) }}"
|
||||
>
|
||||
<span class="task-goal">{{ task.goal }}</span>
|
||||
<span class="task-meta">
|
||||
{{ device_names.get(task.device_id, task.device_id) }}
|
||||
</span>
|
||||
<span class="status-pill {{ task.status }}">{{ task.status }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -56,6 +56,12 @@ class AssignmentExecutor:
|
||||
should_stop: Callable[[], bool] | None,
|
||||
) -> AssignmentExecutionResult:
|
||||
task = Task(goal=assignment.goal or "", device_id=assignment.device_id)
|
||||
if self.factories.metadata_store is not None:
|
||||
self.factories.metadata_store.create_task(
|
||||
task,
|
||||
source_task_id=assignment.task_id,
|
||||
source_attempt=assignment.attempt,
|
||||
)
|
||||
runner = self.factories.task_runner_factory()
|
||||
runner.on_step_progress = self._progress.update
|
||||
if should_stop is None:
|
||||
|
||||
@@ -13,6 +13,11 @@ class HostAgentConfigurationError(ValueError):
|
||||
|
||||
_LOOPBACK_BIND_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"})
|
||||
_AI_PLANNER_TRANSPORTS = frozenset({"direct", "cloud"})
|
||||
_REMOVED_RUNTIME_SUPERVISION_SETTINGS = (
|
||||
"HOST_AGENT_RUNTIME_SUPERVISED",
|
||||
"HOST_AGENT_RUNTIME_HOST",
|
||||
"HOST_AGENT_RUNTIME_PORT",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -39,9 +44,6 @@ class HostAgentConfig:
|
||||
appium_supervised: bool = False
|
||||
appium_host: str = "127.0.0.1"
|
||||
appium_port: int = 4723
|
||||
runtime_supervised: bool = False
|
||||
runtime_host: str = "127.0.0.1"
|
||||
runtime_port: int = 8000
|
||||
dependency_restart_max_attempts: int = 5
|
||||
task_progress_db_path: Path = Path("host_agent_data/task_progress.sqlite3")
|
||||
task_artifact_dir: Path = Path("host_agent_data/history")
|
||||
@@ -54,6 +56,7 @@ def load_host_agent_config(
|
||||
env: Mapping[str, str] | None = None,
|
||||
) -> HostAgentConfig:
|
||||
values = os.environ if env is None else env
|
||||
_reject_removed_runtime_supervision_settings(values)
|
||||
control_plane_url = (
|
||||
values.get(
|
||||
"HOST_AGENT_CONTROL_PLANE_URL",
|
||||
@@ -134,9 +137,6 @@ def load_host_agent_config(
|
||||
appium_supervised=_truthy(values, "HOST_AGENT_APPIUM_SUPERVISED", False),
|
||||
appium_host=values.get("HOST_AGENT_APPIUM_HOST", "127.0.0.1").strip(),
|
||||
appium_port=_positive_int(values, "HOST_AGENT_APPIUM_PORT", 4723),
|
||||
runtime_supervised=_truthy(values, "HOST_AGENT_RUNTIME_SUPERVISED", False),
|
||||
runtime_host=values.get("HOST_AGENT_RUNTIME_HOST", "127.0.0.1").strip(),
|
||||
runtime_port=_positive_int(values, "HOST_AGENT_RUNTIME_PORT", 8000),
|
||||
dependency_restart_max_attempts=_positive_int(
|
||||
values, "HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS", 5
|
||||
),
|
||||
@@ -177,6 +177,20 @@ def load_host_agent_config(
|
||||
return config
|
||||
|
||||
|
||||
def _reject_removed_runtime_supervision_settings(values: Mapping[str, str]) -> None:
|
||||
configured = [
|
||||
setting
|
||||
for setting in _REMOVED_RUNTIME_SUPERVISION_SETTINGS
|
||||
if setting in values
|
||||
]
|
||||
if configured:
|
||||
raise HostAgentConfigurationError(
|
||||
f"{', '.join(configured)} has been removed with the standalone "
|
||||
"Runtime service. Use the Host Agent console for task evidence "
|
||||
"and HOST_AGENT_APPIUM_SUPERVISED for optional Appium supervision."
|
||||
)
|
||||
|
||||
|
||||
def _parse_ai_planner_transport(value: str | None) -> str:
|
||||
if value is None:
|
||||
return "cloud"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Optional supervisor for the two local external processes the Host Agent
|
||||
depends on for the macOS single-machine real-device workflow: the Appium
|
||||
server (which gates real ``Driver.connect()``) and the local Runtime API
|
||||
(used for local inspection).
|
||||
"""Optional supervisor for Appium in the macOS single-machine real-device
|
||||
workflow. Appium gates real driver connections; task inspection is provided by
|
||||
the Host Agent's own console.
|
||||
|
||||
Lives in ``host_agent`` because it spawns and monitors host-level processes
|
||||
alongside the heartbeat/claim loop. Off by default; see ``HostAgentConfig``.
|
||||
@@ -56,18 +55,6 @@ def probe_appium(host: str, port: int) -> ProbeResult:
|
||||
return _probe_http(host, port, path="/status")
|
||||
|
||||
|
||||
def probe_runtime(host: str, port: int) -> ProbeResult:
|
||||
"""Probe the local Runtime API at ``host:port``. Healthy iff ``GET /devices``
|
||||
returns 200 with a JSON body.
|
||||
|
||||
``api.rest.create_app`` does not expose a dedicated ``/health`` endpoint;
|
||||
``/devices`` is the stable read-only GET that proves the FastAPI app is
|
||||
mounted and the device manager is reachable. Per design.md Decision 2 this
|
||||
is the "equivalent existing endpoint" used for the readiness check.
|
||||
"""
|
||||
return _probe_http(host, port, path="/devices")
|
||||
|
||||
|
||||
def _probe_http(host: str, port: int, *, path: str) -> ProbeResult:
|
||||
# Step 1: plain TCP connect — distinguish "nothing listening" (→ spawn)
|
||||
# from "something is there but wrong" (→ port conflict, skip).
|
||||
@@ -98,18 +85,6 @@ def appium_argv_factory(host: str, port: int) -> list[str]:
|
||||
return ["appium", "--address", host, "--port", str(port)]
|
||||
|
||||
|
||||
def runtime_argv_factory(host: str, port: int) -> list[str]:
|
||||
return [
|
||||
"uvicorn",
|
||||
"api.rest:create_app",
|
||||
"--factory",
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SupervisedDependency:
|
||||
"""Config + mutable runtime state for one supervised external process."""
|
||||
@@ -190,16 +165,6 @@ class DependencySupervisor:
|
||||
probe=probe_appium,
|
||||
)
|
||||
)
|
||||
if ha_config.runtime_supervised:
|
||||
deps.append(
|
||||
SupervisedDependency(
|
||||
name="runtime",
|
||||
host=ha_config.runtime_host,
|
||||
port=ha_config.runtime_port,
|
||||
argv_factory=runtime_argv_factory,
|
||||
probe=probe_runtime,
|
||||
)
|
||||
)
|
||||
return cls(
|
||||
deps,
|
||||
max_attempts=ha_config.dependency_restart_max_attempts,
|
||||
|
||||
@@ -25,6 +25,7 @@ class ExecutionFactories:
|
||||
task_runner_factory: Callable[[], TaskRunner]
|
||||
workflow_runner_factory: Callable[[], WorkflowRunner]
|
||||
workflow_store: WorkflowStore
|
||||
metadata_store: TaskMetadataStore | None = None
|
||||
|
||||
|
||||
def create_execution_factories(
|
||||
@@ -61,6 +62,7 @@ def create_execution_factories(
|
||||
task_runner_factory=create_task_runner,
|
||||
workflow_runner_factory=create_workflow_runner,
|
||||
workflow_store=shared_workflow_store,
|
||||
metadata_store=metadata_store,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -70,9 +70,13 @@ def _device_display_status(device: Any, *, busy_device_id: str | None) -> str:
|
||||
return device.status
|
||||
|
||||
|
||||
def _screenshot_data_uri(record: dict[str, Any]) -> str | None:
|
||||
"""Return a ``data:`` URI for the step's screenshot, or ``None``."""
|
||||
screenshot_path = record.get("screenshot_path")
|
||||
def _screenshot_data_uri(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
path_key: str = "screenshot_path",
|
||||
) -> str | None:
|
||||
"""Return a ``data:`` URI for one step screenshot, or ``None``."""
|
||||
screenshot_path = record.get(path_key)
|
||||
if not screenshot_path:
|
||||
return None
|
||||
path = Path(str(screenshot_path))
|
||||
@@ -82,6 +86,49 @@ def _screenshot_data_uri(record: dict[str, Any]) -> str | None:
|
||||
return f"data:image/png;base64,{encoded}"
|
||||
|
||||
|
||||
def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw_results = record.get("ocr_results")
|
||||
if not isinstance(raw_results, list):
|
||||
return []
|
||||
return [result for result in raw_results if isinstance(result, dict)]
|
||||
|
||||
|
||||
def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
tool_call = record.get("tool_call")
|
||||
if not isinstance(tool_call, dict):
|
||||
return []
|
||||
if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}:
|
||||
return []
|
||||
step_result = record.get("result")
|
||||
if not isinstance(step_result, dict):
|
||||
return []
|
||||
raw_nodes = step_result.get("result")
|
||||
if not isinstance(raw_nodes, list):
|
||||
return []
|
||||
return [node for node in raw_nodes if isinstance(node, dict)]
|
||||
|
||||
|
||||
def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]:
|
||||
tool_call = record.get("tool_call")
|
||||
result = record.get("result")
|
||||
return {
|
||||
"index": record.get("index", ""),
|
||||
"timestamp": record.get("timestamp", ""),
|
||||
"prompt": record.get("prompt") or "",
|
||||
"tool_call": tool_call if isinstance(tool_call, dict) else {},
|
||||
"result": result if isinstance(result, dict) else {},
|
||||
"before_screenshot_src": _screenshot_data_uri(
|
||||
record, path_key="before_screenshot_path"
|
||||
),
|
||||
"after_screenshot_src": _screenshot_data_uri(
|
||||
record, path_key="after_screenshot_path"
|
||||
)
|
||||
or _screenshot_data_uri(record),
|
||||
"ocr_results": _ocr_results(record),
|
||||
"ui_tree_nodes": _ui_tree_nodes(record),
|
||||
}
|
||||
|
||||
|
||||
def _safe_submission_error(detail: str) -> str:
|
||||
"""Return a safe, single-line error message for the operator.
|
||||
|
||||
@@ -525,7 +572,7 @@ def create_console_app(
|
||||
if task_id:
|
||||
notice = (
|
||||
f"Task submitted. Cloud task ID: {task_id}. "
|
||||
"Track it from the Cloud console for execution progress."
|
||||
"It will appear below when this Host begins executing it."
|
||||
)
|
||||
elif request.query_params.get("outcome") == "unknown":
|
||||
unknown = True
|
||||
@@ -664,36 +711,22 @@ def create_console_app(
|
||||
if timeline is not None:
|
||||
timeline_records = await asyncio.to_thread(timeline.read, task_id)
|
||||
task_rows = [
|
||||
(key, task[key])
|
||||
for key in (
|
||||
"id",
|
||||
"goal",
|
||||
"device_id",
|
||||
"status",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
(label, task[key])
|
||||
for key, label in (
|
||||
("source_task_id", "Cloud task ID"),
|
||||
("source_attempt", "Cloud attempt"),
|
||||
("id", "Execution ID"),
|
||||
("goal", "Goal"),
|
||||
("device_id", "Device"),
|
||||
("status", "Status"),
|
||||
("created_at", "Created"),
|
||||
("updated_at", "Updated"),
|
||||
("completed_at", "Completed"),
|
||||
("failure_reason", "Failure reason"),
|
||||
)
|
||||
if task.get(key) is not None
|
||||
]
|
||||
timeline_steps = [
|
||||
{
|
||||
"index": record.get("index", ""),
|
||||
"timestamp": record.get("timestamp", ""),
|
||||
"prompt": record.get("prompt") or "",
|
||||
"tool_call_text": (
|
||||
json.dumps(record.get("tool_call"), ensure_ascii=False)
|
||||
if record.get("tool_call")
|
||||
else ""
|
||||
),
|
||||
"result_text": (
|
||||
json.dumps(record.get("result"), ensure_ascii=False)
|
||||
if record.get("result")
|
||||
else ""
|
||||
),
|
||||
"screenshot_src": _screenshot_data_uri(record),
|
||||
}
|
||||
for record in timeline_records
|
||||
]
|
||||
timeline_steps = [_timeline_step_context(record) for record in timeline_records]
|
||||
return _render(
|
||||
"task_detail.html",
|
||||
title=f"Task {task_id}",
|
||||
|
||||
@@ -1,6 +1,33 @@
|
||||
{% extends "base.html" %}
|
||||
{% block styles %}
|
||||
{{ super() }}
|
||||
.task-back { margin-top: 0; }
|
||||
.timeline-step { border: 1px solid #c8d0d6; background: #fff; padding: 1rem; margin-bottom: 1rem; }
|
||||
.step-heading { display: flex; flex-wrap: wrap; gap: 0.5rem 1rem; align-items: baseline; margin-bottom: 0.75rem; }
|
||||
.step-heading p { margin: 0; color: #4d5a63; }
|
||||
.evidence-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; margin-bottom: 1rem; }
|
||||
.evidence-pane { margin: 0; min-width: 0; }
|
||||
.evidence-pane h3 { font-size: 1rem; margin: 0 0 0.35rem; }
|
||||
.screenshot-frame { min-height: 6rem; border: 1px solid #c8d0d6; background: #f8fafb; display: grid; place-items: center; overflow: hidden; color: #5e6b73; }
|
||||
.screenshot-frame img { display: block; width: 100%; height: auto; }
|
||||
.step-details { margin-top: 0.75rem; }
|
||||
.step-details summary { cursor: pointer; font-weight: 600; }
|
||||
.step-details pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 0.65rem 0 0; padding: 0.65rem; border: 1px solid #d5dce0; background: #f8fafb; }
|
||||
.operation-grid { display: grid; grid-template-columns: minmax(7rem, 0.4fr) minmax(0, 1fr); gap: 0.35rem 0.75rem; margin: 0.65rem 0 0; }
|
||||
.operation-grid dt { font-weight: 600; }
|
||||
.operation-grid dd { margin: 0; overflow-wrap: anywhere; }
|
||||
.observation-list, .ui-tree-nodes { margin: 0.65rem 0 0; padding-left: 1.25rem; }
|
||||
.observation-list li, .ui-tree-nodes li { margin-bottom: 0.45rem; overflow-wrap: anywhere; }
|
||||
.observation-list span, .ui-tree-nodes span { color: #4d5a63; margin-left: 0.4rem; }
|
||||
.ui-tree-nodes code { overflow-wrap: anywhere; }
|
||||
@media (max-width: 640px) {
|
||||
.evidence-grid { grid-template-columns: minmax(0, 1fr); }
|
||||
.timeline-step { padding: 0.75rem; }
|
||||
}
|
||||
{% endblock %}
|
||||
{% block body %}
|
||||
<h1>Task {{ task.get("id") or "" }}</h1>
|
||||
<p class="task-back"><a href="/tasks">← Back to executions</a></p>
|
||||
<h1>Execution</h1>
|
||||
<table>
|
||||
<thead><tr><th>Field</th><th>Value</th></tr></thead>
|
||||
<tbody>{% for row in task_rows %}<tr><td>{{ row[0] }}</td><td>{{ row[1] }}</td></tr>{% endfor %}</tbody>
|
||||
@@ -10,13 +37,81 @@
|
||||
<p>No timeline records.</p>
|
||||
{% else %}
|
||||
{% for step in timeline_steps %}
|
||||
<div style="border:1px solid #ccc;background:#fff;padding:0.75rem;margin-bottom:0.75rem;">
|
||||
<p><strong>Step {{ step.index }}</strong> — {{ step.timestamp }}</p>
|
||||
<p>Prompt: {{ step.prompt }}</p>
|
||||
<p>Tool call: <code>{{ step.tool_call_text }}</code></p>
|
||||
<p>Result: <code>{{ step.result_text }}</code></p>
|
||||
{% if step.screenshot_src %}<img src="{{ step.screenshot_src }}" alt="screenshot" style="max-width:100%;border:1px solid #ccc;margin-top:0.5rem;">{% endif %}
|
||||
</div>
|
||||
<section class="timeline-step">
|
||||
<div class="step-heading">
|
||||
<strong>Step {{ step.index }}</strong>
|
||||
<p>{{ step.timestamp }}</p>
|
||||
</div>
|
||||
<div class="evidence-grid">
|
||||
<figure class="evidence-pane">
|
||||
<h3>Before action</h3>
|
||||
<div class="screenshot-frame">
|
||||
{% if step.before_screenshot_src %}
|
||||
<img src="{{ step.before_screenshot_src }}" alt="Screenshot before action">
|
||||
{% else %}
|
||||
<span>No screenshot</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</figure>
|
||||
<figure class="evidence-pane">
|
||||
<h3>After action</h3>
|
||||
<div class="screenshot-frame">
|
||||
{% if step.after_screenshot_src %}
|
||||
<img src="{{ step.after_screenshot_src }}" alt="Screenshot after action">
|
||||
{% else %}
|
||||
<span>No screenshot</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</figure>
|
||||
</div>
|
||||
<details class="step-details" open>
|
||||
<summary>Operation</summary>
|
||||
<dl class="operation-grid">
|
||||
<dt>Action</dt><dd>{{ step.tool_call.get("action") or "-" }}</dd>
|
||||
<dt>Description</dt><dd>{{ step.tool_call.get("description") or "-" }}</dd>
|
||||
</dl>
|
||||
<pre>{{ step.tool_call | tojson(indent=2) }}</pre>
|
||||
</details>
|
||||
<details class="step-details">
|
||||
<summary>Result</summary>
|
||||
<pre>{{ step.result | tojson(indent=2) }}</pre>
|
||||
</details>
|
||||
{% if step.prompt %}
|
||||
<details class="step-details">
|
||||
<summary>Planner prompt</summary>
|
||||
<pre>{{ step.prompt }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% if step.ocr_results %}
|
||||
<details class="step-details" open>
|
||||
<summary>OCR results ({{ step.ocr_results|length }})</summary>
|
||||
<ul class="observation-list">
|
||||
{% for ocr in step.ocr_results %}
|
||||
<li>
|
||||
<strong>{{ ocr.get("text") or "-" }}</strong>
|
||||
<span>{{ ocr.get("bounds") | tojson }}</span>
|
||||
{% if ocr.get("confidence") is not none %}<span>confidence {{ "%.3f" | format(ocr.get("confidence")) }}</span>{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% if step.ui_tree_nodes %}
|
||||
<details class="step-details">
|
||||
<summary>UI tree ({{ step.ui_tree_nodes|length }} normalized nodes)</summary>
|
||||
<ul class="ui-tree-nodes">
|
||||
{% for node in step.ui_tree_nodes %}
|
||||
<li>
|
||||
<strong>{{ node.get("type") or "unknown" }}</strong>
|
||||
<span>{{ node.get("text") or node.get("id") or "-" }}</span>
|
||||
<code>{{ node.get("bounds") | tojson }}</code>
|
||||
{% if node.get("confidence") is not none %}<span>confidence {{ "%.3f" | format(node.get("confidence")) }}</span>{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</details>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
@@ -29,17 +29,19 @@
|
||||
</section>
|
||||
|
||||
<section id="local-tasks">
|
||||
<h2>Local Runtime tasks</h2>
|
||||
<h2>Executed tasks on this Host</h2>
|
||||
{% if metadata_store_missing %}
|
||||
<p class="error">Task metadata store is not configured.</p>
|
||||
{% elif not tasks %}
|
||||
<p>No tasks recorded.</p>
|
||||
<p>No executions recorded yet.</p>
|
||||
{% else %}
|
||||
<table>
|
||||
<thead><tr><th>Task ID</th><th>Status</th><th>Device</th><th>Created</th><th>Updated</th></tr></thead>
|
||||
<thead><tr><th>Execution ID</th><th>Cloud task</th><th>Attempt</th><th>Status</th><th>Device</th><th>Created</th><th>Updated</th></tr></thead>
|
||||
<tbody>{% for task in tasks %}
|
||||
<tr>
|
||||
<td><a href="/tasks/{{ task["id"] }}">{{ task["id"] }}</a></td>
|
||||
<td>{{ task.get("source_task_id") or "" }}</td>
|
||||
<td>{{ task.get("source_attempt") if task.get("source_attempt") is not none else "" }}</td>
|
||||
<td>{{ task.get("status") or "" }}</td>
|
||||
<td>{{ task.get("device_id") or "" }}</td>
|
||||
<td>{{ task.get("created_at") or "" }}</td>
|
||||
@@ -49,4 +51,4 @@
|
||||
</table>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
|
||||
@@ -25,6 +25,7 @@ def _assignment(**overrides) -> AssignmentModel:
|
||||
|
||||
def test_goal_assignment_executes_through_task_runner() -> None:
|
||||
received: list[Task] = []
|
||||
created: list[tuple[str, str | None, int | None]] = []
|
||||
|
||||
class FakeTaskRunner:
|
||||
def run(self, task: Task) -> Task:
|
||||
@@ -32,10 +33,21 @@ def test_goal_assignment_executes_through_task_runner() -> None:
|
||||
task.status = "completed"
|
||||
return task
|
||||
|
||||
class FakeMetadataStore:
|
||||
def create_task(
|
||||
self,
|
||||
task: Task,
|
||||
*,
|
||||
source_task_id: str | None = None,
|
||||
source_attempt: int | None = None,
|
||||
) -> None:
|
||||
created.append((task.id, source_task_id, source_attempt))
|
||||
|
||||
factories = ExecutionFactories(
|
||||
task_runner_factory=lambda: FakeTaskRunner(), # type: ignore[arg-type,return-value]
|
||||
workflow_runner_factory=lambda: object(), # type: ignore[arg-type,return-value]
|
||||
workflow_store=object(), # type: ignore[arg-type]
|
||||
metadata_store=FakeMetadataStore(), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
result = AssignmentExecutor(factories).execute(_assignment())
|
||||
@@ -44,6 +56,7 @@ def test_goal_assignment_executes_through_task_runner() -> None:
|
||||
assert received[0].goal == "open settings"
|
||||
assert received[0].device_id == "device-a"
|
||||
assert result.metadata["runtime_task_id"] == received[0].id
|
||||
assert created == [(received[0].id, "cloud-task", 1)]
|
||||
|
||||
|
||||
def test_goal_assignment_preserves_runtime_failure_reason() -> None:
|
||||
|
||||
@@ -191,9 +191,6 @@ def test_dependency_supervisor_defaults_to_disabled() -> None:
|
||||
assert config.appium_supervised is False
|
||||
assert config.appium_host == "127.0.0.1"
|
||||
assert config.appium_port == 4723
|
||||
assert config.runtime_supervised is False
|
||||
assert config.runtime_host == "127.0.0.1"
|
||||
assert config.runtime_port == 8000
|
||||
assert config.dependency_restart_max_attempts == 5
|
||||
|
||||
|
||||
@@ -204,9 +201,6 @@ def test_dependency_supervisor_env_vars_parse_bool_and_numeric_fields() -> None:
|
||||
"HOST_AGENT_APPIUM_SUPERVISED": "1",
|
||||
"HOST_AGENT_APPIUM_HOST": "0.0.0.0",
|
||||
"HOST_AGENT_APPIUM_PORT": "4724",
|
||||
"HOST_AGENT_RUNTIME_SUPERVISED": "true",
|
||||
"HOST_AGENT_RUNTIME_HOST": "localhost",
|
||||
"HOST_AGENT_RUNTIME_PORT": "8001",
|
||||
"HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS": "8",
|
||||
}
|
||||
)
|
||||
@@ -215,9 +209,6 @@ def test_dependency_supervisor_env_vars_parse_bool_and_numeric_fields() -> None:
|
||||
assert config.appium_supervised is True
|
||||
assert config.appium_host == "0.0.0.0"
|
||||
assert config.appium_port == 4724
|
||||
assert config.runtime_supervised is True
|
||||
assert config.runtime_host == "localhost"
|
||||
assert config.runtime_port == 8001
|
||||
assert config.dependency_restart_max_attempts == 8
|
||||
|
||||
|
||||
@@ -225,7 +216,6 @@ def test_dependency_supervisor_env_vars_parse_bool_and_numeric_fields() -> None:
|
||||
"overrides",
|
||||
[
|
||||
{"HOST_AGENT_APPIUM_PORT": "0"},
|
||||
{"HOST_AGENT_RUNTIME_PORT": "not-a-number"},
|
||||
{"HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS": "-1"},
|
||||
],
|
||||
)
|
||||
@@ -234,3 +224,19 @@ def test_dependency_supervisor_numeric_fields_reject_invalid_values(
|
||||
) -> None:
|
||||
with pytest.raises(HostAgentConfigurationError):
|
||||
load_host_agent_config(overrides)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"setting,value",
|
||||
[
|
||||
("HOST_AGENT_RUNTIME_SUPERVISED", "true"),
|
||||
("HOST_AGENT_RUNTIME_HOST", "127.0.0.1"),
|
||||
("HOST_AGENT_RUNTIME_PORT", "8000"),
|
||||
],
|
||||
)
|
||||
def test_removed_runtime_supervision_settings_are_rejected(
|
||||
setting: str,
|
||||
value: str,
|
||||
) -> None:
|
||||
with pytest.raises(HostAgentConfigurationError, match="standalone Runtime service"):
|
||||
load_host_agent_config({setting: value})
|
||||
|
||||
@@ -16,8 +16,6 @@ from host_agent.dependency_supervisor import (
|
||||
_SupervisorKnobs,
|
||||
appium_argv_factory,
|
||||
probe_appium,
|
||||
probe_runtime,
|
||||
runtime_argv_factory,
|
||||
)
|
||||
from host_agent.config import HostAgentConfig
|
||||
|
||||
@@ -175,7 +173,6 @@ def test_probe_returns_no_listener_when_port_is_closed(monkeypatch) -> None:
|
||||
_raise_connection_refused,
|
||||
)
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.NO_LISTENER
|
||||
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.NO_LISTENER
|
||||
|
||||
|
||||
def test_probe_returns_healthy_on_appium_status_endpoint(monkeypatch) -> None:
|
||||
@@ -190,18 +187,6 @@ def test_probe_returns_healthy_on_appium_status_endpoint(monkeypatch) -> None:
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.HEALTHY
|
||||
|
||||
|
||||
def test_probe_returns_healthy_on_runtime_devices_endpoint(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_ok_connection,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.httpx.get",
|
||||
lambda url, timeout=2.0: httpx.Response(200, json=[]),
|
||||
)
|
||||
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.HEALTHY
|
||||
|
||||
|
||||
def test_probe_returns_unhealthy_when_listener_returns_non_200(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
@@ -223,7 +208,7 @@ def test_probe_returns_unhealthy_when_listener_returns_non_json(monkeypatch) ->
|
||||
"host_agent.dependency_supervisor.httpx.get",
|
||||
lambda url, timeout=2.0: httpx.Response(200, text="not json"),
|
||||
)
|
||||
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.UNHEALTHY_LISTENER
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
|
||||
|
||||
|
||||
def test_probe_returns_unhealthy_on_http_transport_error(monkeypatch) -> None:
|
||||
@@ -335,38 +320,6 @@ def test_spawn_uses_appium_argv_factory() -> None:
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_spawn_uses_runtime_argv_factory() -> None:
|
||||
async def scenario() -> None:
|
||||
captured: list[list[str]] = []
|
||||
|
||||
def recording_popen(argv, **kwargs):
|
||||
captured.append(list(argv))
|
||||
return _FakePopen(argv)
|
||||
|
||||
dep, _ = _dep(
|
||||
name="runtime",
|
||||
port=8000,
|
||||
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
|
||||
argv_factory=runtime_argv_factory,
|
||||
)
|
||||
sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
|
||||
await sup.start()
|
||||
|
||||
assert captured == [
|
||||
[
|
||||
"uvicorn",
|
||||
"api.rest:create_app",
|
||||
"--factory",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"8000",
|
||||
]
|
||||
]
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_readiness_timeout_leaves_process_running_without_restart_loop() -> None:
|
||||
async def scenario() -> None:
|
||||
# NO_LISTENER for initial probe; probe never becomes HEALTHY → startup
|
||||
@@ -529,7 +482,7 @@ def test_from_host_agent_config_builds_empty_supervisor_when_no_dep_selected() -
|
||||
assert sup.dependencies == []
|
||||
|
||||
|
||||
def test_from_host_agent_config_includes_appium_and_runtime_when_selected() -> None:
|
||||
def test_from_host_agent_config_includes_appium_when_selected() -> None:
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
@@ -538,17 +491,12 @@ def test_from_host_agent_config_includes_appium_and_runtime_when_selected() -> N
|
||||
appium_supervised=True,
|
||||
appium_host="10.0.0.5",
|
||||
appium_port=4724,
|
||||
runtime_supervised=True,
|
||||
runtime_host="10.0.0.5",
|
||||
runtime_port=8001,
|
||||
dependency_restart_max_attempts=7,
|
||||
)
|
||||
sup = DependencySupervisor.from_host_agent_config(config)
|
||||
names = [dep.name for dep in sup.dependencies]
|
||||
assert names == ["appium", "runtime"]
|
||||
assert names == ["appium"]
|
||||
appium = sup.dependencies[0]
|
||||
assert appium.host == "10.0.0.5"
|
||||
assert appium.port == 4724
|
||||
runtime = sup.dependencies[1]
|
||||
assert runtime.port == 8001
|
||||
assert sup._max_attempts == 7
|
||||
|
||||
+20
-17
@@ -37,14 +37,14 @@ uv run --package device-host-agent device-host-agent
|
||||
```
|
||||
|
||||
At startup, the Host Agent loads device registrations from
|
||||
`tasks/device_config.sqlite3`, the same `DeviceConfigStore` used by the local
|
||||
Runtime console API. Register or update devices before starting the Host Agent,
|
||||
then restart it to reload changes. In Compose,
|
||||
`tasks/device_config.sqlite3`. Register or update devices before starting the
|
||||
Host Agent, then restart it to reload changes. In Compose,
|
||||
`HOST_AGENT_TASKS_PATH` selects the host directory mounted at `/app/tasks`; it
|
||||
defaults to `./tasks`.
|
||||
|
||||
The Host Agent only initiates outbound HTTP requests. It does not expose an
|
||||
inbound port.
|
||||
The Host Agent only initiates outbound requests to the Cloud Control Plane. It
|
||||
does expose an authenticated local console on loopback by default; this is not
|
||||
a Cloud-facing inbound API.
|
||||
|
||||
## Direct Edge Enrollment
|
||||
|
||||
@@ -96,9 +96,9 @@ image, repository, log, or general backup.
|
||||
The Host Agent always serves a small local-only web console on the
|
||||
edge machine: heartbeat/enrollment status, registered local devices, current
|
||||
assignment progress, local device add/edit/remove, a local account password
|
||||
change, and recent assignment/heartbeat history. It authenticates with the
|
||||
same local account created by `device-host-agent setup` above — there is no
|
||||
separate console credential.
|
||||
change, assignment/heartbeat history, and complete execution evidence. It
|
||||
authenticates with the same local account created by `device-host-agent setup`
|
||||
above; there is no separate console credential.
|
||||
|
||||
```text
|
||||
HOST_AGENT_CONSOLE_BIND_HOST=127.0.0.1
|
||||
@@ -123,10 +123,10 @@ HOST_AGENT_CONSOLE_HISTORY_LIMIT=200
|
||||
|
||||
### Task progress storage and retention
|
||||
|
||||
The Host Agent persists step-by-step task execution state (metadata +
|
||||
timeline screenshots) to local SQLite/files on the edge machine. These
|
||||
paths are independent from the Runtime's own `tasks/tasks.sqlite3` and
|
||||
do not collide when both processes run on the same host.
|
||||
The Host Agent persists step-by-step task execution state (metadata, Timeline
|
||||
artifacts, screenshots, OCR, and UI-tree results) to local SQLite/files on the
|
||||
edge machine. Runtime is an in-process library, so there is no second Runtime
|
||||
database or service to inspect.
|
||||
|
||||
```text
|
||||
HOST_AGENT_TASK_PROGRESS_DB_PATH=host_agent_data/task_progress.sqlite3
|
||||
@@ -147,12 +147,15 @@ HOST_AGENT_TASK_RETENTION_MAX_AGE_DAYS=7
|
||||
|
||||
**Viewing live and historical task progress:**
|
||||
|
||||
- **Host Agent console**: Open `http://127.0.0.1:8765/tasks` for the task
|
||||
list (status, device, timestamps). Click a task ID to see the detail page
|
||||
with full step-by-step timeline and inlined screenshots.
|
||||
- **Host Agent console**: Open `http://127.0.0.1:8765/tasks` for the
|
||||
authoritative list of tasks actually executing on that Host (Cloud task ID,
|
||||
attempt, status, device, timestamps). Click an execution ID for the full
|
||||
step-by-step timeline with before/after screenshots, operation details, OCR,
|
||||
and UI-tree results.
|
||||
- **Cloud console**: The Cloud Console task detail page shows the latest
|
||||
coarse-grained progress badge (step index, status, summary) that the Host
|
||||
Agent piggybacks on each lease renewal.
|
||||
Agent piggybacks on each lease renewal. For Cloud-proxy hosts it also shows
|
||||
retained LLM prompt/decision history, but it does not store screenshots.
|
||||
|
||||
Treat `HOST_AGENT_CONSOLE_ALLOW_NON_LOOPBACK` as an explicit,
|
||||
operator-accepted risk: the console has no built-in TLS and no rate
|
||||
@@ -380,7 +383,7 @@ back to the upstream.
|
||||
|
||||
## Runtime AI Planner
|
||||
|
||||
The Host Agent reuses the local Runtime planner. Unlike the shared Runtime
|
||||
The Host Agent reuses the shared Runtime planner. Unlike the shared Runtime
|
||||
library (whose own default is the deterministic stub planner), the **Host
|
||||
Agent defaults `AI_PLANNER_ENABLED` to on** -- it is the actual device-control
|
||||
path, so goal assignments use a model unless an operator explicitly opts out.
|
||||
|
||||
@@ -48,11 +48,11 @@ owned by `packages/cloud-platform` and may depend on the Runtime through an
|
||||
explicit workspace source; the Runtime distribution must never depend on or
|
||||
package `cloud`.
|
||||
|
||||
All Python members share the committed root `uv.lock`. The Runtime operator
|
||||
console is server-rendered by the `api` layer through Jinja2 templates and
|
||||
static assets packaged with `device-agent-runtime`; there is no separate
|
||||
frontend project or Node build step for the Runtime console. The unrelated
|
||||
`cloud-console/` Vue/Vite application keeps its own independent npm lifecycle.
|
||||
All Python members share the committed root `uv.lock`. Runtime is an
|
||||
in-process execution library; its operator evidence view is the authenticated,
|
||||
server-rendered Host Agent console rather than a separately packaged Runtime
|
||||
service. The unrelated `cloud-console/` Vue/Vite application keeps its own
|
||||
independent npm lifecycle.
|
||||
|
||||
## Change Discipline
|
||||
|
||||
|
||||
+27
-118
@@ -273,77 +273,15 @@ uv run --package device-agent-runtime pytest -m integration tests/test_wda_integ
|
||||
该测试只从环境变量读取 server URL、UDID 和 device name，不传签名 capabilities，
|
||||
因此应在 WDA 已成功签名/安装后运行。
|
||||
|
||||
## 8. 启动可控制真机的 Runtime API
|
||||
## 8. 独立 Runtime API 已撤销
|
||||
|
||||
当前不能只运行 README 中的普通 `uvicorn ... --factory` 命令，因为它只会加载已登记
|
||||
设备，不会调用 `DeviceManager.connect()`。使用下面的启动方式，在同一进程中完成
|
||||
设备注册、WDA 连接和 REST API 启动：
|
||||
Runtime 现在是由 Host Agent 在进程内调用的执行库，不再提供 `api.rest`、
|
||||
端口 `8000`、`/ui/` 或 `/console/*`。不要再启动单独的 Runtime 服务，
|
||||
也不要使用无鉴权的 REST 调用控制设备。
|
||||
|
||||
```bash
|
||||
python - <<'PY'
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
from api.rest import create_app
|
||||
from device.manager import DeviceManager
|
||||
from driver.registry import build_driver_factory
|
||||
|
||||
device_id = "iphone-1"
|
||||
connection_info = {
|
||||
"server_url": "http://127.0.0.1:4723",
|
||||
"device_name": "iPhone",
|
||||
"udid": os.environ["DEVICE_UDID"],
|
||||
"xcodeOrgId": os.environ["APPLE_TEAM_ID"],
|
||||
"xcodeSigningId": "Apple Development",
|
||||
"updatedWDABundleId": os.environ["WDA_BUNDLE_ID"],
|
||||
}
|
||||
|
||||
manager = DeviceManager()
|
||||
app = create_app(manager=manager)
|
||||
manager.register_device(
|
||||
device_id,
|
||||
build_driver_factory("wda", connection_info),
|
||||
name="Local iPhone",
|
||||
driver_type="wda",
|
||||
connection_info=connection_info,
|
||||
)
|
||||
manager.connect(device_id, max_retries=1)
|
||||
|
||||
uvicorn.run(app, host="127.0.0.1", port=8000)
|
||||
PY
|
||||
```
|
||||
|
||||
保持进程运行，在另一个 Terminal 验证：
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8000/devices
|
||||
|
||||
curl -s -X POST http://127.0.0.1:8000/devices/iphone-1/tap \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"x": 100, "y": 200}'
|
||||
|
||||
curl -s -X POST http://127.0.0.1:8000/devices/iphone-1/launch \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"app_id": "com.apple.Preferences"}'
|
||||
```
|
||||
|
||||
点击坐标必须按当前设备屏幕坐标选择。先截图或使用 Appium Inspector 确认坐标，避免
|
||||
误操作。
|
||||
|
||||
Runtime API 自带同源 Web Console，无需额外的前端进程、Node 工具链或
|
||||
`RUNTIME_CONSOLE_STATIC_DIR`。保持 Runtime API 运行，浏览器访问
|
||||
`http://127.0.0.1:8000/`（会自动 307 跳转到 `/ui/`）即可：
|
||||
|
||||
- `/ui/`：设备状态面板，约每 10 秒自动刷新一次；
|
||||
- `/ui/tasks`：任务列表与筛选；
|
||||
- `/ui/tasks/{task_id}`：任务详情与逐步 timeline（含截图）；
|
||||
- `/ui/config`：登记/移除设备、调整 `max_steps`。
|
||||
|
||||
已由上面启动脚本连接的 `iphone-1` 会出现在设备列表中。不要在 Console 中
|
||||
重复登记同一台设备；当前登记操作只写入配置，不会自动 connect。Console 与
|
||||
`/console/*` JSON API 共用同一份 Runtime 状态，两者行为一致。Runtime Console
|
||||
仅假设受信任本地网络访问，不提供鉴权 / CSRF；如需暴露到非受信网络请另行评估。
|
||||
真机连接和任务执行都由下一节的受管 Host Agent 完成。保持 §6 的 Appium 服务
|
||||
可用，然后启动 Host Agent；本机执行记录、截图、OCR 和 UI 树均从其已鉴权的
|
||||
Console 查看。
|
||||
|
||||
## 9. 启动云端受管 Host Agent
|
||||
|
||||
@@ -459,15 +397,16 @@ http://127.0.0.1:8765
|
||||
`DeviceManager` 上生效，无需重启 Host Agent。
|
||||
- 修改密码：更新本地操作账号密码，需要先输入当前密码。
|
||||
- 最近历史：近期 assignment 与 heartbeat 的执行记录。
|
||||
- **任务进度页面**：`http://127.0.0.1:8765/tasks` 展示本机 Host Agent 上已执行/正在执行
|
||||
的任务列表（状态、设备、时间戳），点击任务 ID 可查看逐步 timeline 含截图。
|
||||
- **任务进度页面**：`http://127.0.0.1:8765/tasks` 是本机实际执行任务的权威查看入口。
|
||||
它展示已执行/正在执行的任务、Cloud task ID 与 attempt；点击执行 ID 可查看每步
|
||||
timeline 的操作、前后截图、OCR 和 UI 树结果。
|
||||
|
||||
完整的 `HOST_AGENT_CONSOLE_*` 环境变量列表（端口、非回环 bind 的显式 opt-in、
|
||||
session TTL、历史记录条数上限等）参见 `docs/CLOUD_DEPLOYMENT.md`；生产/远程场景下
|
||||
应优先使用 SSH 端口转发访问该 Console，而不是直接把它暴露到非回环地址。
|
||||
|
||||
Host Agent 会把每步执行状态与截图持久化到本地 SQLite/文件系统，路径与 Runtime 自身的
|
||||
`tasks/tasks.sqlite3` 不冲突：
|
||||
Host Agent 会把每步执行状态、前后截图、OCR 和 UI 树结果持久化到本地
|
||||
SQLite/文件系统。该 Host Console 是这些实际执行证据的唯一 Web 查看入口：
|
||||
|
||||
```bash
|
||||
# 任务进度持久化路径（默认值，可通过环境变量覆盖）
|
||||
@@ -477,64 +416,34 @@ Host Agent 会把每步执行状态与截图持久化到本地 SQLite/文件系
|
||||
# HOST_AGENT_TASK_RETENTION_MAX_AGE_DAYS=7 # 超过 7 天的任务自动清理
|
||||
```
|
||||
|
||||
Retention 策略取"数量上限与天数上限中更严格的"——即先按 `max_count` 取最近 N 个、
|
||||
Retention 策略取"数量上限与天数上限中更严格的" - 即先按 `max_count` 取最近 N 个、
|
||||
再按 `max_age_days` 过滤掉过老的，最终保留两者中较小的集合。任务完成后，这些记录
|
||||
在 Console 的 `/tasks` 页面可查。
|
||||
|
||||
### 可选：由 Host Agent 托管 Appium 和 Runtime API
|
||||
### 可选：由 Host Agent 托管 Appium
|
||||
|
||||
默认情况下 Host Agent **不会**自动启动 Appium 或本地 Runtime API：必须按
|
||||
§6 在独立 Terminal 中保持 `appium --address 127.0.0.1 --port 4723` 运行，按
|
||||
§8 在另一个 Terminal 中启动 Runtime API。忘记其中任意一个，Host Agent 不会报错，
|
||||
heartbeat 仍会成功，但设备会静默保持 `offline`、所有任务卡在 `queued`。
|
||||
|
||||
`host-agent-dependency-supervisor` 是一个可选模式，让 Host Agent 自己把这两个
|
||||
外部进程作为子进程托管，覆盖单机真机工作流。它默认关闭，需要显式 opt-in：
|
||||
默认情况下 Host Agent 不会自动启动 Appium；可继续按 §6 在独立 Terminal 中保持
|
||||
`appium --address 127.0.0.1 --port 4723` 运行。也可以显式让 Host Agent 托管
|
||||
Appium，避免忘记启动导致设备保持 `offline`：
|
||||
|
||||
```bash
|
||||
export HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED="true"
|
||||
|
||||
# 任选其一或两者都开。两者默认 false。
|
||||
export HOST_AGENT_APPIUM_SUPERVISED="true"
|
||||
export HOST_AGENT_RUNTIME_SUPERVISED="true"
|
||||
|
||||
# 可覆盖默认地址/端口（默认值与 §6/§8 手动流程一致）：
|
||||
# export HOST_AGENT_APPIUM_HOST="127.0.0.1"
|
||||
# export HOST_AGENT_APPIUM_PORT="4723"
|
||||
# export HOST_AGENT_RUNTIME_HOST="127.0.0.1"
|
||||
# export HOST_AGENT_RUNTIME_PORT="8000"
|
||||
|
||||
# 单次 Host Agent 进程生命周期内允许的最大重启次数，默认 5。
|
||||
# export HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS="5"
|
||||
|
||||
uv run --package device-host-agent device-host-agent
|
||||
```
|
||||
|
||||
启用后的行为（详见 `openspec/changes/host-agent-dependency-supervisor/`）：
|
||||
|
||||
- **启动顺序**：Host Agent 在第一次 `connect_devices()` 之前，先按上面选中的
|
||||
依赖项依次做"先探测后启动"。这样一旦开启，Appium 不再需要单独的 Terminal。
|
||||
- **Adopt-don't-fight**：探测 `(host, port)` 时若已经有进程在监听并通过健康检查
|
||||
（Appium `GET /status` 返回 200 JSON，Runtime `GET /devices` 返回 200 JSON），
|
||||
Host Agent 会以 *adopted* 方式记录日志，**不会**再 spawn 一个重复进程，也不会
|
||||
在退出/崩溃时杀掉或重启它。如果端口被占但健康检查失败，记一条 port-conflict
|
||||
错误并跳过该依赖，不抢端口、不静默继续。
|
||||
- **崩溃重启**：只有 Host Agent 自己 spawn 出来的子进程才会被监控。子进程意外
|
||||
退出时，按指数退避（1s、2s、4s、8s，封顶 30s）重启；当某个依赖在本进程生命
|
||||
周期内累计达到 `HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS` 次重启后，停止
|
||||
再次尝试直到 Host Agent 重启。被 adopt 的进程永远不会被 Host Agent 重启或杀死。
|
||||
- **spawn 失败 ≠ crash**：如果 `appium` 可执行文件不在 `PATH` 上，启动会以
|
||||
"dependency-supervisor: appium spawn failed — executable not found" 形式记一条
|
||||
依赖主管特有的错误，与正常 crash 区分开。请确认按 §4.3 安装好 `appium` 和
|
||||
XCUITest/UiAutomator2 driver。
|
||||
- **退出时**：Host Agent 在自身 graceful shutdown 阶段只会 `terminate` 它自己
|
||||
spawn 的子进程；adopted 进程保留不动。
|
||||
- **不影响 Docker/Compose**：本模式只针对 macOS 单机真机工作流；
|
||||
`compose.yaml` / `compose.deploy.yaml` 完全不受影响。
|
||||
Host Agent 会先探测 Appium；健康实例会被 adopt 而不会重复启动或停止。仅由 Host
|
||||
Agent 自己启动的 Appium 子进程会在异常退出后按有界指数退避重启，并在 Host Agent
|
||||
正常关闭时 terminate。已移除的 `HOST_AGENT_RUNTIME_*` 变量会使启动明确失败；
|
||||
不要再配置或启动独立 Runtime 服务。
|
||||
|
||||
如果偏好保持对 Appium 终端日志的完全控制、或者已经在用其他进程管理工具
|
||||
（launchd、systemd、tmux 等）托管 Appium，可以继续使用 §6/§8 的手动流程，
|
||||
不开启本模式即可。
|
||||
（launchd、systemd、tmux 等）托管 Appium，可以继续使用 §6 的手动流程，不开启
|
||||
本模式即可。
|
||||
|
||||
## 10. 多设备与端口
|
||||
|
||||
@@ -629,16 +538,16 @@ wheel 与 Apple Silicon 兼容性。
|
||||
- [ ] `curl http://127.0.0.1:4723/status` 返回正常。
|
||||
- [ ] 直接 Python 验证能生成 `/tmp/device-agent-runtime.png`。
|
||||
- [ ] 实机 integration test 通过。
|
||||
- [ ] Runtime API 返回 `iphone-1`，并能执行 screenshot/tap/launch。
|
||||
- [ ] Host Agent Console 显示已连接设备，并能在 `/tasks` 查看一次完成任务的完整证据。
|
||||
- [ ] 如需 OCR，另行确认 PaddleOCR 在当前 Mac/Python 架构下可运行。
|
||||
- [ ] 云端受管部署已保存 Host identity，并能在 `/v1/hosts`、`/v1/devices` 中看到。
|
||||
|
||||
## 13. 后续代码改进建议
|
||||
|
||||
为了让后续执行不再依赖内联 Python 启动脚本，建议另开变更实现：
|
||||
为了让后续设备管理和验收更易操作，建议另开变更实现：
|
||||
|
||||
- 为 Console/REST 增加显式 connect/disconnect endpoint。
|
||||
- 增加正式 CLI，例如 `device-runtime serve --device-config ...`。
|
||||
- 为 Host Agent Console 增加显式 connect/disconnect 状态诊断。
|
||||
- 增加正式 CLI，用于校验 Host 的设备配置和 Appium 连通性。
|
||||
- 统一将遗留的 `APEX_WDA_*` 环境变量改名为 `DEVICE_RUNTIME_WDA_*`，并保留兼容期。
|
||||
- 将 PaddleOCR 改成 optional dependency，拆分基础控制与 OCR 安装路径。
|
||||
- 增加 macOS CI 的无真机 smoke test，以及受控环境中的真机验收脚本。
|
||||
|
||||
@@ -1,137 +1,176 @@
|
||||
## Context
|
||||
|
||||
Three surfaces currently cannot show a task's in-progress status:
|
||||
Host Agent executes Cloud assignments in-process through the shared
|
||||
`runtime.TaskRunner`. Its `HostAgentApplication` already constructs a
|
||||
Host-local `TaskMetadataStore` and `Timeline` and injects them into the
|
||||
runner factory. The missing link is task creation: `AssignmentExecutor`
|
||||
constructs a `Task`, calls `runner.run(task)`, and `TaskRunner` only
|
||||
issues updates. SQLite therefore receives updates for a row that does not
|
||||
exist, so the Host Agent `/tasks` UI is empty.
|
||||
|
||||
1. **Host Agent local console** (`apps/device-host-agent/host_agent/web/app.py`): `HostAgentApplication`'s builder calls `create_execution_factories(resolved_manager, host_agent_config=resolved_config)` without `metadata_store`/`timeline` (`app.py:218-223`), so the in-process `TaskRunner` (`execution.py:39-46`) runs with `metadata_store=None, timeline=None`. `runtime/task.py::TaskRunner._update_task()` only persists when `metadata_store` is truthy, so step transitions vanish. The console's only signal is `AgentStatusTracker.snapshot()` (`status.py:14-88`), a single opaque span: `task_id/device_id/goal/started_at`, cleared on `mark_assignment_finished()`.
|
||||
2. **Cloud Control Plane / Cloud Console**: the internal protocol (`packages/cloud-platform/cloud/internal_api/models.py`) only has claim, heartbeat, lease-renewal, and terminal-result messages. Lease renewal (`ActiveAssignmentRunner._renew_while_running`, `apps/device-host-agent/host_agent/lease.py:79-106`) already fires periodically (~1/3 of remaining lease) for every in-flight assignment, but carries no task-progress payload.
|
||||
3. **Runtime `console/` SPA**: `api/rest.py::create_app()` builds its own `TaskMetadataStore`/`Timeline`/`TaskRunner` (`api/rest.py:24-41`), queried by `api/console.py::create_console_router()`'s `/console/tasks*` routes, which `console/src/api.ts` calls against `API_BASE_URL` (default `http://127.0.0.1:8000`). This is a wholly separate process from Host Agent, with no shared store or IPC. `runtime/`-owned packages are forbidden from importing host/cloud concerns (`test_runtime_owned_packages_do_not_import_host_or_cloud_concerns`), and `host-agent-local-console/design.md` already treats Host Agent's local history as "not a system of record," distinct from Cloud's.
|
||||
The separate `api.rest` process was never on this execution path. It owned a
|
||||
different database and artifact root, so its REST endpoints and UI could only
|
||||
show tasks submitted directly to that unrelated process. Running it alongside
|
||||
a Host Agent created two competing operator entry points without transferring
|
||||
any history between them.
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals:**
|
||||
- Host Agent's local console shows live step-level progress for its current assignment (step index, step status, short summary), not just a static started-at snapshot.
|
||||
- Cloud Control Plane learns step-level progress near-real-time (piggybacked on existing periodic traffic, not a new polling loop) and Cloud Console renders it for an operator watching a remote task.
|
||||
- An operator can inspect a Host-Agent-executed task's progress/history from a web console, without coupling `runtime/` to host/cloud concerns.
|
||||
- Reuse existing, already-tested building blocks (`TaskMetadataStore`, `Timeline`, `api/console.py`'s query shape, the lease-renewal cadence) instead of inventing new storage or transport primitives.
|
||||
- Both the Host Agent's local task history and the Cloud Control Plane persist the *actual* per-step content sent to and received from the LLM (the real prompt text, not just the task's overall goal; the model's resulting decision, not just the parsed tool call) — so a step's record answers "what did we actually ask the model and what did it say," not just "what call did we make." Cloud's copy is the durable, centrally-searchable record for troubleshooting; the Host Agent's local copy remains the on-device record of what actually ran.
|
||||
|
||||
- Make durable task-row creation an invariant of `TaskRunner.run()` whenever
|
||||
a metadata store is configured.
|
||||
- Correlate a Host-local goal execution with the Cloud task ID and attempt that
|
||||
caused it, without making shared Runtime or storage packages import Cloud
|
||||
models.
|
||||
- Make the authenticated Host Agent console at port `8765` the authoritative
|
||||
web view for actual task execution: task list, detail, live status, before
|
||||
screenshot, operation, after screenshot, OCR observations, and normalized
|
||||
UI-tree results.
|
||||
- Preserve the shared Runtime Timeline as the evidence model and keep its
|
||||
before/after screenshot, OCR, and UI-tree capture behavior.
|
||||
- Retire the standalone Runtime REST service/UI and its Host Agent supervisor
|
||||
configuration while retaining Runtime, storage, MCP, and skill-sync library
|
||||
modules.
|
||||
- Retain the existing Cloud latest-progress and Cloud-proxy planner-decision
|
||||
history behavior. Cloud remains a fleet-level view and never stores
|
||||
screenshots.
|
||||
|
||||
**Non-Goals:**
|
||||
- No SSE/WebSocket push. All three surfaces keep polling (Host Agent console 5s, Cloud Console/`cloud-console` and `console/` at their existing intervals). Nothing here requires push infra, and adding it would be a bigger, separate change.
|
||||
- The coarse live step index/status/summary reported via lease renewal (D4/D5) stays *latest-snapshot-only* on the Cloud side (overwrite semantics) — it is a "what's happening right now" indicator, not a history mechanism, and is unaffected by the LLM-content decisions below.
|
||||
- Full step-by-step **LLM interaction** history *is* now synced to and durably persisted by Cloud (see D7/D8) — this is a deliberate reversal of this change's earlier draft, which had scoped Cloud to latest-only. The motivation: Host Agent deployments already route every AI Planner decision through Cloud API's existing `cloud-planner-proxy` endpoint when configured for the `cloud` transport, so Cloud is already on the natural path for this data and centralizing it there (rather than only on whichever edge host happens to still be running) is the more useful place for an operator doing centralized troubleshooting. This durable log is scoped strictly to LLM prompt/response text — it is not a general-purpose duplicate of the Host Agent's full `Timeline` (no screenshots, no scene JSON dumps beyond what's embedded in the prompt text itself).
|
||||
- No multi-backend Runtime console or cross-origin Host Agent integration. Runtime changes remain local to its existing same-origin API/UI and shared timeline evidence model.
|
||||
- No change to `driver/`/`device/`/`core/` device-control internals.
|
||||
- No screenshot or full-scene data leaves the Host Agent process as part of progress reporting, and no screenshot bytes are ever persisted by the Cloud Control Plane. (Full LLM prompt/response *text* is, by contrast, an explicit Goal below — see D7/D8 — which partially supersedes the original `cloud-planner-proxy` proposal's "does not durably persist ... full prompt text" statement. Screenshots remain excluded; prompts no longer are.)
|
||||
|
||||
- No SSE/WebSocket push; Host and Cloud consoles keep their existing polling.
|
||||
- No duplicate full evidence upload to Cloud. Screenshots and Timeline
|
||||
artifacts remain Host-local.
|
||||
- No new Host/Cloud dependencies in `runtime/`, `storage/`, `driver/`, or
|
||||
`device/`.
|
||||
- No replacement general-purpose device-control REST API. Operators use Host
|
||||
Agent device management and task pages for the managed execution workflow.
|
||||
|
||||
## Decisions
|
||||
|
||||
### D1: Wire a real `metadata_store`/`timeline` into Host Agent's `TaskRunner`, reusing the existing classes
|
||||
### D1: TaskRunner owns idempotent task metadata creation
|
||||
|
||||
`create_execution_factories()` already accepts `metadata_store: TaskMetadataStore | None` and `timeline: Timeline | None` (`execution.py:28-34`) — these are the same classes `api/rest.py` uses for the Runtime's own console. Reuse them as-is rather than inventing a Host-Agent-specific store: `HostAgentApplication`'s builder constructs a `TaskMetadataStore(db_path=<host-agent-local path>)` and `Timeline(ArtifactStore(<host-agent-local path>))` and passes them into `create_execution_factories()`.
|
||||
`TaskRunner.run()` will call `TaskMetadataStore.create_task(task)` before
|
||||
the first running-state update. `create_task()` will use idempotent insert
|
||||
semantics so callers that already created a direct Runtime task remain
|
||||
compatible and workflow-created tasks are captured automatically.
|
||||
|
||||
**Alternative considered**: a bespoke in-memory-only step recorder (cheaper, no disk I/O). Rejected because `TaskMetadataStore`/`Timeline` are already proven in production (Runtime's own console has run on them since the original `web-console` change) and reusing them means Host Agent's task/timeline data has the *exact* same shape as `api/console.py` already serializes — a prerequisite for D3 below. A bespoke format would need its own (de)serialization and its own console rendering code for no real benefit.
|
||||
This places the invariant at the shared execution boundary rather than relying
|
||||
on every caller to remember an out-of-band persistence call. It fixes goal
|
||||
assignments and prevents the same failure for `WorkflowRunner` planned-goal
|
||||
steps.
|
||||
|
||||
New config: `HOST_AGENT_TASK_PROGRESS_DB_PATH` (default e.g. `host_agent_data/task_progress.sqlite3`) and a matching artifact directory, kept separate from any local Runtime `tasks/` directory that might exist on the same machine, to avoid two unrelated processes silently sharing or colliding on a path.
|
||||
### D2: Host assignment correlation stays in the Host adapter
|
||||
|
||||
### D2: Bounded retention for the Host-Agent-local store
|
||||
Before executing a Cloud goal assignment, `AssignmentExecutor` creates the
|
||||
local task record with optional generic `source_task_id` and
|
||||
`source_attempt` metadata. Shared storage uses generic names and does not
|
||||
import Cloud types. The Host Agent task list/detail renders those fields as the
|
||||
Cloud task ID and attempt.
|
||||
|
||||
Unlike a Runtime dev session (short-lived, manually cleared), a Host Agent process runs indefinitely and executes many assignments over its lifetime. Unbounded `TaskMetadataStore` rows and `Timeline` screenshot artifacts would grow without limit.
|
||||
The local `Task.id` remains generated by the Runtime. Reusing the Cloud task
|
||||
ID as an artifact directory name would make retries overwrite each other and
|
||||
would admit unsafe path characters from an external identifier.
|
||||
|
||||
Decision: add a lightweight retention pass (e.g. on a timer, or opportunistically after each assignment finishes) that prunes tasks older than a configurable window or beyond a configurable count, deleting both the `tasks` row and its `Timeline`/`ArtifactStore` files. This is new logic — neither `TaskMetadataStore` nor `Timeline` currently supports deletion — scoped as a small addition to `storage/task_metadata.py` / `storage/timeline.py` (or a Host-Agent-side wrapper if adding delete methods to the shared `storage` package feels too broad; prefer extending `storage` since both Runtime and Host Agent benefit from bounded retention).
|
||||
### D3: Host Agent console is the authoritative evidence UI
|
||||
|
||||
### D3: Give Host Agent's own console read-only task/timeline pages instead of making `console/` multi-backend
|
||||
The Host Agent already owns the device manager, assignment executor,
|
||||
metadata store, Timeline, local account, session, and CSRF boundary. Its
|
||||
same-origin `/tasks` and `/tasks/{task_id}` pages therefore render the
|
||||
shared Timeline directly. The task list is named for Host executions rather
|
||||
than "Local Runtime tasks", and submission feedback tells the operator that
|
||||
the submitted Cloud task appears there when this Host begins execution.
|
||||
|
||||
Two options were evaluated for capability `host-agent-console-task-pages`:
|
||||
No browser needs to point a separate frontend at the Host Agent. This keeps
|
||||
the conservative local-account/session model and does not add CORS.
|
||||
|
||||
- **(a) Multi-backend Runtime `console/` SPA**: let the existing Vue SPA point at a Host Agent's console origin (a saved/selectable base URL) in addition to the local Runtime. Since D1 makes Host Agent's data shape identical to what `api/console.py` already serializes, the SPA's existing fetch/render code would work unmodified against a Host Agent origin.
|
||||
- **(b) Extend Host Agent's own server-rendered console** (`host_agent/web/app.py`) with new read-only pages for task list/detail/timeline, reusing `TaskMetadataStore`/`Timeline` query calls directly (the same calls `api/console.py`'s handlers make), rendered as server-side HTML via the existing f-string + `html.escape()` convention (no SPA, no frontend build) established by `host-agent-local-console`.
|
||||
### D4: Complete per-step evidence is rendered by the Host Agent
|
||||
|
||||
**Decision: (b).** (a) requires Host Agent's console to serve cross-origin, credentialed requests from whatever origin `console/` is running on (Vite dev server, or a different deployed origin) — meaning either relaxing Host Agent's CORS posture for its session-cookie+CSRF-protected endpoints, or reworking its auth model to tolerate cross-origin fetches. That is real new attack surface on a console whose entire local-only threat model (`console_bind_host` defaulting to loopback, explicit opt-in for non-loopback) was deliberately conservative. (b) reuses an already-accepted pattern (server-rendered, same-origin, same auth) and fully satisfies the actual operator need — a web page showing what a Host Agent is doing — without touching `runtime/`, `console/`, or Host Agent's CORS/auth posture at all. The cost is a small amount of view duplication (Host Agent's console and Runtime's `console/` render similar-shaped data with different templates/frameworks), judged acceptable given they serve different operational contexts (local direct-connect Runtime vs. remote Host Agent fleet).
|
||||
Timeline records retain distinct pre-action and post-action screenshot paths,
|
||||
the action description and arguments, the execution result, raw OCR
|
||||
observations, and the existing normalized UI-tree result. The Host Agent page
|
||||
creates data URIs only for available local artifacts and supports legacy
|
||||
records where the single `screenshot_path` is the post-action image.
|
||||
|
||||
Any future desire for a unified SPA across both surfaces is left as a follow-up, not blocked by this decision (D1's shared data shape keeps that door open).
|
||||
OCR is rendered when present. UI-tree output is rendered only for
|
||||
`get_ui_tree` and `ui_tree` records that contain normalized nodes; it uses a
|
||||
collapsible structured view while retaining the JSON result. No tool contract
|
||||
or duplicate persistence field is introduced.
|
||||
|
||||
### D4: Piggyback progress reporting on the existing lease-renewal call
|
||||
### D5: Host-local retention remains bounded
|
||||
|
||||
`ActiveAssignmentRunner._renew_while_running` (`lease.py:79-106`) already makes an authenticated, per-assignment, periodic call (`client.renew(assignment)`, roughly every 1/3 of remaining lease) while a task executes, validated server-side against `host_id`/`task_id`/`attempt`/`lease_id` (`internal_api/api.py:291-320`). This is the natural piggyback point for progress: extend `LeaseRenewalRequest` with an optional `progress: TaskProgressModel | None` field (new small model: `step_index: int`, `step_status: Literal[...]`, `summary: str` bounded length — no screenshot/scene payload) in `cloud/internal_api/models.py`, the single schema both sides already import directly (no parallel schema, matching existing convention noted for this protocol).
|
||||
The existing Host retention pass continues to remove metadata rows, Timeline
|
||||
records, and artifacts according to the configured count and age thresholds.
|
||||
The new task creation invariant must use that same store so it cannot create
|
||||
an unbounded second history source.
|
||||
|
||||
A small thread-safe "latest progress" holder (similar in spirit to `LeaseGuard`) is written by the execution thread (via a hook on `TaskRunner`/`AssignmentExecutor`, updated once per step) and read by `_renew_while_running` just before each renewal call, so no new timer/cadence is introduced.
|
||||
### D6: Cloud progress remains a latest snapshot on lease renewal
|
||||
|
||||
**Alternative considered**: a dedicated `POST /internal/v1/hosts/{host_id}/assignments/{task_id}/progress` endpoint called on its own cadence. Rejected — it would duplicate the auth/validation `renew_assignment` already does, and introduce a second periodic call where one already exists and fires at a reasonable frequency for this purpose.
|
||||
The Host execution thread writes a bounded latest-progress holder. The lease
|
||||
renewal path optionally carries its step index, status, and summary; Cloud
|
||||
stores only the latest snapshot for an active assignment and stops exposing it
|
||||
after terminal completion. No screenshot or scene payload enters this
|
||||
protocol.
|
||||
|
||||
### D5: Cloud persists only the latest progress snapshot, overwritten alongside lease renewal
|
||||
### D7: Cloud-proxy planner decisions remain durable Cloud history
|
||||
|
||||
`renew_lease()` (`repository.py:407`, `sql_repository.py:1480`) already takes a row lock and writes a lease-expiry update on every renewal; extend it to also accept and store the optional progress fields (few scalar columns — `progress_step_index`, `progress_step_status`, `progress_summary`, `progress_updated_at` — rather than a schema-flexible JSON blob, keeping it queryable and consistent with the rest of the row's typed columns). This requires a new Alembic migration (head is currently `0007_llm_provider_management`) plus the equivalent SQLite schema change, applied to both `repository.py` and `sql_repository.py` to keep the dual-backend contract (`cloud-control-plane` spec's "Deployment and local persistence modes share one contract" requirement).
|
||||
For `AI_PLANNER_TRANSPORT=cloud`, Cloud's existing planner-decision endpoint
|
||||
persists successful system/user prompts, tool calls, arguments, and a
|
||||
task/attempt-scoped step index. The direct transport intentionally produces no
|
||||
such Cloud history. This log has bounded terminal-task retention and never
|
||||
persists request screenshot bytes.
|
||||
|
||||
Progress columns are cleared (or simply superseded and ignored) once a terminal result is recorded — they represent "what's happening right now," not history; Cloud's durable attempt/result history is unaffected and unduplicated.
|
||||
### D8: Retire the standalone Runtime REST service and UI
|
||||
|
||||
### D6: Cloud Console reads progress from the existing task/attempt query path, not a new endpoint
|
||||
Remove `api/rest.py`, `api/console.py`, `api/console_web.py`, their
|
||||
templates/static assets, their package-data declarations, and their
|
||||
service/UI tests. Preserve `api/mcp.py`, `api/errors.py`, skill-sync, and
|
||||
skill-catalog modules because they are independent library integrations.
|
||||
|
||||
Extend whatever response model Cloud Console's task list/detail already uses (Cloud API public router) with the optional latest-progress fields from D5, rather than adding a new endpoint. Cloud Console renders it as a small inline badge/line ("step 4: tapping login button") next to the existing status, refreshed on the SPA's existing polling interval.
|
||||
Remove `HOST_AGENT_RUNTIME_SUPERVISED`,
|
||||
`HOST_AGENT_RUNTIME_HOST`, and `HOST_AGENT_RUNTIME_PORT`. The optional
|
||||
dependency supervisor continues to support Appium only. Configuration with a
|
||||
removed Runtime-supervision variable fails with an actionable migration error
|
||||
instead of silently doing nothing.
|
||||
|
||||
### D7: Capture full LLM interaction history for free via the existing `cloud-planner-proxy` decide endpoint, not a new reporting channel
|
||||
### D9: Documentation points operators to Host Agent
|
||||
|
||||
Investigated the Host Agent → Cloud call path in detail: when a Host Agent is configured with `AI_PLANNER_TRANSPORT=cloud`, `AIPlanner.plan()` (`runtime/ai_planner.py`) calls `CloudProxyToolCallingClient.decide()` (`apps/device-host-agent/host_agent/cloud_planner_client.py:46-95`), which `POST`s the *complete* `system_prompt`/`user_prompt` (plus `task_id`/`attempt`/`lease_id` from `current_planner_execution_context()`) to Cloud API's `/hosts/{host_id}/planner/decide` (`packages/cloud-platform/cloud/internal_api/api.py::decide_planner_call`, line ~367). That handler already resolves and returns a `ToolCallDecision` (`tool_name`/`arguments`/`usage`) and already does per-call bookkeeping (`pool.store.settle_host_token_reservation(...)`, line ~463) using the same repository object this change's D5 already touches for lease renewal.
|
||||
|
||||
This means **every planning step's actual prompt and resulting decision already flows through a Cloud-owned request handler** when the `cloud` transport is used — no new endpoint, no new protocol field, no queue/batching scheme is needed to get full LLM content to Cloud. The only change needed is to make that handler *persist* what it currently discards.
|
||||
|
||||
**Decision**: extend `decide_planner_call` to, immediately after computing `decision` (success path only — a `ToolCallUnavailable`/502 path persists nothing), insert one row into a new log table (D8) keyed by `(task_id, attempt, step_index)`, where `step_index` is assigned by the Cloud side itself (an auto-incrementing counter scoped to `task_id`+`attempt`, e.g. `select count(*) + 1` under the same row lock, or a DB sequence/identity column) — Host Agent does not need to track or send a step counter for this.
|
||||
|
||||
**Explicit limitation, called out rather than papered over**: this only captures LLM content for hosts using the `cloud` transport. A host on the (still-default) `direct` transport never sends its prompts to Cloud at all — Cloud has zero LLM content for that host's tasks, and only ever sees the coarse index/status/summary from D4/D5's lease-renewal piggyback (which is transport-agnostic, since it's driven by `TaskRunner` step completion, not by the planner's transport choice). This is a real operational dependency: centralized LLM-interaction troubleshooting via Cloud Console requires the fleet (or the hosts an operator cares about) to run with `AI_PLANNER_TRANSPORT=cloud`. This change does not make `cloud` the new default transport — that remains a separate, already-existing configuration decision outside this change's scope.
|
||||
|
||||
**Alternative considered**: extend the D4 lease-renewal piggyback to also carry full prompt/response text (queued, not overwritten, so no step is lost between renewals). Rejected: it would duplicate a transport that already exists for exactly this payload (the decide call itself) whenever `cloud` transport is active, and would still need a *separate* new channel for the `direct`-transport case where Cloud never sees the prompt anyway — i.e., it does not actually solve the `direct`-transport gap, so it only adds complexity without expanding coverage.
|
||||
|
||||
### D8: New bounded-retention table for the full per-step LLM decision log, extended on both repository backends
|
||||
|
||||
Add a new table (e.g. `planner_decision_log`): `id`, `host_id`, `task_id`, `attempt`, `step_index`, `system_prompt` (text), `user_prompt` (text), `tool_name`, `arguments_json` (text), `created_at`. No screenshot column — screenshots are never sent to this endpoint's persistence path (the request's `screenshot_base64` is used only to call the LLM provider and is never written to this log, consistent with the Non-Goals screenshot exclusion).
|
||||
|
||||
Like D5, this needs a new Alembic migration and the equivalent SQLite path, implemented on both `repository.py`'s SQLite-backed implementation and `sql_repository.py::SQLAlchemyCloudRepository` to preserve the existing dual-backend contract. Unlike D5's few-nullable-columns-on-an-existing-row approach, this is an independent append-only table (one row per decide call, not an overwrite), since the whole point is durable per-step history rather than a live snapshot.
|
||||
|
||||
**Retention**: this table grows once per planning step across the whole fleet, indefinitely, on a shared multi-tenant Cloud database — unbounded growth is a real risk here in a way D5's single-row-per-assignment overwrite never was. Decision: a scheduled prune job (mirrors D2's Host-Agent-local retention) deletes rows whose owning task reached a terminal state more than a configurable window ago (default: prune 7 days after task terminal, or once the task itself is pruned/archived by whatever existing Cloud task-retention policy applies — reuse that cadence rather than inventing a second one if `cloud-control-plane` already has one; otherwise default to a simple time-based prune).
|
||||
|
||||
### D9: Fix the shared `Timeline`/`TaskRunner`/`AIPlanner` path so the *actual* per-step prompt and response are recorded locally, not just the task goal and the parsed tool call
|
||||
|
||||
Independent of Cloud persistence, the existing local recording is itself wrong today: `TaskRunner._append_timeline()` (`runtime/task.py:296-319`) calls `Timeline.append(prompt=task.goal, tool_call={"action": ..., "description": ..., "args": ...}, ...)`. `task.goal` is the overall task goal, not the per-step prompt actually sent to the LLM — the real per-step prompt (`planner_user_prompt(goal, scene_json, history_summary)`, built in `ai_planner.py::plan()`) is constructed, sent, and discarded entirely within `AIPlanner.plan()`, never reaching `TaskRunner`. Likewise `ToolCallDecision` (`runtime/tool_calling_client.py:23-27`) carries only the parsed `tool_name`/`arguments`/`usage` — any raw response text/content the model returned is discarded during parsing (`_decision_from_anthropic_response`/`_decision_from_openai_response`).
|
||||
|
||||
This is a pre-existing gap in a component shared by Runtime and Host Agent alike (not new to this change), and it undermines the very "step-level detail" goal D1 already committed to — a persisted step whose "prompt" field is just the task's goal repeated on every row is not useful for troubleshooting.
|
||||
|
||||
**Decision**: extend `ToolCallDecision` with the actual `user_prompt`/`system_prompt` it was given (or have `AIPlanner.plan()` return a small side-channel result instead of changing the `Planner` interface's return type) so `TaskRunner._append_timeline()` can pass the real per-step prompt into `Timeline.append()`. Rename `Timeline`/`TimelineRecord`'s `prompt` field's meaning (or add a new field) to unambiguously mean "the prompt actually sent to the LLM for this step." This fix lands in the shared `runtime`/`storage` packages, so both Host Agent's local console (D3) and the existing Runtime `console/` automatically benefit — it is not Host-Agent-specific plumbing.
|
||||
|
||||
### D10: Persist and render before/after action evidence in the Runtime timeline
|
||||
|
||||
The existing Timeline captures only one screenshot after an action, while the planning screenshot used before the action is transient. Extend each Timeline record with distinct before/after screenshot paths, preserving the existing `screenshot_path` as a backward-compatible alias for the after screenshot. `TaskRunner` captures each image immediately before and after invoking the executor, including failed action attempts. The Runtime console API inlines both images and the existing task-detail page renders them beside the recorded action and result.
|
||||
|
||||
OCR is already available during perception but fused into the normalized Scene, where an OCR value that overlaps a UI-tree node can lose its raw provenance. Preserve raw OCR observations on `Scene` for local timeline capture only; keep them out of `Scene.to_dict()` so the LLM-facing planner payload does not grow with duplicated text. The Runtime task-detail page renders the persisted OCR list when it is available and handles legacy records without it.
|
||||
|
||||
The existing `get_ui_tree`/`ui_tree` tool returns a normalized flat list of UI nodes, not the driver-specific raw XML hierarchy. Its StepResult is already persisted in the Timeline. Detect those action names in the Runtime UI and render the returned nodes in a collapsible structured view while retaining the full JSON result below it. This makes the inspection result readable without changing the tool response contract or storing a second tree copy.
|
||||
Operator documentation no longer instructs users to start `uvicorn
|
||||
api.rest:create_app` or browse port `8000`. It identifies the Host Agent
|
||||
console at `http://127.0.0.1:8765/tasks` as the execution-history authority,
|
||||
explains its local authentication, and documents that the Cloud Console is a
|
||||
fleet/progress and Cloud-proxy LLM-history surface rather than a screenshot
|
||||
store.
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
- [Risk] Per-step SQLite writes in a long-lived Host Agent process add I/O overhead → Mitigation: this is the same write pattern Runtime's own console has always used per step; no new proof of acceptability needed. If profiling later shows it matters for very high step-rate tasks, batching/debouncing is a follow-up, not a blocker here.
|
||||
- [Risk] Unbounded local disk growth from `Timeline` screenshots on an indefinitely-running Host Agent → Mitigation: D2's bounded retention pass; must ship in the same change as D1, not deferred, since D1 alone would otherwise introduce an unbounded-growth regression.
|
||||
- [Risk] New DB columns/migration touch both `repository.py` (SQLite) and `sql_repository.py` (Postgres) — drift between the two has been a real defect category in this codebase (see `cloud-control-plane-integration` archive notes) → Mitigation: contract/parity tests already exist for this dual-backend boundary; extend them to cover the new progress columns.
|
||||
- [Risk] View duplication between Host Agent's server-rendered task pages (D3) and Runtime `console/`'s task pages (different frameworks, same data shape) → Mitigation: accepted trade-off (see D3 rationale); shared data shape keeps future unification possible without rework.
|
||||
- [Risk] Progress payload could accidentally grow to include sensitive data (scene dumps, prompts) if a future change casually extends `TaskProgressModel` → Mitigation: `summary` field is explicitly bounded/plain-text only; code review for this change and future extensions should treat this the same as the existing `cloud-planner-proxy` no-screenshot-persistence rule (screenshots specifically remain excluded from every Cloud-side table this change introduces).
|
||||
- [Risk] `planner_decision_log` (D8) grows unboundedly across the whole fleet, unlike D5's single-row-per-assignment overwrite → Mitigation: D8's retention/prune job must ship in the same change as D7/D8, not deferred, for the same reason D2 must ship alongside D1.
|
||||
- [Risk] Full prompt text can itself contain sensitive on-screen content (whatever text was visible in the scene description embedded in the prompt) — persisting it centrally is a deliberate trade-off the user has explicitly requested for centralized troubleshooting, but it is a real expansion of what Cloud stores → Mitigation: no additional mitigation beyond what's already decided (screenshots still excluded); flagged here so it is a visible, intentional decision rather than a silent scope creep.
|
||||
- [Risk] Full LLM history in Cloud is silently absent for any host on the `direct` transport, which could read as "it's broken" rather than "expected" → Mitigation: Cloud Console should visibly distinguish "no progress reported yet" from "this host does not report LLM content" (e.g. by also surfacing the host's configured transport), rather than just showing an empty history with no explanation.
|
||||
- Per-step metadata writes add small SQLite I/O. This is the same local store
|
||||
already selected for Host history, and bounded retention limits growth.
|
||||
- A Host-local evidence record is only available while retained on that Host.
|
||||
This is intentional: it reflects the actual device execution and avoids
|
||||
sending screenshots to Cloud.
|
||||
- Removing the unauthenticated Runtime REST service is a breaking operator
|
||||
change. Clear configuration errors and documentation avoid a silent
|
||||
fallback to a nonexistent inspection surface.
|
||||
- Full Cloud-proxy prompts can contain visible screen text. This is the
|
||||
previously accepted Cloud troubleshooting trade-off; screenshot bytes remain
|
||||
excluded.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
1. Add `TaskProgressModel` and the optional `progress` field to `LeaseRenewalRequest`/response in `cloud/internal_api/models.py`. Backward compatible: `progress` is optional, older Host Agents omit it.
|
||||
2. Add progress columns + Alembic migration `0008_task_progress_columns`; extend `renew_lease()` in both repository implementations to accept/store them.
|
||||
3. Extend Cloud Console's task read model and UI to render the new fields (no-op if absent, keeping rollback trivial).
|
||||
4. Wire `metadata_store`/`timeline` into Host Agent's `create_execution_factories()` call site, add the retention pass (D2), and add the progress-holder hook feeding D4's renewal piggyback.
|
||||
5. Extend Host Agent's local console with the read-only task/timeline pages (D3).
|
||||
6. Fix `Timeline`/`TaskRunner`/`AIPlanner`/`ToolCallingClient` to record the real per-step prompt and response locally (D9) — independent of Cloud, benefits both Host Agent and Runtime consoles immediately.
|
||||
7. Add `planner_decision_log` + Alembic migration for it (D8) on both repository backends, with its retention/prune job.
|
||||
8. Extend `decide_planner_call` (`internal_api/api.py`) to persist each resolved decision into `planner_decision_log` (D7).
|
||||
9. Extend Cloud Console with a per-task LLM interaction history view reading the new table, including the "host uses `direct` transport, no LLM content available" distinction from the Risks section.
|
||||
10. Rollback: each step is independently revertible (optional field, additive columns, additive tables, additive UI, additive Host Agent wiring) — no destructive migration is required; both `0008` and the new decision-log migration's down-revisions drop only what they added.
|
||||
1. Upgrade the Host Agent code. Existing task databases gain nullable source
|
||||
correlation columns on startup; legacy Timeline records remain readable.
|
||||
2. Remove any `HOST_AGENT_RUNTIME_*` environment variables and stop any
|
||||
`api.rest` process. Start or browse only the Host Agent console for local
|
||||
execution evidence.
|
||||
3. Confirm a completed Host assignment appears at `/tasks` with its Cloud
|
||||
task/attempt correlation and complete Timeline evidence.
|
||||
4. Roll back only by restoring the prior release. The retired REST/UI routes
|
||||
are deliberately not kept as a compatibility alias because their storage
|
||||
was not connected to Host execution.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Exact retention window/count defaults for D2 (time-based vs. count-based, or both) — left for tasks.md to pick a concrete, documented default (e.g. keep last 50 tasks or 7 days, whichever is smaller) rather than block design on it.
|
||||
- Whether Cloud Console's existing task list/detail component can absorb the new fields with a small edit or needs a new sub-component — an implementation detail, not an architectural fork.
|
||||
- Whether a future change should unify Host Agent's server-rendered task pages and Runtime `console/`'s SPA into one shared frontend, now that D1 gives them an identical underlying data shape — explicitly deferred, not part of this change.
|
||||
- Exact retention default for D8's `planner_decision_log` (prune-after-terminal window, or reuse an existing Cloud task-retention cadence if one already exists) — left for tasks.md to pick a concrete default rather than block design on it.
|
||||
- Whether `step_index` in D8 should be assigned via a `SELECT count(*) + 1` under a row lock or a dedicated per-`(task_id, attempt)` counter/sequence — an implementation detail to resolve in tasks.md, not an architectural fork.
|
||||
- Manual verification still requires a real Host Agent, Appium, and device.
|
||||
Automated coverage verifies persistence, correlation, rendering, and
|
||||
service removal; real hardware validates screenshots and OCR availability.
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
## Why
|
||||
|
||||
Nobody can see a task while it is running. Host Agent's local console only shows a coarse "current assignment" snapshot (task id, device id, goal, started-at) because the `TaskRunner` it drives is wired with `metadata_store=None, timeline=None` — every step transition happens in memory and is discarded the instant the assignment finishes. Cloud Control Plane only learns about an assignment at claim, heartbeat, and terminal-result time, so Cloud Console has nothing better to show. The local Runtime (`api/rest.py` + `console/`) is a separate process with its own independent `TaskMetadataStore`/`Timeline`, so it never sees a task that a Host Agent executed at all. An operator debugging a stuck or misbehaving task currently has no live signal anywhere in the system until the task finishes or times out.
|
||||
Host Agent now wires a local `TaskMetadataStore` and `Timeline` into its `TaskRunner`, but its Cloud-assignment path still creates a `Task` and immediately calls `runner.run(task)`. `TaskRunner` only updates an existing metadata row, so every update affects zero rows and the Host Agent task page remains empty. The standalone Runtime REST service/UI is a separate process with unrelated storage, so it cannot be the inspection surface for Host Agent executions. An operator who submits or receives a task therefore has no authoritative web view of the actions that actually ran on that Host.
|
||||
|
||||
## What Changes
|
||||
|
||||
- Wire Host Agent's `AssignmentExecutor` / `create_execution_factories()` with a real `metadata_store` and `timeline` (or Host-Agent-local equivalents) so the in-process `TaskRunner` actually records step-by-step status instead of discarding it.
|
||||
- Make `TaskRunner.run()` create its metadata row idempotently before its first status update, so every execution path, including workflow-owned tasks, persists history when a metadata store is configured. Have Host Agent register Cloud task/attempt correlation before goal execution so operators can match a submitted Cloud task to its local execution record.
|
||||
- Expose that step-level detail through Host Agent's local console: extend `AgentStatusTracker`/`/api/status` (or add a focused endpoint) with current step index, step status, and a short in-progress step log; render it in the dashboard's "Current assignment" section instead of the static started-at-only view.
|
||||
- Add a progress-reporting path from Host Agent to Cloud Control Plane so the control plane learns step-level state near-real-time rather than only at claim/heartbeat/terminal-result. Extend the existing `cloud.internal_api.models` Pydantic schema (the single source of truth Host Agent already imports directly) rather than introducing a parallel schema.
|
||||
- Persist and expose the latest per-assignment progress on the Cloud Control Plane side, and surface it in Cloud Console so an operator watching a remote task sees live step progress, not just "dispatched" / "succeeded" / "failed".
|
||||
- Give an operator a web console view of Host-Agent-executed task progress without making the `runtime/`-owned packages import host or cloud concerns (enforced by `test_runtime_owned_packages_do_not_import_host_or_cloud_concerns`). After evaluating the alternative of making the Runtime `console/` SPA multi-backend (pointing its existing JS bundle at a Host Agent's console origin), this change instead extends Host Agent's own server-rendered local console with read-only task list/detail/timeline pages, reusing the same query shape `api/console.py` already exposes to the Runtime SPA. This avoids new cross-origin/session-cookie surface between the SPA and Host Agent, and keeps `runtime/` untouched. See design.md for the full trade-off analysis.
|
||||
- Make Host Agent's own authenticated, server-rendered `:8765/tasks` pages the authoritative web entry point for actual execution history. They render the shared Timeline records directly and retain the existing same-origin session/CSRF boundary.
|
||||
- All three surfaces continue to use polling (matching current behavior); this change does not introduce SSE/WebSocket infrastructure unless design.md finds a compelling reason to.
|
||||
- Fix the shared `Timeline`/`TaskRunner`/`AIPlanner` recording path so a persisted step's "prompt" is the *actual* prompt sent to the LLM for that step (not the task's overall goal) and the model's resulting decision is captured too — this pre-existing gap affects Runtime and Host Agent alike and undermines the step-level detail this change otherwise adds.
|
||||
- Persist a durable, per-step log of full LLM prompt/response content on the Cloud Control Plane, for centralized troubleshooting — reusing the already-existing `cloud-planner-proxy` decide endpoint as the capture point (no new protocol/endpoint) rather than the coarse lease-renewal piggyback used for live index/status. This durable log is populated only for hosts using the `cloud` planner transport; hosts on the `direct` transport still get only the coarse index/status via lease renewal. Cloud Console gains a view to browse a task's full LLM interaction history.
|
||||
- Extend the local Runtime timeline so every executed action retains a before screenshot, operation detail, after screenshot, and any available raw OCR observations; render that evidence in the Runtime task-detail UI and expose it from the existing Runtime console API. When a step uses the existing UI-tree tool, render its normalized node result as a structured, collapsible view as well.
|
||||
- Keep before/after screenshots, operation detail, raw OCR observations, and normalized UI-tree results in the shared Runtime Timeline; render all of that evidence in the Host Agent task-detail page.
|
||||
- Retire the standalone Runtime REST service, its unauthenticated console/UI, and Host Agent Runtime-supervision settings. Preserve the shared `runtime/`, `storage/`, and non-REST `api/` library modules used by the Host Agent and MCP integrations.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- `host-agent-task-progress`: Host Agent captures step-level execution progress for its in-flight assignment (via a wired `metadata_store`/`timeline`) and exposes it through its local console/API.
|
||||
- `host-agent-task-progress`: Host Agent captures step-level execution progress for every in-process task, correlates Cloud assignments with local records, and exposes it through its local console/API.
|
||||
- `cloud-task-progress-visibility`: Cloud Control Plane receives, persists, and exposes near-real-time step-level progress for assignments it has dispatched to a Host Agent, and Cloud Console renders it.
|
||||
- `host-agent-console-task-pages`: Host Agent's local server-rendered console gains read-only task list/detail/timeline pages (mirroring `api/console.py`'s task query shape) so an operator can inspect a Host-Agent-executed task's progress and history without needing the separate Runtime `console/` SPA or violating the `runtime/`-package host/cloud isolation boundary.
|
||||
- `runtime-task-evidence`: Runtime task history retains and renders pre/post action evidence, raw OCR observations, and existing UI-tree inspection results.
|
||||
- `host-agent-console-task-pages`: Host Agent's local server-rendered console gains the complete before/after evidence, OCR, and UI-tree views for Host-Agent-executed tasks and is the only web inspection surface for those executions.
|
||||
- `runtime-task-evidence`: Runtime task history retains complete pre/post action evidence, raw OCR observations, and existing UI-tree inspection results for Host Agent rendering.
|
||||
- `runtime-standalone-service`: the standalone Runtime REST service and its UI are removed; Runtime remains an execution library rather than a second operational console.
|
||||
|
||||
### Modified Capabilities
|
||||
- `host-agent-protocol`: add a requirement that the Host Agent reports in-progress step-level status updates to the control plane (in addition to the existing heartbeat/claim/renewal/result operations), and that the control plane accepts and stores them per active assignment.
|
||||
@@ -32,5 +34,6 @@ Nobody can see a task while it is running. Host Agent's local console only shows
|
||||
- `runtime/task.py`, `runtime/ai_planner.py`, `runtime/tool_calling_client.py`, `storage/timeline.py`, `storage/artifact_store.py`, `core/models.py`, `perception/scene_builder.py` — fix the shared step-recording path so the real per-step prompt and the model's response are captured, retain pre/post action screenshots and raw OCR observations, not just the task goal and parsed tool call.
|
||||
- `packages/cloud-platform/cloud/internal_api/models.py`, `packages/cloud-platform/cloud/internal_api/api.py` (`decide_planner_call`), `repository.py`/`sql_repository.py`, a new Alembic migration — new progress-reporting request/response models and a persistence + query path for latest per-assignment coarse progress, *and* a new durable per-step `planner_decision_log` table (with its own retention job) populated from the existing planner-decision endpoint.
|
||||
- `cloud-console/` (Vue3 SPA) — new UI to render live per-assignment progress, and a new view to browse a task's full LLM interaction history.
|
||||
- `api/console.py`, `api/console_web.py`, `api/templates/runtime_console/*` — expose and render Runtime-local step evidence; Host Agent task pages receive the data through the shared timeline shape.
|
||||
- No changes anticipated to `driver/`, `device/`, `core/` device-control internals.
|
||||
- `api/rest.py`, `api/console.py`, `api/console_web.py`, `api/templates/runtime_console/*`, `api/static/runtime_console/*`, their tests, package data, and Runtime-supervision configuration — removed.
|
||||
- `apps/device-host-agent/host_agent/assignment.py`, `execution.py`, `web/app.py`, and its task templates — create correlated execution records and render the complete shared Timeline evidence.
|
||||
- `runtime/task.py`, `storage/task_metadata.py`, and `storage/timeline.py` — make durable task creation an execution invariant while retaining generic storage boundaries.
|
||||
|
||||
+56
-23
@@ -1,34 +1,67 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Host Agent local console exposes step-level status for the current assignment
|
||||
The Host Agent's local console SHALL display, for its currently executing assignment, the current step index, step status, and a short summary, sourced from the Host Agent's local task metadata store, refreshed on the console's existing polling interval.
|
||||
|
||||
#### Scenario: An assignment is currently executing
|
||||
- **WHEN** an operator views the Host Agent local console dashboard while an assignment is executing
|
||||
- **THEN** the dashboard shows the current step index, step status, and a short summary for that assignment, updating on subsequent polls
|
||||
|
||||
#### Scenario: No assignment is currently executing
|
||||
- **WHEN** an operator views the dashboard while the Host Agent is idle
|
||||
- **THEN** the dashboard shows no in-progress step information
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Host Agent local console exposes read-only task history with per-step detail and screenshots
|
||||
The Host Agent's local console SHALL provide authenticated, read-only pages listing recently executed tasks and, for a selected task, its full per-step history including any captured screenshots, sourced from the Host Agent's local task metadata store and timeline.
|
||||
The Host Agent's local console SHALL provide authenticated, read-only pages
|
||||
listing recently executed local tasks and, for a selected task, its full
|
||||
per-step history from the Host-local metadata store and Timeline. The task
|
||||
detail SHALL show available before and after screenshots, operation details and
|
||||
arguments, execution result, OCR observations, and normalized UI-tree results.
|
||||
It SHALL render legacy Timeline records that only have a single screenshot as a
|
||||
post-action image.
|
||||
|
||||
#### Scenario: Operator lists recent tasks
|
||||
- **WHEN** an authenticated operator opens the Host Agent local console's task list page
|
||||
- **THEN** it shows tasks from the local task metadata store, most recent first, including tasks that have already reached a terminal state
|
||||
#### Scenario: Operator lists recent Host executions
|
||||
- **WHEN** an authenticated operator opens the Host Agent local console's task
|
||||
list page
|
||||
- **THEN** it shows local executions most recent first, including terminal
|
||||
tasks and any available Cloud task ID and attempt correlation
|
||||
|
||||
#### Scenario: Operator inspects a completed task's step history
|
||||
- **WHEN** an authenticated operator opens the detail page for a specific completed task
|
||||
- **THEN** the page shows each recorded step in order, including its tool call, result, and any captured screenshot
|
||||
- **WHEN** an authenticated operator opens the detail page for a completed
|
||||
Host execution
|
||||
- **THEN** the page shows each recorded step in order with its tool call,
|
||||
result, and available before/after screenshots
|
||||
|
||||
#### Scenario: OCR was captured for a step
|
||||
- **WHEN** the selected Timeline record contains OCR observations
|
||||
- **THEN** the detail page shows each observation's text, confidence, and
|
||||
bounds
|
||||
|
||||
#### Scenario: A UI-tree tool returned normalized nodes
|
||||
- **WHEN** the selected Timeline record invoked `get_ui_tree` or `ui_tree`
|
||||
and its result contains normalized nodes
|
||||
- **THEN** the detail page exposes a structured, collapsible node view while
|
||||
retaining the persisted result JSON
|
||||
|
||||
#### Scenario: A legacy timeline record is displayed
|
||||
- **WHEN** a Timeline record has only `screenshot_path`
|
||||
- **THEN** the detail page renders it as the post-action image without failing
|
||||
|
||||
#### Scenario: Unauthenticated request
|
||||
- **WHEN** a request to the task list or task detail pages is made without a valid Host Agent console session
|
||||
- **THEN** the Host Agent rejects the request the same way it rejects unauthenticated requests to its other console pages
|
||||
- **WHEN** a request to the task list or task detail pages is made without a
|
||||
valid Host Agent console session
|
||||
- **THEN** the Host Agent rejects the request the same way it rejects
|
||||
unauthenticated requests to its other console pages
|
||||
|
||||
### Requirement: Host Agent local console task pages require no new cross-origin surface
|
||||
The Host Agent local console's task pages SHALL be served same-origin from the Host Agent's existing web application, without introducing new CORS allowances or a dependency on the separate Runtime `console/` frontend.
|
||||
The Host Agent local console's task pages SHALL be served same-origin from the
|
||||
Host Agent's existing web application, without introducing new CORS allowances
|
||||
or a dependency on a separate Runtime frontend.
|
||||
|
||||
#### Scenario: Task pages are requested
|
||||
- **WHEN** an operator's browser requests the Host Agent local console's task pages
|
||||
- **THEN** the pages are served by the Host Agent's own application using its existing session/CSRF protections, with no additional cross-origin configuration required
|
||||
- **WHEN** an operator's browser requests the Host Agent local console's task
|
||||
pages
|
||||
- **THEN** the pages are served by the Host Agent's own application using its
|
||||
existing session/CSRF protections, with no additional cross-origin
|
||||
configuration required
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Host Agent console is the authority for actual execution evidence
|
||||
The Host Agent local console SHALL be the web authority for task evidence
|
||||
produced by that Host's in-process execution path. A standalone Runtime
|
||||
service/UI SHALL NOT be required or consulted to inspect a Host execution.
|
||||
|
||||
#### Scenario: A Cloud task is executed by a Host Agent
|
||||
- **WHEN** an operator opens that Host Agent's task page after execution starts
|
||||
- **THEN** the page reads the same Host-local metadata and Timeline that the
|
||||
executing `TaskRunner` writes
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Supervisor is opt-in and disabled by default
|
||||
The Host Agent SHALL NOT start, adopt-check, or supervise Appium unless
|
||||
`HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED` is explicitly set to true. Appium
|
||||
supervision SHALL additionally require `HOST_AGENT_APPIUM_SUPERVISED=true`
|
||||
and SHALL default to false.
|
||||
|
||||
#### Scenario: Default configuration behaves exactly as before
|
||||
- **WHEN** a Host Agent starts with no
|
||||
`HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED` or related Appium environment
|
||||
variable set
|
||||
- **THEN** the Host Agent does not attempt to connect to, probe, or spawn
|
||||
Appium, and its heartbeat/claim behavior is unchanged
|
||||
|
||||
#### Scenario: Top-level flag on and Appium flag off
|
||||
- **WHEN** `HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED=true` and
|
||||
`HOST_AGENT_APPIUM_SUPERVISED=false`
|
||||
- **THEN** the Host Agent does not probe, adopt, or spawn Appium
|
||||
|
||||
### Requirement: Spawn supervised dependencies that are not already running
|
||||
The Host Agent SHALL spawn Appium as a child process when Appium supervision is
|
||||
enabled and no healthy Appium instance is adopted, via
|
||||
`appium --address <host> --port <port>` and SHALL forward the child
|
||||
process's stdout/stderr into the Host Agent's own logging, tagged by dependency
|
||||
name.
|
||||
|
||||
#### Scenario: Appium is not running at Host Agent startup
|
||||
- **WHEN** Appium supervision is enabled and no healthy Appium instance is
|
||||
already listening
|
||||
- **THEN** the Host Agent spawns Appium before proceeding to its first
|
||||
device-connect attempt, and its output is visible in Host Agent logs
|
||||
|
||||
#### Scenario: Spawn fails because the executable is missing
|
||||
- **WHEN** the Host Agent attempts to spawn Appium but `appium` is not found
|
||||
on `PATH`
|
||||
- **THEN** the Host Agent logs a dependency-supervisor-specific startup error
|
||||
naming the missing dependency, distinct from a runtime crash of an
|
||||
already-started process
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Runtime supervision settings are retired
|
||||
The Host Agent SHALL reject `HOST_AGENT_RUNTIME_SUPERVISED`,
|
||||
`HOST_AGENT_RUNTIME_HOST`, and `HOST_AGENT_RUNTIME_PORT` because the
|
||||
standalone Runtime service no longer exists.
|
||||
|
||||
#### Scenario: A legacy Runtime supervision variable is set
|
||||
- **WHEN** startup configuration includes any removed Runtime supervision
|
||||
variable
|
||||
- **THEN** configuration fails with an actionable migration error
|
||||
+58
-14
@@ -1,30 +1,74 @@
|
||||
## ADDED Requirements
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Host Agent records step-level execution detail for its in-process TaskRunner
|
||||
The Host Agent SHALL construct its in-process `TaskRunner` with a durable metadata store and timeline so that every step transition (status, index, the actual prompt submitted to the LLM for that step, the model's resulting decision, result, and screenshot when captured) is persisted as it happens, rather than discarded when the assignment completes. The persisted prompt SHALL be the prompt actually sent to the LLM for that specific step, not the task's overall goal.
|
||||
The Host Agent SHALL construct its in-process `TaskRunner` with a durable
|
||||
metadata store and Timeline. `TaskRunner.run()` SHALL create the task's
|
||||
metadata row idempotently before its first status update, so every execution
|
||||
path persists its task status and evidence rather than discarding updates for a
|
||||
missing row. Every completed step SHALL retain its index, actual per-step LLM
|
||||
prompt and decision when available, tool call, result, distinct before/after
|
||||
screenshots when captured, raw OCR observations when available, and normalized
|
||||
UI-tree result when the invoked tool returned one.
|
||||
|
||||
#### Scenario: A goal assignment starts execution
|
||||
- **WHEN** the Host Agent's `AssignmentExecutor` invokes its `TaskRunner`
|
||||
- **THEN** the task metadata row exists before the runner records its running
|
||||
status
|
||||
|
||||
#### Scenario: A step completes during goal execution
|
||||
- **WHEN** the Host Agent's `TaskRunner` completes a step while executing an assigned goal
|
||||
- **THEN** the step's status, index, the actual per-step LLM prompt and response, tool call, result, and any captured screenshot are persisted to the Host Agent's local task metadata store and timeline before the next step begins
|
||||
- **WHEN** the Host Agent's `TaskRunner` completes a step while executing an
|
||||
assigned goal
|
||||
- **THEN** the step's status, index, actual per-step LLM prompt and response,
|
||||
tool call, result, and available evidence are persisted before the next step
|
||||
begins
|
||||
|
||||
#### Scenario: A workflow creates a planned-goal task
|
||||
- **WHEN** a `WorkflowRunner` invokes a Host Agent-configured
|
||||
`TaskRunner` for a planned-goal step
|
||||
- **THEN** that task is persisted without requiring the workflow caller to
|
||||
create a metadata row separately
|
||||
|
||||
#### Scenario: An assignment finishes
|
||||
- **WHEN** an assignment reaches a terminal state (succeeded or failed)
|
||||
- **THEN** its full step history remains queryable from the Host Agent's local store after the in-memory `Task` object is discarded
|
||||
- **THEN** its full step history remains queryable from the Host Agent's local
|
||||
store after the in-memory `Task` object is discarded
|
||||
|
||||
### Requirement: Host-Agent-local task history is retained within a bounded window
|
||||
The Host Agent SHALL prune persisted task metadata, timeline records, and associated screenshot artifacts once they exceed a configurable retention window or count, so that indefinite process uptime does not cause unbounded local disk growth.
|
||||
The Host Agent SHALL prune persisted task metadata, Timeline records, and
|
||||
associated screenshot artifacts once they exceed a configurable retention
|
||||
window or count, so that indefinite process uptime does not cause unbounded
|
||||
local disk growth.
|
||||
|
||||
#### Scenario: Retention window is exceeded
|
||||
- **WHEN** a persisted task's age or position exceeds the configured retention threshold
|
||||
- **THEN** the Host Agent removes that task's metadata row, timeline records, and screenshot artifacts from local storage
|
||||
- **WHEN** a persisted task's age or position exceeds the configured retention
|
||||
threshold
|
||||
- **THEN** the Host Agent removes that task's metadata row, Timeline records,
|
||||
and screenshot artifacts from local storage
|
||||
|
||||
#### Scenario: Retention has not been exceeded
|
||||
- **WHEN** a persisted task is within the configured retention threshold
|
||||
- **THEN** its metadata, timeline records, and screenshot artifacts remain available for query
|
||||
- **THEN** its metadata, Timeline records, and screenshot artifacts remain
|
||||
available for query
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Host Agent correlates local execution records with Cloud assignments
|
||||
For a Cloud-dispatched goal assignment, the Host Agent SHALL persist the Cloud
|
||||
task ID and attempt alongside its generated local Runtime task ID before
|
||||
execution starts. The correlation fields SHALL remain optional and generic in
|
||||
the shared storage layer.
|
||||
|
||||
#### Scenario: A Cloud goal assignment begins
|
||||
- **WHEN** the Host Agent begins executing a Cloud goal assignment
|
||||
- **THEN** the local task row records that assignment's Cloud task ID and
|
||||
attempt
|
||||
|
||||
#### Scenario: A task is not Cloud-dispatched
|
||||
- **WHEN** a shared Runtime caller executes a task without Host/Cloud
|
||||
assignment context
|
||||
- **THEN** the task metadata row is created and the optional source
|
||||
correlation fields remain empty
|
||||
|
||||
## REMOVED Requirements
|
||||
|
||||
### Requirement: Host Agent local task storage is isolated from an unrelated local Runtime
|
||||
The Host Agent SHALL use a configurable, Host-Agent-specific database and artifact path for its task metadata store and timeline, distinct from any local Runtime API's own task storage path, so that the two processes cannot silently collide or share state when run on the same machine.
|
||||
|
||||
#### Scenario: Host Agent and local Runtime run on the same machine
|
||||
- **WHEN** both a Host Agent process and a local Runtime API process run on the same machine with their default configurations
|
||||
- **THEN** each process reads and writes its own task metadata store and timeline without observing or modifying the other's data
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Runtime is not exposed as a standalone REST service or web console
|
||||
The repository SHALL not ship a standalone Runtime REST application, its
|
||||
unauthenticated web console, or JSON console routes. The shared Runtime and
|
||||
storage packages SHALL remain reusable execution libraries for the Host Agent
|
||||
and other in-process callers.
|
||||
|
||||
#### Scenario: An operator needs to inspect a Host-executed task
|
||||
- **WHEN** an operator needs task evidence for a Host Agent execution
|
||||
- **THEN** the operator uses the authenticated Host Agent console rather than
|
||||
starting or querying a separate Runtime service
|
||||
|
||||
#### Scenario: A package uses shared Runtime execution
|
||||
- **WHEN** the Host Agent or another in-process caller creates a
|
||||
`TaskRunner`
|
||||
- **THEN** it continues to use the shared Runtime and storage packages without
|
||||
importing a REST or UI adapter
|
||||
|
||||
### Requirement: Host Agent does not supervise a retired Runtime service
|
||||
The Host Agent SHALL not expose Runtime-supervision configuration or spawn a
|
||||
Runtime REST subprocess. It MAY continue to optionally supervise Appium.
|
||||
|
||||
#### Scenario: Host Agent dependency supervision is enabled
|
||||
- **WHEN** `HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED=true` and Appium
|
||||
supervision is enabled
|
||||
- **THEN** the Host Agent probes and supervises Appium only
|
||||
|
||||
#### Scenario: A removed Runtime-supervision variable is configured
|
||||
- **WHEN** a Host Agent configuration includes a removed
|
||||
`HOST_AGENT_RUNTIME_*` variable
|
||||
- **THEN** startup fails with a message directing the operator to the Host
|
||||
Agent console and Appium-only supervision
|
||||
+39
-16
@@ -1,38 +1,61 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Runtime persists complete evidence for each executed action
|
||||
The Runtime SHALL persist, for each action it attempts, a screenshot captured immediately before the executor call, the action description and arguments, the execution result, and a screenshot captured immediately after the executor call. Existing timeline records that contain only the legacy single screenshot SHALL remain readable, with that screenshot treated as the post-action image.
|
||||
The shared Runtime SHALL persist, for each action it attempts, a screenshot
|
||||
captured immediately before the executor call, the action description and
|
||||
arguments, the execution result, and a screenshot captured immediately after
|
||||
the executor call. Existing Timeline records that contain only the legacy
|
||||
single screenshot SHALL remain readable, with that screenshot treated as the
|
||||
post-action image.
|
||||
|
||||
#### Scenario: An action succeeds
|
||||
- **WHEN** the Runtime executes an action for a task
|
||||
- **THEN** its timeline record includes distinct before and after screenshots, the action detail, and the execution result
|
||||
- **THEN** its Timeline record includes distinct before and after screenshots,
|
||||
action detail, and execution result
|
||||
|
||||
#### Scenario: An action fails
|
||||
- **WHEN** the Runtime executor exhausts its retries for an action
|
||||
- **THEN** the action's timeline record still includes any screenshots that were captured and the failure result before the task is marked failed
|
||||
- **THEN** the Timeline record still includes any captured screenshots and the
|
||||
failure result before the task is marked failed
|
||||
|
||||
#### Scenario: A legacy timeline record is read
|
||||
- **WHEN** a timeline record has only the prior `screenshot_path` field
|
||||
- **THEN** the Runtime exposes it as the post-action screenshot without failing to render the record
|
||||
#### Scenario: A legacy Timeline record is read
|
||||
- **WHEN** a Timeline record has only the prior `screenshot_path` field
|
||||
- **THEN** the Runtime exposes it as the post-action screenshot without
|
||||
failing to render the record
|
||||
|
||||
### Requirement: Runtime task evidence exposes available OCR observations
|
||||
The Runtime SHALL persist raw OCR observations associated with the scene used to plan an action when available, without adding duplicate OCR data to the LLM-facing normalized Scene payload. The Runtime task-detail UI SHALL render available OCR text, confidence, and bounds, and SHALL render normally when no OCR result exists.
|
||||
### Requirement: Runtime task evidence retains available OCR observations
|
||||
The shared Runtime SHALL persist raw OCR observations associated with the scene
|
||||
used to plan an action when available, without adding duplicate OCR data to the
|
||||
LLM-facing normalized Scene payload. The Host Agent task-detail UI SHALL render
|
||||
available OCR text, confidence, and bounds, and SHALL render normally when no
|
||||
OCR result exists.
|
||||
|
||||
#### Scenario: OCR found text while planning an action
|
||||
- **WHEN** perception produced one or more OCR observations for the action's planning scene
|
||||
- **THEN** the corresponding timeline record includes those observations and the Runtime task-detail UI displays them
|
||||
- **WHEN** perception produced one or more OCR observations for the action's
|
||||
planning scene
|
||||
- **THEN** the corresponding Timeline record includes those observations and
|
||||
the Host Agent task-detail page displays them
|
||||
|
||||
#### Scenario: OCR was unavailable or found no text
|
||||
- **WHEN** perception yields no OCR observations
|
||||
- **THEN** the Runtime records the action evidence and renders the task detail without an OCR result list
|
||||
- **THEN** the Runtime records the action evidence and the Host Agent task
|
||||
detail renders without an OCR result list
|
||||
|
||||
### Requirement: Runtime task evidence renders UI-tree inspection results
|
||||
When a Runtime step invokes the existing `get_ui_tree` or `ui_tree` tool and the persisted result contains normalized UI nodes, the Runtime task-detail UI SHALL render those nodes in a structured, collapsible view while retaining the recorded JSON result. The Runtime SHALL NOT change the tool's response contract or duplicate the result in a separate persistence field.
|
||||
### Requirement: Runtime task evidence retains UI-tree inspection results
|
||||
The Runtime SHALL retain a UI-tree inspection result when a step invokes the
|
||||
existing `get_ui_tree` or `ui_tree` tool and the result contains normalized
|
||||
nodes. The Host Agent task-detail UI SHALL render those nodes in a structured,
|
||||
collapsible view while retaining the recorded JSON result. The Runtime SHALL
|
||||
NOT change the tool response contract or duplicate the result in a separate
|
||||
persistence field.
|
||||
|
||||
#### Scenario: UI-tree inspection succeeds
|
||||
- **WHEN** a task step uses `get_ui_tree` or `ui_tree` and returns one or more normalized nodes
|
||||
- **THEN** the task-detail UI displays each node's type, visible text or identifier, bounds, and available confidence
|
||||
- **WHEN** a task step uses `get_ui_tree` or `ui_tree` and returns one or
|
||||
more normalized nodes
|
||||
- **THEN** the Host Agent task-detail page displays each node's type, visible
|
||||
text or identifier, bounds, and available confidence
|
||||
|
||||
#### Scenario: A non-UI-tree step is displayed
|
||||
- **WHEN** a task step did not invoke a UI-tree tool
|
||||
- **THEN** the task-detail UI does not render an empty UI-tree section
|
||||
- **THEN** the Host Agent task-detail page does not render an empty UI-tree
|
||||
section
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
## 1. Host Agent local task storage (D1, D2)
|
||||
|
||||
- [x] 1.1 Add `HOST_AGENT_TASK_PROGRESS_DB_PATH` (and matching artifact directory config) to `HostAgentConfig`/`load_host_agent_config()`, with a Host-Agent-specific default distinct from any local Runtime `tasks/` path.
|
||||
- [x] 1.1 Add `HOST_AGENT_TASK_PROGRESS_DB_PATH` (and matching artifact directory config) to `HostAgentConfig`/`load_host_agent_config()`, with Host-Agent-specific defaults for durable execution history.
|
||||
- [x] 1.2 In `HostAgentApplication`'s builder (`app.py:218-223`), construct a `TaskMetadataStore`/`Timeline` from that config and pass them into `create_execution_factories(..., metadata_store=..., timeline=...)`.
|
||||
- [x] 1.3 Add delete/prune methods to `storage/task_metadata.py::TaskMetadataStore` and `storage/timeline.py::Timeline` (remove a task's row, timeline records, and screenshot artifacts).
|
||||
- [x] 1.4 Implement a bounded retention pass in the Host Agent (default: keep the newer of "last 50 tasks" or "7 days", whichever keeps fewer rows) that runs after each assignment finishes, calling the new prune methods.
|
||||
- [x] 1.5 Add unit tests: step transitions persist during execution; a task's history is queryable after the assignment completes and the in-memory `Task` is discarded; retention prunes tasks beyond the configured threshold; Host Agent and a local Runtime process using default paths on the same machine do not collide.
|
||||
- [x] 1.5 Add unit tests: step transitions persist during execution; a task's history is queryable after the assignment completes and the in-memory `Task` is discarded; retention prunes tasks beyond the configured threshold; Host Agent task metadata and artifacts use their configured local paths.
|
||||
|
||||
## 2. Host Agent reports progress during lease renewal (D4)
|
||||
|
||||
@@ -31,14 +31,14 @@
|
||||
## 5. Host Agent local console task pages (D3)
|
||||
|
||||
- [x] 5.1 Extend `AgentStatusTracker`/`/api/status` (or a small addition alongside it) to surface the current step index/status/summary for the in-progress assignment, sourced from the same progress holder built in section 2, and update the dashboard's "Current assignment" rendering to show it.
|
||||
- [x] 5.2 Add authenticated, read-only task list and task detail/timeline routes to `host_agent/web/app.py`, querying the Host-Agent-local `TaskMetadataStore`/`Timeline` (reusing the same query calls `api/console.py` makes) and rendering server-side HTML consistent with the existing dashboard's f-string + `html.escape()` convention, including inlined screenshots on the detail/timeline page.
|
||||
- [x] 5.2 Add authenticated, read-only task list and task detail/timeline routes to `host_agent/web/app.py`, querying the Host-Agent-local `TaskMetadataStore`/`Timeline` and rendering server-side Jinja HTML consistent with the existing console, including inlined screenshots on the detail/timeline page.
|
||||
- [x] 5.3 Gate the new routes behind the existing Host Agent console session/CSRF protection; verify no new CORS configuration is introduced.
|
||||
- [x] 5.4 Add tests: unauthenticated requests to the new routes are rejected the same way as other console routes; task list/detail/timeline pages render expected data including screenshots for a completed task.
|
||||
|
||||
## 6. Documentation and verification
|
||||
|
||||
- [x] 6.1 Update `docs/MACOS_IPHONE_SETUP.md` and/or `docs/CLOUD_DEPLOYMENT.md` with the new Host Agent config vars (`HOST_AGENT_TASK_PROGRESS_DB_PATH` and retention settings) and a short note on where to view live/historical task progress in each console.
|
||||
- [x] 6.2 Run `uv run --all-packages pytest -m "not integration"` and targeted Cloud API / Host Agent test suites; run `cloud-console/` and `console/`-equivalent Vitest suites for the touched frontend.
|
||||
- [x] 6.2 Run `uv run --all-packages pytest -m "not integration"` and targeted Cloud API / Host Agent test suites; run the Cloud Console Vitest suite for its touched frontend.
|
||||
- [x] 6.3 Run Ruff check/format and `compileall` across touched packages.
|
||||
- [x] 6.4 Run `openspec validate --strict` for this change.
|
||||
- [ ] 6.5 Manual verification (requires a real Host Agent + Appium/device setup per `docs/MACOS_IPHONE_SETUP.md`): run a real task end-to-end and confirm step progress appears live in the Host Agent console and Cloud Console, and that full step history with screenshots is browsable afterward in the Host Agent console.
|
||||
@@ -48,7 +48,7 @@
|
||||
- [x] 7.1 Extend `ToolCallDecision` (`runtime/tool_calling_client.py:23-27`) with the actual prompt content given to that call (or return it via a small side-channel from `AIPlanner.plan()`, not by changing the `Planner.plan()` return type used by other planners).
|
||||
- [x] 7.2 Update `TaskRunner._append_timeline()` (`runtime/task.py:296-319`) to pass the real per-step prompt and the model's resulting decision into `Timeline.append()`, instead of `task.goal`.
|
||||
- [x] 7.3 Update `storage/timeline.py`'s `Timeline`/`TimelineRecord` field(s) so the persisted meaning is unambiguously "the prompt actually sent to the LLM for this step" (rename or add a field; keep backward-compatible read of any already-persisted rows if a Runtime dev DB might already have old-shaped rows).
|
||||
- [x] 7.4 Update any renderer of this data (`api/console.py`, Runtime `console/`, and the new Host Agent console task pages from section 5) to show the corrected field.
|
||||
- [x] 7.4 Update the Host Agent console task pages to show the corrected field without requiring a separate Runtime renderer.
|
||||
- [x] 7.5 Add unit tests: a persisted step's prompt matches what `AIPlanner.plan()` actually sent for that step (not the task goal), across at least one multi-step task.
|
||||
|
||||
## 8. Cloud persists full per-step LLM decision history via the existing cloud-planner-proxy endpoint (D7, D8)
|
||||
@@ -67,10 +67,20 @@
|
||||
- [x] 9.3 When a task's host used the `direct` transport (no persisted decisions and the host's configured transport is known to be `direct`), show an explicit "not reported by this host's transport" state rather than an empty list.
|
||||
- [x] 9.4 Add Vitest coverage for populated history, empty-but-cloud-transport (task hasn't produced any decisions yet), and direct-transport-hidden cases.
|
||||
|
||||
## 10. Runtime per-step evidence (D10)
|
||||
## 10. Shared Runtime per-step evidence (D4)
|
||||
|
||||
- [x] 10.1 Extend the shared Scene/Timeline/ArtifactStore model to retain raw OCR observations and separate before/after screenshots while preserving compatibility with existing single-screenshot records.
|
||||
- [x] 10.2 Capture before and after screenshots around every TaskRunner executor call, then persist the action, result, and available OCR observations in the same timeline record.
|
||||
- [x] 10.3 Extend the Runtime console JSON API and task-detail UI to render before/after screenshots, operation details, and available OCR results.
|
||||
- [x] 10.4 Render persisted normalized UI-tree tool results in the Runtime task-detail UI without changing the tool contract or duplicating stored data.
|
||||
- [x] 10.5 Add focused Timeline, TaskRunner, perception, JSON API, Runtime UI, and UI-tree regression coverage; run the relevant format, lint, test, and strict OpenSpec validation commands.
|
||||
- [x] 10.3 Preserve the shared Timeline representation needed to render before/after screenshots, operation details, and available OCR results without changing Runtime execution contracts.
|
||||
- [x] 10.4 Preserve persisted normalized UI-tree tool results without changing the tool contract or duplicating stored data.
|
||||
- [x] 10.5 Add focused Timeline, TaskRunner, perception, and UI-tree regression coverage; run the relevant format, lint, test, and strict OpenSpec validation commands.
|
||||
|
||||
## 11. Host Agent execution authority and Runtime service retirement (D1-D9)
|
||||
|
||||
- [x] 11.1 Make `TaskRunner.run()` create a task metadata row idempotently before status updates; add optional generic source task/attempt fields to task metadata and record Cloud assignment correlation in the Host adapter.
|
||||
- [x] 11.2 Add regression coverage for Host-dispatched goal execution, workflow-owned TaskRunner execution, retry-safe metadata creation, and Cloud task/attempt correlation.
|
||||
- [x] 11.3 Extend Host Agent `/tasks` and task-detail rendering so the actual execution list and detail page show correlation, before/after screenshots, operation detail, OCR observations, and structured normalized UI-tree results, including legacy screenshot compatibility.
|
||||
- [x] 11.4 Remove the standalone Runtime REST service/UI, its package data and dedicated tests, and remove Runtime supervision from Host Agent configuration/supervision with an actionable legacy-config error.
|
||||
- [x] 11.5 Update operator documentation and OpenSpec artifacts to direct execution inspection to Host Agent `:8765/tasks` and remove port `8000` instructions.
|
||||
- [x] 11.6 Run focused Host Agent and shared Runtime tests, workspace non-integration tests, Ruff, compileall, and strict OpenSpec validation.
|
||||
- [ ] 11.7 Manual verification (requires a real Host Agent + Appium/device setup): submit or dispatch a task, then confirm the Host Agent console is the only local execution-history UI and shows the complete retained evidence.
|
||||
|
||||
@@ -6,15 +6,11 @@ requires-python = ">=3.13,<3.14"
|
||||
dependencies = [
|
||||
"anthropic>=0.69.0",
|
||||
"Appium-Python-Client>=5.1.1",
|
||||
"fastapi>=0.115.0",
|
||||
"httpx>=0.27.0",
|
||||
"jinja2>=3.1",
|
||||
"mcp>=1.27,<2",
|
||||
"openai>=1.0.0",
|
||||
"paddlepaddle>=3.0.0",
|
||||
"paddleocr>=3.0.0",
|
||||
"python-multipart>=0.0.20",
|
||||
"uvicorn[standard]>=0.30.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
@@ -55,13 +51,6 @@ include = [
|
||||
"workflow*",
|
||||
]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
api = [
|
||||
"templates/runtime_console/*.html",
|
||||
"static/runtime_console/*.css",
|
||||
"static/runtime_console/*.js",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
addopts = "-ra --import-mode=importlib"
|
||||
|
||||
@@ -99,6 +99,8 @@ class TaskRunner:
|
||||
*,
|
||||
should_stop: StopRequested | None = None,
|
||||
) -> Task:
|
||||
if self.metadata_store:
|
||||
self.metadata_store.create_task(task)
|
||||
context = TaskContext(task_id=task.id, goal=task.goal)
|
||||
world_handle = self._start_world_view(task.id)
|
||||
if world_handle is not None:
|
||||
|
||||
@@ -13,14 +13,20 @@ class TaskMetadataStore:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._ensure_schema()
|
||||
|
||||
def create_task(self, task: Task) -> None:
|
||||
def create_task(
|
||||
self,
|
||||
task: Task,
|
||||
*,
|
||||
source_task_id: str | None = None,
|
||||
source_attempt: int | None = None,
|
||||
) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
insert into tasks (
|
||||
insert or ignore into tasks (
|
||||
id, goal, device_id, status, created_at, updated_at,
|
||||
completed_at, failure_reason
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
completed_at, failure_reason, source_task_id, source_attempt
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
task.id,
|
||||
@@ -31,6 +37,8 @@ class TaskMetadataStore:
|
||||
task.updated_at.isoformat(),
|
||||
task.completed_at.isoformat() if task.completed_at else None,
|
||||
task.failure_reason,
|
||||
source_task_id,
|
||||
source_attempt,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -100,10 +108,27 @@ class TaskMetadataStore:
|
||||
created_at text not null,
|
||||
updated_at text not null,
|
||||
completed_at text,
|
||||
failure_reason text
|
||||
failure_reason text,
|
||||
source_task_id text,
|
||||
source_attempt integer
|
||||
)
|
||||
"""
|
||||
)
|
||||
self._ensure_column(connection, "source_task_id", "text")
|
||||
self._ensure_column(connection, "source_attempt", "integer")
|
||||
|
||||
@staticmethod
|
||||
def _ensure_column(
|
||||
connection: sqlite3.Connection,
|
||||
name: str,
|
||||
definition: str,
|
||||
) -> None:
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in connection.execute("pragma table_info(tasks)").fetchall()
|
||||
}
|
||||
if name not in columns:
|
||||
connection.execute(f"alter table tasks add column {name} {definition}")
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(self.db_path)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Verifies that the cloud workspace package is purely additive: every existing
|
||||
module it composes (``runtime.task``, ``workflow.runner``, ``driver.registry``,
|
||||
``api.console``) remains unaware of the ``cloud`` package in its source.
|
||||
``api.mcp``) remains unaware of the ``cloud`` package in its source.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -38,9 +38,6 @@ def _existing_module_paths() -> list[Path]:
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
files.append(path)
|
||||
console = PROJECT_ROOT / "api" / "console.py"
|
||||
if console.exists():
|
||||
files.append(console)
|
||||
mcp = PROJECT_ROOT / "api" / "mcp.py"
|
||||
if mcp.exists():
|
||||
files.append(mcp)
|
||||
|
||||
@@ -1,242 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from core.models import Task
|
||||
from device.manager import DeviceManager
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from storage.artifact_store import ArtifactStore
|
||||
from storage.device_config import DeviceConfigStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
from tests.fakes import PNG_10X20, FakeDriver
|
||||
|
||||
|
||||
def _client(tmp_path, *, manager=None, runner=None, config_store=None, timeline=None):
|
||||
pytest.importorskip("fastapi")
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api.rest import create_app
|
||||
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
app = create_app(
|
||||
manager=manager or DeviceManager(),
|
||||
metadata_store=metadata_store,
|
||||
task_runner=runner,
|
||||
device_config_store=config_store
|
||||
or DeviceConfigStore(tmp_path / "device_config.sqlite3"),
|
||||
timeline=timeline or Timeline(ArtifactStore(tmp_path / "history")),
|
||||
)
|
||||
return TestClient(app), metadata_store
|
||||
|
||||
|
||||
def test_console_status_endpoints_cover_empty_and_populated_states(tmp_path) -> None:
|
||||
manager = DeviceManager()
|
||||
client, metadata_store = _client(tmp_path, manager=manager)
|
||||
|
||||
assert client.get("/console/devices").json() == []
|
||||
assert client.get("/console/tasks").json() == []
|
||||
|
||||
manager.register_device(
|
||||
"iphone-1",
|
||||
lambda: FakeDriver(),
|
||||
name="Desk iPhone",
|
||||
driver_type="wda",
|
||||
)
|
||||
older = Task(
|
||||
id="task-old",
|
||||
goal="open settings",
|
||||
device_id="iphone-1",
|
||||
created_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
)
|
||||
newer = Task(
|
||||
id="task-new",
|
||||
goal="search",
|
||||
device_id="iphone-2",
|
||||
status="running",
|
||||
created_at=datetime(2026, 1, 2, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 1, 2, tzinfo=UTC),
|
||||
)
|
||||
metadata_store.create_task(older)
|
||||
metadata_store.create_task(newer)
|
||||
|
||||
devices = client.get("/console/devices").json()
|
||||
assert devices == [
|
||||
{
|
||||
"id": "iphone-1",
|
||||
"name": "Desk iPhone",
|
||||
"status": "idle",
|
||||
"driver_type": "wda",
|
||||
"connection_info": {},
|
||||
"capability_tags": [],
|
||||
}
|
||||
]
|
||||
assert [task["id"] for task in client.get("/console/tasks").json()] == [
|
||||
"task-new",
|
||||
"task-old",
|
||||
]
|
||||
assert [
|
||||
task["id"] for task in client.get("/console/tasks?device_id=iphone-1").json()
|
||||
] == ["task-old"]
|
||||
assert [
|
||||
task["id"] for task in client.get("/console/tasks?status=running").json()
|
||||
] == ["task-new"]
|
||||
assert client.get("/console/tasks/task-old").json()["goal"] == "open settings"
|
||||
assert client.get("/console/tasks/missing").status_code == 404
|
||||
|
||||
|
||||
def test_console_timeline_inlines_screenshot_and_handles_empty_history(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
client, metadata_store = _client(tmp_path, timeline=timeline)
|
||||
task = Task(id="task-1", goal="tap search", device_id="iphone-1")
|
||||
metadata_store.create_task(task)
|
||||
|
||||
assert client.get("/console/tasks/task-1/timeline").json() == []
|
||||
|
||||
timeline.append(
|
||||
task_id="task-1",
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="tap search",
|
||||
tool_call={"action": "tap", "args": {"x": 1, "y": 2}},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
|
||||
records = client.get("/console/tasks/task-1/timeline").json()
|
||||
assert records[0]["index"] == 1
|
||||
assert records[0]["image_base64"] == base64.b64encode(PNG_10X20).decode("ascii")
|
||||
assert client.get("/console/tasks/missing/timeline").status_code == 404
|
||||
|
||||
|
||||
def test_console_timeline_inlines_before_and_after_screenshots(tmp_path) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
client, metadata_store = _client(tmp_path, timeline=timeline)
|
||||
metadata_store.create_task(Task(id="task-evidence", goal="tap", device_id="phone"))
|
||||
before = b"before"
|
||||
after = b"after"
|
||||
timeline.append(
|
||||
task_id="task-evidence",
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="tap",
|
||||
tool_call={"action": "tap", "description": "tap search"},
|
||||
result={"ok": True},
|
||||
before_screenshot=before,
|
||||
after_screenshot=after,
|
||||
)
|
||||
|
||||
record = client.get("/console/tasks/task-evidence/timeline").json()[0]
|
||||
assert record["before_image_base64"] == base64.b64encode(before).decode("ascii")
|
||||
assert record["after_image_base64"] == base64.b64encode(after).decode("ascii")
|
||||
assert record["image_base64"] == base64.b64encode(after).decode("ascii")
|
||||
|
||||
|
||||
def test_console_device_registration_and_unregistration(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
manager = DeviceManager()
|
||||
client, _ = _client(tmp_path, manager=manager, config_store=config_store)
|
||||
|
||||
rejected = client.post(
|
||||
"/console/devices",
|
||||
json={"driver_type": "android", "connection_info": {}},
|
||||
)
|
||||
assert rejected.status_code == 400
|
||||
assert config_store.list() == []
|
||||
assert manager.list_devices() == []
|
||||
|
||||
response = client.post(
|
||||
"/console/devices",
|
||||
json={
|
||||
"driver_type": "wda",
|
||||
"name": "Desk iPhone",
|
||||
"connection_info": {
|
||||
"server_url": "http://127.0.0.1:4723",
|
||||
"udid": "abc123",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
device_id = response.json()["id"]
|
||||
assert response.json()["status"] == "idle"
|
||||
assert config_store.get(device_id)["connection_info"]["udid"] == "abc123"
|
||||
assert [device.id for device in manager.list_devices()] == [device_id]
|
||||
|
||||
delete_response = client.delete(f"/console/devices/{device_id}")
|
||||
assert delete_response.status_code == 204
|
||||
assert config_store.get(device_id) is None
|
||||
assert manager.list_devices() == []
|
||||
assert client.delete("/console/devices/missing").status_code == 404
|
||||
|
||||
|
||||
def test_console_config_get_update_and_validation(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
config_store.set_setting("max_steps", 7)
|
||||
runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
|
||||
client, _ = _client(tmp_path, runner=runner, config_store=config_store)
|
||||
|
||||
assert runner.config.max_steps == 7
|
||||
assert client.get("/console/config").json() == {"max_steps": 7}
|
||||
|
||||
response = client.put("/console/config", json={"max_steps": 30})
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"max_steps": 30}
|
||||
assert runner.config.max_steps == 30
|
||||
assert config_store.get_setting("max_steps") == "30"
|
||||
|
||||
rejected = client.put("/console/config", json={"max_steps": 0})
|
||||
assert rejected.status_code == 400
|
||||
assert runner.config.max_steps == 30
|
||||
assert config_store.get_setting("max_steps") == "30"
|
||||
|
||||
|
||||
def test_console_startup_reloads_persisted_devices_and_settings(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
config_store.add(
|
||||
device_id="persisted-1",
|
||||
name="Persisted iPhone",
|
||||
driver_type="wda",
|
||||
connection_info={"udid": "abc123"},
|
||||
)
|
||||
config_store.set_setting("max_steps", 31)
|
||||
manager = DeviceManager()
|
||||
runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
|
||||
|
||||
client, _ = _client(
|
||||
tmp_path,
|
||||
manager=manager,
|
||||
runner=runner,
|
||||
config_store=config_store,
|
||||
)
|
||||
|
||||
assert runner.config.max_steps == 31
|
||||
assert [device.id for device in manager.list_devices()] == ["persisted-1"]
|
||||
assert client.get("/console/devices").json()[0]["name"] == "Persisted iPhone"
|
||||
|
||||
|
||||
def test_console_json_reflects_page_form_mutations(tmp_path) -> None:
|
||||
"""A device registered via the /ui/ form must be visible through /console/* JSON."""
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
client, _ = _client(tmp_path, config_store=config_store)
|
||||
|
||||
response = client.post(
|
||||
"/ui/config/devices",
|
||||
data={
|
||||
"name": "From Form",
|
||||
"driver_type": "wda",
|
||||
"server_url": "http://127.0.0.1:4723",
|
||||
"udid": "form-udid",
|
||||
"wda_local_port": "8100",
|
||||
},
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert response.status_code == 303
|
||||
|
||||
json_devices = client.get("/console/devices").json()
|
||||
assert len(json_devices) == 1
|
||||
assert json_devices[0]["name"] == "From Form"
|
||||
assert json_devices[0]["connection_info"]["udid"] == "form-udid"
|
||||
@@ -139,7 +139,11 @@ def test_authenticated_tasks_list_renders_completed_task(tmp_path) -> None:
|
||||
updated_at=datetime(2026, 7, 10, 1, tzinfo=UTC),
|
||||
completed_at=datetime(2026, 7, 10, 1, tzinfo=UTC),
|
||||
)
|
||||
metadata_store.create_task(task)
|
||||
metadata_store.create_task(
|
||||
task,
|
||||
source_task_id="cloud-task-visible",
|
||||
source_attempt=2,
|
||||
)
|
||||
|
||||
client, _ = _build_client(
|
||||
tmp_path, metadata_store=metadata_store, timeline=timeline
|
||||
@@ -148,7 +152,10 @@ def test_authenticated_tasks_list_renders_completed_task(tmp_path) -> None:
|
||||
|
||||
response = client.get("/tasks")
|
||||
assert response.status_code == 200
|
||||
assert "Executed tasks on this Host" in response.text
|
||||
assert "task-visible" in response.text
|
||||
assert "cloud-task-visible" in response.text
|
||||
assert "2" in response.text
|
||||
assert "completed" in response.text
|
||||
|
||||
|
||||
@@ -190,6 +197,69 @@ def test_authenticated_task_detail_renders_timeline_with_screenshot(tmp_path) ->
|
||||
assert "find search button" in response.text
|
||||
# Screenshot inlined as base64 data URI.
|
||||
assert "data:image/png;base64," in response.text
|
||||
assert "Before action" in response.text
|
||||
assert "After action" in response.text
|
||||
|
||||
|
||||
def test_authenticated_task_detail_renders_complete_step_evidence(tmp_path) -> None:
|
||||
from core.models import Task
|
||||
|
||||
metadata_store = TaskMetadataStore(tmp_path / "task_progress.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
task = Task(
|
||||
id="task-evidence",
|
||||
goal="inspect search",
|
||||
device_id="iphone-1",
|
||||
status="completed",
|
||||
created_at=datetime(2026, 7, 10, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 7, 10, 1, tzinfo=UTC),
|
||||
)
|
||||
metadata_store.create_task(
|
||||
task,
|
||||
source_task_id="cloud-evidence",
|
||||
source_attempt=3,
|
||||
)
|
||||
timeline.append(
|
||||
task_id=task.id,
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="inspect the current UI tree",
|
||||
tool_call={"action": "get_ui_tree", "description": "inspect UI tree"},
|
||||
result={
|
||||
"result": [
|
||||
{
|
||||
"id": "search",
|
||||
"type": "button",
|
||||
"text": "Search",
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
"confidence": 0.98,
|
||||
}
|
||||
]
|
||||
},
|
||||
before_screenshot=PNG_10X20,
|
||||
after_screenshot=PNG_10X20 + b"after",
|
||||
ocr_results=[
|
||||
{
|
||||
"text": "Search",
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
"confidence": 0.95,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
client, _ = _build_client(
|
||||
tmp_path, metadata_store=metadata_store, timeline=timeline
|
||||
)
|
||||
_login(client)
|
||||
|
||||
response = client.get(f"/tasks/{task.id}")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "cloud-evidence" in response.text
|
||||
assert "Cloud attempt" in response.text
|
||||
assert "OCR results (1)" in response.text
|
||||
assert "UI tree (1 normalized nodes)" in response.text
|
||||
assert "inspect the current UI tree" in response.text
|
||||
assert response.text.count("data:image/png;base64,") == 2
|
||||
|
||||
|
||||
def test_task_detail_404_for_unknown_task(tmp_path) -> None:
|
||||
@@ -207,4 +277,4 @@ def test_tasks_list_shows_empty_state(tmp_path) -> None:
|
||||
|
||||
response = client.get("/tasks")
|
||||
assert response.status_code == 200
|
||||
assert "No tasks recorded" in response.text
|
||||
assert "No executions recorded yet" in response.text
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
@@ -75,6 +76,57 @@ def test_step_transitions_persist_during_execution(tmp_path) -> None:
|
||||
assert [r["index"] for r in records] == [1, 2]
|
||||
|
||||
|
||||
def test_task_metadata_creation_is_idempotent_and_keeps_source_correlation(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
task = _make_task("task-source")
|
||||
|
||||
metadata_store.create_task(
|
||||
task,
|
||||
source_task_id="cloud-task-source",
|
||||
source_attempt=2,
|
||||
)
|
||||
metadata_store.create_task(task)
|
||||
|
||||
row = metadata_store.get_task(task.id)
|
||||
assert row is not None
|
||||
assert row["source_task_id"] == "cloud-task-source"
|
||||
assert row["source_attempt"] == 2
|
||||
|
||||
|
||||
def test_existing_task_database_gains_source_correlation_columns(tmp_path) -> None:
|
||||
db_path = tmp_path / "legacy.sqlite3"
|
||||
with sqlite3.connect(db_path) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
create table tasks (
|
||||
id text primary key,
|
||||
goal text not null,
|
||||
device_id text not null,
|
||||
status text not null,
|
||||
created_at text not null,
|
||||
updated_at text not null,
|
||||
completed_at text,
|
||||
failure_reason text
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
metadata_store = TaskMetadataStore(db_path)
|
||||
task = _make_task("task-migrated")
|
||||
metadata_store.create_task(
|
||||
task,
|
||||
source_task_id="cloud-task-migrated",
|
||||
source_attempt=1,
|
||||
)
|
||||
|
||||
row = metadata_store.get_task(task.id)
|
||||
assert row is not None
|
||||
assert row["source_task_id"] == "cloud-task-migrated"
|
||||
assert row["source_attempt"] == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Queryable after completion (in-memory Task discarded)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from device.manager import DeviceManager
|
||||
from core.models import Task
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from tests.fakes import FakeDriver
|
||||
|
||||
|
||||
def test_rest_start_task_and_poll_until_complete(tmp_path) -> None:
|
||||
pytest.importorskip("fastapi")
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api.rest import create_app
|
||||
|
||||
driver = FakeDriver()
|
||||
manager = DeviceManager()
|
||||
manager.register_device("iphone-1", lambda: driver)
|
||||
manager.connect("iphone-1", max_retries=1)
|
||||
store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
|
||||
class CompletingRunner:
|
||||
def run(self, task: Task) -> None:
|
||||
store.update_task(task.id, status="completed", completed=True)
|
||||
|
||||
app = create_app(
|
||||
manager=manager,
|
||||
metadata_store=store,
|
||||
task_runner=CompletingRunner(),
|
||||
)
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(
|
||||
"/agent/task",
|
||||
json={"goal": "search", "device_id": "iphone-1"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
task_id = response.json()["task_id"]
|
||||
|
||||
status_response = client.get(f"/task/{task_id}")
|
||||
assert status_response.status_code == 200
|
||||
assert status_response.json()["status"] == "completed"
|
||||
@@ -1,79 +0,0 @@
|
||||
"""Smoke test that console templates and assets ship inside the Runtime wheel.
|
||||
|
||||
Builds ``device-agent-runtime`` into a temporary directory, installs it into an
|
||||
isolated venv that cannot reach the source checkout, and asserts the packaged
|
||||
``api`` package carries the Jinja2 templates and static assets needed by
|
||||
``/ui/``. This guards against setuptools package-data regressions that would
|
||||
let the console work from an editable checkout but break from a real install.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import venv
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _run(cmd: list[str], *, cwd: Path | None = None) -> str:
|
||||
return subprocess.check_output(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_runtime_wheel_packages_console_templates_and_assets(tmp_path: Path) -> None:
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
|
||||
wheel_dir = tmp_path / "wheels"
|
||||
wheel_dir.mkdir()
|
||||
_run(
|
||||
["uv", "build", "--package", "device-agent-runtime", "--wheel", "--no-sources"],
|
||||
cwd=repo_root,
|
||||
)
|
||||
wheels = list(repo_root.glob("dist/*.whl"))
|
||||
assert wheels, "uv build did not produce a wheel"
|
||||
wheel_path = wheels[0]
|
||||
|
||||
venv_dir = tmp_path / "venv"
|
||||
venv.create(venv_dir, with_pip=True, clear=True)
|
||||
pip = str(venv_dir / "Scripts" / "pip.exe")
|
||||
if not Path(pip).exists():
|
||||
pip = str(venv_dir / "bin" / "pip")
|
||||
_run([pip, "install", str(wheel_path)], cwd=tmp_path)
|
||||
|
||||
python = str(venv_dir / "Scripts" / "python.exe")
|
||||
if not Path(python).exists():
|
||||
python = str(venv_dir / "bin" / "python")
|
||||
|
||||
probe = _run(
|
||||
[
|
||||
python,
|
||||
"-c",
|
||||
(
|
||||
"from importlib.resources import files; "
|
||||
"api_root = files('api'); "
|
||||
"templates = sorted(p.name for p in "
|
||||
"(api_root / 'templates' / 'runtime_console').iterdir()); "
|
||||
"assets = sorted(p.name for p in "
|
||||
"(api_root / 'static' / 'runtime_console').iterdir()); "
|
||||
"print(','.join(templates)); "
|
||||
"print(','.join(assets))"
|
||||
),
|
||||
],
|
||||
cwd=tmp_path,
|
||||
)
|
||||
template_names, asset_names = probe.strip().splitlines()
|
||||
assert "base.html" in template_names
|
||||
assert "dashboard.html" in template_names
|
||||
assert "config.html" in template_names
|
||||
assert "console.css" in asset_names
|
||||
assert "dashboard.js" in asset_names
|
||||
|
||||
# Clean up the build artifact so it does not leak into the working tree.
|
||||
for wheel in wheels:
|
||||
wheel.unlink()
|
||||
@@ -1,447 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.models import Task
|
||||
from device.manager import DeviceManager
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from storage.artifact_store import ArtifactStore
|
||||
from storage.device_config import DeviceConfigStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
from tests.fakes import PNG_10X20, FakeDriver
|
||||
|
||||
|
||||
def _client(tmp_path, *, manager=None, runner=None, config_store=None, timeline=None):
|
||||
pytest.importorskip("fastapi")
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api.rest import create_app
|
||||
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
app = create_app(
|
||||
manager=manager or DeviceManager(),
|
||||
metadata_store=metadata_store,
|
||||
task_runner=runner,
|
||||
device_config_store=config_store
|
||||
or DeviceConfigStore(tmp_path / "device_config.sqlite3"),
|
||||
timeline=timeline or Timeline(ArtifactStore(tmp_path / "history")),
|
||||
)
|
||||
return TestClient(app), metadata_store
|
||||
|
||||
|
||||
_TEMPLATE_NAMES = [
|
||||
"base.html",
|
||||
"dashboard.html",
|
||||
"_status_fragment.html",
|
||||
"tasks.html",
|
||||
"task_detail.html",
|
||||
"config.html",
|
||||
]
|
||||
|
||||
|
||||
# -- 5.2 Template tests ------------------------------------------------------
|
||||
|
||||
|
||||
def test_module_jinja_environment_autoescapes_html_and_xml() -> None:
|
||||
from api.console_web import _ENV
|
||||
|
||||
# select_autoescape(["html", "xml"]) returns a callable used by Jinja2.
|
||||
assert callable(_ENV.autoescape)
|
||||
assert _ENV.autoescape("foo.html") is True
|
||||
assert _ENV.autoescape("foo.xml") is True
|
||||
|
||||
|
||||
def test_every_console_template_is_known_and_loadable() -> None:
|
||||
from api.console_web import _ENV
|
||||
|
||||
for name in _TEMPLATE_NAMES:
|
||||
assert _ENV.get_template(name) is not None
|
||||
|
||||
|
||||
def test_no_template_uses_safe_filter_bypass() -> None:
|
||||
template_dir = (
|
||||
Path(__file__).resolve().parent.parent / "api" / "templates" / "runtime_console"
|
||||
)
|
||||
for path in template_dir.glob("*.html"):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
assert "| safe" not in source, f"{path.name} uses |safe bypass"
|
||||
assert "|safe" not in source, f"{path.name} uses |safe bypass"
|
||||
|
||||
|
||||
def test_dashboard_escapes_untrusted_device_name(tmp_path) -> None:
|
||||
manager = DeviceManager()
|
||||
manager.register_device(
|
||||
"dev-xss",
|
||||
lambda: FakeDriver(),
|
||||
name="<script>alert(1)</script>",
|
||||
driver_type="wda",
|
||||
)
|
||||
client, _ = _client(tmp_path, manager=manager)
|
||||
|
||||
body = client.get("/ui/").text
|
||||
assert "<script>" in body
|
||||
assert "<script>alert(1)</script>" not in body
|
||||
|
||||
|
||||
def test_tasks_list_escapes_untrusted_goal(tmp_path) -> None:
|
||||
client, metadata_store = _client(tmp_path)
|
||||
metadata_store.create_task(
|
||||
Task(
|
||||
id="task-xss",
|
||||
goal="<script>alert('xss')</script>",
|
||||
device_id="dev-1",
|
||||
)
|
||||
)
|
||||
body = client.get("/ui/tasks").text
|
||||
assert "<script>" in body
|
||||
assert "<script>alert('xss')</script>" not in body
|
||||
|
||||
|
||||
def test_task_detail_escapes_failure_reason_and_structured_output(tmp_path) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
client, metadata_store = _client(tmp_path, timeline=timeline)
|
||||
metadata_store.create_task(
|
||||
Task(
|
||||
id="task-detail",
|
||||
goal="do thing",
|
||||
device_id="dev-1",
|
||||
failure_reason="<script>alert('fail')</script>",
|
||||
)
|
||||
)
|
||||
timeline.append(
|
||||
task_id="task-detail",
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="do thing",
|
||||
tool_call={"action": "<script>", "args": {"x": 1}},
|
||||
result={"err": "</script><script>alert(1)</script>"},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
body = client.get("/ui/tasks/task-detail").text
|
||||
assert "<script>alert('fail')</script>" not in body
|
||||
assert "</script><script>" not in body
|
||||
# Structured JSON output uses tojson which escapes angle brackets.
|
||||
assert "\\u003c" in body or "<script>" in body
|
||||
|
||||
|
||||
def test_config_form_error_preserves_and_escapes_submitted_name(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
response = client.post(
|
||||
"/ui/config/devices",
|
||||
data={
|
||||
"name": "<script>alert(1)</script>",
|
||||
"driver_type": "bad-driver",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "<script>alert(1)</script>" not in response.text
|
||||
assert "<script>" in response.text
|
||||
|
||||
|
||||
# -- 5.3 TestClient page-route coverage --------------------------------------
|
||||
|
||||
|
||||
def test_root_redirects_to_ui(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
response = client.get("/", follow_redirects=False)
|
||||
assert response.status_code == 307
|
||||
assert response.headers["location"] == "/ui/"
|
||||
|
||||
|
||||
def test_dashboard_serves_html_with_empty_state(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
response = client.get("/ui/")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
assert "No devices registered" in response.text
|
||||
|
||||
|
||||
def test_dashboard_shows_populated_metrics_and_devices(tmp_path) -> None:
|
||||
manager = DeviceManager()
|
||||
manager.register_device(
|
||||
"iphone-1",
|
||||
lambda: FakeDriver(),
|
||||
name="Desk iPhone",
|
||||
driver_type="wda",
|
||||
)
|
||||
client, metadata_store = _client(tmp_path, manager=manager)
|
||||
metadata_store.create_task(
|
||||
Task(id="t-running", goal="run", device_id="iphone-1", status="running")
|
||||
)
|
||||
metadata_store.create_task(
|
||||
Task(id="t-failed", goal="fail", device_id="iphone-1", status="failed")
|
||||
)
|
||||
body = client.get("/ui/").text
|
||||
assert "Desk iPhone" in body
|
||||
assert "iphone-1" in body
|
||||
|
||||
|
||||
def test_status_fragment_endpoint_returns_html_partial(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
response = client.get("/ui/_status_fragment")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
assert "Device Status" in response.text
|
||||
|
||||
|
||||
def test_tasks_page_supports_device_and_status_filters(tmp_path) -> None:
|
||||
manager = DeviceManager()
|
||||
manager.register_device(
|
||||
"iphone-1",
|
||||
lambda: FakeDriver(),
|
||||
name="Desk",
|
||||
driver_type="wda",
|
||||
)
|
||||
client, metadata_store = _client(tmp_path, manager=manager)
|
||||
older = Task(
|
||||
id="task-old",
|
||||
goal="open settings",
|
||||
device_id="iphone-1",
|
||||
created_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
)
|
||||
newer = Task(
|
||||
id="task-new",
|
||||
goal="search",
|
||||
device_id="iphone-2",
|
||||
status="running",
|
||||
created_at=datetime(2026, 1, 2, tzinfo=UTC),
|
||||
updated_at=datetime(2026, 1, 2, tzinfo=UTC),
|
||||
)
|
||||
metadata_store.create_task(older)
|
||||
metadata_store.create_task(newer)
|
||||
|
||||
body_all = client.get("/ui/tasks").text
|
||||
assert "task-old" in body_all
|
||||
assert "task-new" in body_all
|
||||
|
||||
body_filtered = client.get("/ui/tasks?device_id=iphone-1").text
|
||||
assert "task-old" in body_filtered
|
||||
assert "task-new" not in body_filtered
|
||||
|
||||
body_status = client.get("/ui/tasks?status=running").text
|
||||
assert "task-new" in body_status
|
||||
assert "task-old" not in body_status
|
||||
|
||||
|
||||
def test_task_detail_renders_timeline_with_screenshot(tmp_path) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
client, metadata_store = _client(tmp_path, timeline=timeline)
|
||||
metadata_store.create_task(
|
||||
Task(id="task-with-timeline", goal="tap search", device_id="iphone-1")
|
||||
)
|
||||
timeline.append(
|
||||
task_id="task-with-timeline",
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="tap search",
|
||||
tool_call={
|
||||
"action": "tap",
|
||||
"description": "tap search",
|
||||
"args": {"x": 1, "y": 2},
|
||||
},
|
||||
result={"ok": True},
|
||||
before_screenshot=PNG_10X20 + b"before",
|
||||
after_screenshot=PNG_10X20 + b"after",
|
||||
ocr_results=[
|
||||
{
|
||||
"text": "Search",
|
||||
"confidence": 0.98,
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
}
|
||||
],
|
||||
)
|
||||
body = client.get("/ui/tasks/task-with-timeline").text
|
||||
before_data_uri = "data:image/png;base64," + base64.b64encode(
|
||||
PNG_10X20 + b"before"
|
||||
).decode("ascii")
|
||||
after_data_uri = "data:image/png;base64," + base64.b64encode(
|
||||
PNG_10X20 + b"after"
|
||||
).decode("ascii")
|
||||
assert before_data_uri in body
|
||||
assert after_data_uri in body
|
||||
assert "Before action" in body
|
||||
assert "After action" in body
|
||||
assert "Operation" in body
|
||||
assert "OCR results" in body
|
||||
assert "Search" in body
|
||||
assert "UI tree" not in body
|
||||
|
||||
|
||||
def test_task_detail_404_for_unknown_task(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
response = client.get("/ui/tasks/does-not-exist")
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_task_detail_renders_normalized_ui_tree_result(tmp_path) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
client, metadata_store = _client(tmp_path, timeline=timeline)
|
||||
metadata_store.create_task(
|
||||
Task(id="task-ui-tree", goal="inspect the screen", device_id="iphone-1")
|
||||
)
|
||||
timeline.append(
|
||||
task_id="task-ui-tree",
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="inspect the screen",
|
||||
tool_call={"action": "get_ui_tree", "description": "inspect UI tree"},
|
||||
result={
|
||||
"success": True,
|
||||
"result": [
|
||||
{
|
||||
"id": "ui-000",
|
||||
"type": "button",
|
||||
"text": "Search",
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
"confidence": 1.0,
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
body = client.get("/ui/tasks/task-ui-tree").text
|
||||
|
||||
assert "UI tree" in body
|
||||
assert "1 normalized nodes" in body
|
||||
assert "button" in body
|
||||
assert "Search" in body
|
||||
|
||||
|
||||
def test_config_page_lists_supported_drivers_and_current_max_steps(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
config_store.set_setting("max_steps", 5)
|
||||
runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
|
||||
client, _ = _client(tmp_path, runner=runner, config_store=config_store)
|
||||
body = client.get("/ui/config").text
|
||||
assert "wda" in body
|
||||
assert 'value="5"' in body
|
||||
|
||||
|
||||
def test_register_device_prg_redirects_and_persists(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
client, _ = _client(tmp_path, config_store=config_store)
|
||||
response = client.post(
|
||||
"/ui/config/devices",
|
||||
data={
|
||||
"name": "Desk iPhone",
|
||||
"driver_type": "wda",
|
||||
"server_url": "http://127.0.0.1:4723",
|
||||
"udid": "abc123",
|
||||
"wda_local_port": "8100",
|
||||
},
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert response.status_code == 303
|
||||
assert response.headers["location"].endswith("/ui/config")
|
||||
assert len(config_store.list()) == 1
|
||||
|
||||
|
||||
def test_register_device_rejects_bad_driver_without_partial_mutation(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
client, _ = _client(tmp_path, config_store=config_store)
|
||||
response = client.post(
|
||||
"/ui/config/devices",
|
||||
data={"name": "Bad", "driver_type": "android"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert config_store.list() == []
|
||||
assert "unsupported driver_type" in response.text
|
||||
|
||||
|
||||
def test_register_device_rejects_non_numeric_port_without_partial_mutation(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
client, _ = _client(tmp_path, config_store=config_store)
|
||||
response = client.post(
|
||||
"/ui/config/devices",
|
||||
data={
|
||||
"name": "Bad Port",
|
||||
"driver_type": "wda",
|
||||
"wda_local_port": "not-a-number",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert config_store.list() == []
|
||||
assert "wda_local_port must be a number" in response.text
|
||||
|
||||
|
||||
def test_remove_device_prg_redirects_and_removes(tmp_path) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
config_store.add(
|
||||
device_id="removable-1",
|
||||
name="To Remove",
|
||||
driver_type="wda",
|
||||
connection_info={"udid": "abc"},
|
||||
)
|
||||
client, _ = _client(tmp_path, config_store=config_store)
|
||||
response = client.post(
|
||||
"/ui/config/devices/removable-1/delete",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert response.status_code == 303
|
||||
assert config_store.list() == []
|
||||
|
||||
|
||||
def test_update_max_steps_prg_redirects_and_applies(tmp_path) -> None:
|
||||
runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
|
||||
client, _ = _client(tmp_path, runner=runner)
|
||||
response = client.post(
|
||||
"/ui/config/max-steps",
|
||||
data={"max_steps": "25"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert response.status_code == 303
|
||||
assert runner.config.max_steps == 25
|
||||
|
||||
|
||||
def test_update_max_steps_rejects_non_positive_without_partial_mutation(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
|
||||
config_store.set_setting("max_steps", 10)
|
||||
runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
|
||||
client, _ = _client(tmp_path, runner=runner, config_store=config_store)
|
||||
response = client.post("/ui/config/max-steps", data={"max_steps": "0"})
|
||||
assert response.status_code == 400
|
||||
assert runner.config.max_steps == 10
|
||||
assert "max_steps must be positive" in response.text
|
||||
|
||||
|
||||
def test_update_max_steps_rejects_non_integer(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
response = client.post("/ui/config/max-steps", data={"max_steps": "abc"})
|
||||
assert response.status_code == 400
|
||||
assert "max_steps must be an integer" in response.text
|
||||
|
||||
|
||||
def test_static_assets_are_served(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
assert client.get("/ui/assets/console.css").status_code == 200
|
||||
assert client.get("/ui/assets/dashboard.js").status_code == 200
|
||||
|
||||
|
||||
# -- 5.4 Regression: no SPA static dir, no wildcard CORS ---------------------
|
||||
|
||||
|
||||
def test_ui_works_without_runtime_console_static_dir(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.delenv("RUNTIME_CONSOLE_STATIC_DIR", raising=False)
|
||||
client, _ = _client(tmp_path)
|
||||
assert client.get("/ui/").status_code == 200
|
||||
|
||||
|
||||
def test_runtime_app_does_not_register_wildcard_cors(tmp_path) -> None:
|
||||
client, _ = _client(tmp_path)
|
||||
# A same-origin browser client must not require CORS preflight. If wildcard
|
||||
# CORS were still registered, an explicit Origin header would produce
|
||||
# access-control-allow-origin in the response; assert it is absent for an
|
||||
# arbitrary same-origin page request.
|
||||
response = client.get(
|
||||
"/ui/",
|
||||
headers={"Origin": "http://127.0.0.1:8000"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "access-control-allow-origin" not in {k.lower() for k in response.headers}
|
||||
@@ -60,7 +60,6 @@ def test_task_runner_executes_loop_and_writes_timeline(tmp_path) -> None:
|
||||
metadata = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
task = Task(goal="open app and search", device_id="iphone-1")
|
||||
metadata.create_task(task)
|
||||
|
||||
runner = TaskRunner(
|
||||
planner=planner,
|
||||
|
||||
@@ -6,6 +6,7 @@ from runtime.planner import Planner
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from skills_learning.models import FlowStep, FlowTemplateSkill, SkillMetadata
|
||||
from skills_learning.store import SkillStore
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from tests.fakes import PNG_10X20
|
||||
from workflow.models import (
|
||||
BranchStep,
|
||||
@@ -39,6 +40,14 @@ class FakeTaskRunner:
|
||||
return task
|
||||
|
||||
|
||||
class ImmediateCompletionPlanner(Planner):
|
||||
def plan(self, *, goal, scene, context):
|
||||
return []
|
||||
|
||||
def goal_reached(self, *, goal, scene, context):
|
||||
return True
|
||||
|
||||
|
||||
def _scene(text: str = "Ready") -> Scene:
|
||||
return Scene(
|
||||
width=10,
|
||||
@@ -96,6 +105,32 @@ def test_workflow_runner_linear_planned_goal_completes(tmp_path) -> None:
|
||||
assert [result.step_id for result in run.step_results] == ["first", "second"]
|
||||
|
||||
|
||||
def test_workflow_planned_goal_creates_task_metadata_without_precreation(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
|
||||
definition = WorkflowDefinition(
|
||||
name="persisted goal",
|
||||
entry_step_id="goal",
|
||||
steps=[PlannedGoalStep("goal", "inspect screen")],
|
||||
)
|
||||
runner = WorkflowRunner(
|
||||
_store(tmp_path),
|
||||
task_runner_factory=lambda: TaskRunner(
|
||||
planner=ImmediateCompletionPlanner(),
|
||||
metadata_store=metadata_store,
|
||||
observer=lambda device_id: _scene(),
|
||||
config=TaskRunnerConfig(max_steps=1),
|
||||
),
|
||||
)
|
||||
|
||||
run = runner.run(definition, "phone")
|
||||
|
||||
task_id = run.step_results[0].task_id
|
||||
assert task_id is not None
|
||||
assert metadata_store.get_task(task_id)["status"] == "completed"
|
||||
|
||||
|
||||
def test_workflow_runner_stops_before_the_next_step(tmp_path) -> None:
|
||||
stop_requested = False
|
||||
calls: list[str] = []
|
||||
|
||||
@@ -401,15 +401,11 @@ source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "anthropic" },
|
||||
{ name = "appium-python-client" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "jinja2" },
|
||||
{ name = "mcp" },
|
||||
{ name = "openai" },
|
||||
{ name = "paddleocr" },
|
||||
{ name = "paddlepaddle" },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
@@ -421,15 +417,11 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "anthropic", specifier = ">=0.69.0" },
|
||||
{ name = "appium-python-client", specifier = ">=5.1.1" },
|
||||
{ name = "fastapi", specifier = ">=0.115.0" },
|
||||
{ name = "httpx", specifier = ">=0.27.0" },
|
||||
{ name = "jinja2", specifier = ">=3.1" },
|
||||
{ name = "mcp", specifier = ">=1.27,<2" },
|
||||
{ name = "openai", specifier = ">=1.0.0" },
|
||||
{ name = "paddleocr", specifier = ">=3.0.0" },
|
||||
{ name = "paddlepaddle", specifier = ">=3.0.0" },
|
||||
{ name = "python-multipart", specifier = ">=0.0.20" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
|
||||
Reference in New Issue
Block a user