This commit is contained in:
@@ -43,6 +43,7 @@ from cloud.pool import DevicePool
|
||||
from cloud.scheduler import TaskScheduler
|
||||
from cloud.schema import require_current_schema
|
||||
from cloud.sdk.api import create_cloud_router
|
||||
from cloud.sdk.governance_api import create_governance_router
|
||||
from cloud.sdk.user_api import create_user_auth_router
|
||||
from cloud.user_auth import USER_CSRF_COOKIE, USER_SESSION_COOKIE, UserAuthService, UserAuthSettings
|
||||
from core.models import utc_now
|
||||
@@ -288,11 +289,18 @@ def create_app(
|
||||
config=control_config,
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
create_governance_router(
|
||||
repository=repository,
|
||||
auth_provider=auth_provider,
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
create_internal_router(
|
||||
pool=pool,
|
||||
auth_provider=auth_provider,
|
||||
lease_duration_seconds=control_config.lease_duration_seconds,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from cloud.internal_api.models import (
|
||||
DeviceSnapshotModel,
|
||||
HeartbeatResponse,
|
||||
HostEnrollmentResponse,
|
||||
HostTaskSubmissionResponse,
|
||||
LeaseRenewalResponse,
|
||||
TerminalResultResponse,
|
||||
)
|
||||
@@ -146,6 +147,7 @@ class HostAgentClient:
|
||||
devices: list[DeviceSnapshotModel],
|
||||
*,
|
||||
address: str | None = None,
|
||||
policy_revision: int = 0,
|
||||
) -> HeartbeatResponse:
|
||||
response = await self._request(
|
||||
"PUT",
|
||||
@@ -154,10 +156,28 @@ class HostAgentClient:
|
||||
"host_id": self.config.host_id,
|
||||
"address": address,
|
||||
"devices": [device.model_dump(mode="json") for device in devices],
|
||||
"policy_revision": policy_revision,
|
||||
},
|
||||
)
|
||||
return HeartbeatResponse.model_validate(response.json())
|
||||
|
||||
async def submit_self_task(
|
||||
self,
|
||||
*,
|
||||
goal: str,
|
||||
device_id: str | None = None,
|
||||
) -> HostTaskSubmissionResponse:
|
||||
response = await self._request(
|
||||
"POST",
|
||||
f"/internal/v1/hosts/{self.config.host_id}/tasks",
|
||||
json={
|
||||
"host_id": self.config.host_id,
|
||||
"goal": goal,
|
||||
"device_id": device_id,
|
||||
},
|
||||
)
|
||||
return HostTaskSubmissionResponse.model_validate(response.json())
|
||||
|
||||
async def claim(self) -> AssignmentModel | None:
|
||||
response = await self._request(
|
||||
"POST",
|
||||
|
||||
@@ -27,7 +27,7 @@ import httpx
|
||||
|
||||
from cloud.internal_api.models import PlannerDecisionError, PlannerDecisionResponse
|
||||
from host_agent.config import HostAgentConfig
|
||||
from runtime.tool_calling_client import ToolCallDecision, ToolCallUnavailable
|
||||
from runtime.tool_calling_client import ToolCallDecision, ToolCallUnavailable, ToolCallUsage
|
||||
from runtime.tool_specs import ToolSpec
|
||||
|
||||
|
||||
@@ -85,6 +85,22 @@ class CloudProxyToolCallingClient:
|
||||
return ToolCallDecision(
|
||||
tool_name=decoded.tool_name,
|
||||
arguments=dict(decoded.arguments),
|
||||
usage=(
|
||||
ToolCallUsage(
|
||||
input_tokens=decoded.input_tokens,
|
||||
output_tokens=decoded.output_tokens,
|
||||
total_tokens=decoded.total_tokens,
|
||||
)
|
||||
if any(
|
||||
value is not None
|
||||
for value in (
|
||||
decoded.input_tokens,
|
||||
decoded.output_tokens,
|
||||
decoded.total_tokens,
|
||||
)
|
||||
)
|
||||
else None
|
||||
),
|
||||
)
|
||||
raise ToolCallUnavailable(_error_detail(response))
|
||||
|
||||
|
||||
@@ -45,13 +45,19 @@ class HeartbeatSynchronizer:
|
||||
self._sleep = sleep
|
||||
self.status_tracker = status_tracker
|
||||
self.on_sync = on_sync
|
||||
self.policy_revision = 0
|
||||
self.policy = None
|
||||
|
||||
async def sync_once(self) -> HeartbeatResponse:
|
||||
snapshot = build_device_snapshot(self.manager)
|
||||
response = await self.client.heartbeat(
|
||||
snapshot,
|
||||
address=self.address,
|
||||
policy_revision=self.policy_revision,
|
||||
)
|
||||
self.policy_revision = response.policy_revision
|
||||
if response.policy is not None:
|
||||
self.policy = response.policy
|
||||
if self.status_tracker is not None:
|
||||
self.status_tracker.mark_heartbeat(ok=True, device_count=len(snapshot))
|
||||
if self.on_sync is not None:
|
||||
|
||||
@@ -58,7 +58,7 @@ def test_heartbeat_synchronizer_runs_at_configured_interval_until_stopped() -> N
|
||||
calls: list[list[str]] = []
|
||||
|
||||
class FakeClient:
|
||||
async def heartbeat(self, devices, *, address=None):
|
||||
async def heartbeat(self, devices, *, address=None, policy_revision=0):
|
||||
calls.append([device.device_id for device in devices])
|
||||
return HeartbeatResponse(
|
||||
host_id="host-a",
|
||||
@@ -96,7 +96,7 @@ def test_sync_once_notifies_status_tracker_and_on_sync_with_device_count() -> No
|
||||
)
|
||||
|
||||
class FakeClient:
|
||||
async def heartbeat(self, devices, *, address=None):
|
||||
async def heartbeat(self, devices, *, address=None, policy_revision=0):
|
||||
return HeartbeatResponse(
|
||||
host_id="host-a",
|
||||
accepted_devices=len(devices),
|
||||
|
||||
@@ -32,6 +32,11 @@ const canAdminPlugins = computed(
|
||||
currentUser.value?.scopes.includes("*") ||
|
||||
currentUser.value?.scopes.includes("plugins:admin"),
|
||||
);
|
||||
const canSubmitTasks = computed(
|
||||
() =>
|
||||
currentUser.value?.scopes.includes("*") ||
|
||||
currentUser.value?.scopes.includes("tasks:submit"),
|
||||
);
|
||||
const isAuthenticated = computed(() => currentUser.value !== null);
|
||||
const currentUserLabel = computed(() =>
|
||||
currentUser.value ? `${currentUser.value.display_name} (${currentUser.value.role})` : "",
|
||||
@@ -128,7 +133,7 @@ const activeComponent = computed(() => {
|
||||
</nav>
|
||||
<main class="app-main">
|
||||
<PluginsView v-if="activeView === 'plugins'" :can-admin="canAdminPlugins" />
|
||||
<component v-else :is="activeComponent" />
|
||||
<component v-else :is="activeComponent" :can-submit="canSubmitTasks" />
|
||||
</main>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
PluginRegistrationPayload,
|
||||
TaskAttempt,
|
||||
TaskListResponse,
|
||||
TaskSubmissionPayload,
|
||||
TaskStatus,
|
||||
} from "./types";
|
||||
|
||||
@@ -135,6 +136,13 @@ export function getTaskAttempts(taskId: string): Promise<TaskAttempt[]> {
|
||||
return request<TaskAttempt[]>(`/v1/tasks/${encodeURIComponent(taskId)}/attempts`);
|
||||
}
|
||||
|
||||
export function submitTask(payload: TaskSubmissionPayload): Promise<{ task_id: string }> {
|
||||
return request<{ task_id: string }>("/v1/tasks", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
}
|
||||
|
||||
export function listDevices(): Promise<DeviceRecord[]> {
|
||||
return request<DeviceRecord[]>("/v1/devices");
|
||||
}
|
||||
|
||||
@@ -14,9 +14,22 @@ export interface TaskListItem {
|
||||
assigned_host_id: string | null;
|
||||
attempt_count: number;
|
||||
failure_reason: string | null;
|
||||
target_host_id: string | null;
|
||||
target_device_id: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface TaskSubmissionPayload {
|
||||
goal?: string;
|
||||
workflow_definition_id?: string;
|
||||
constraints?: {
|
||||
driver_type?: string;
|
||||
capability_tags?: string[];
|
||||
target_host_id?: string;
|
||||
target_device_id?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TaskListResponse {
|
||||
items: TaskListItem[];
|
||||
total: number;
|
||||
|
||||
@@ -1,18 +1,25 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref, watch } from "vue";
|
||||
import { computed, onMounted, ref, watch } from "vue";
|
||||
import { LoaderCircle, RefreshCw } from "@lucide/vue";
|
||||
import {
|
||||
CloudApiError,
|
||||
getTaskAttempts,
|
||||
listTasks,
|
||||
listDevices,
|
||||
listHosts,
|
||||
submitTask,
|
||||
} from "../api";
|
||||
import type {
|
||||
TaskAttempt,
|
||||
TaskListItem,
|
||||
TaskListResponse,
|
||||
TaskStatus,
|
||||
DeviceRecord,
|
||||
HostRecord,
|
||||
} from "../types";
|
||||
|
||||
const props = defineProps<{ canSubmit: boolean }>();
|
||||
|
||||
const STATUSES: TaskStatus[] = [
|
||||
"queued",
|
||||
"assigned",
|
||||
@@ -31,16 +38,36 @@ const selectedTask = ref<TaskListItem | null>(null);
|
||||
const attempts = ref<TaskAttempt[]>([]);
|
||||
const attemptsLoading = ref(false);
|
||||
const attemptsError = ref("");
|
||||
const composerOpen = ref(false);
|
||||
const submitGoal = ref("");
|
||||
const submitWorkflow = ref("");
|
||||
const submitHostId = ref("");
|
||||
const submitDeviceId = ref("");
|
||||
const submitDriverType = ref("");
|
||||
const submitCapabilityTags = ref("");
|
||||
const submitting = ref(false);
|
||||
const hosts = ref<HostRecord[]>([]);
|
||||
const devices = ref<DeviceRecord[]>([]);
|
||||
const availableDevices = computed(() =>
|
||||
devices.value.filter((device) => device.host_id === submitHostId.value),
|
||||
);
|
||||
|
||||
async function refresh() {
|
||||
loading.value = true;
|
||||
errorMessage.value = "";
|
||||
try {
|
||||
result.value = await listTasks({
|
||||
const [tasks, loadedHosts, loadedDevices] = await Promise.all([
|
||||
listTasks({
|
||||
status: statusFilter.value === "" ? undefined : statusFilter.value,
|
||||
limit: pageSize.value,
|
||||
offset: offset.value,
|
||||
});
|
||||
}),
|
||||
listHosts(),
|
||||
listDevices(),
|
||||
]);
|
||||
result.value = tasks;
|
||||
hosts.value = loadedHosts;
|
||||
devices.value = loadedDevices;
|
||||
if (selectedTask.value) {
|
||||
const stillPresent = result.value.items.find(
|
||||
(item) => item.id === selectedTask.value?.id,
|
||||
@@ -57,6 +84,47 @@ async function refresh() {
|
||||
}
|
||||
}
|
||||
|
||||
async function createTask() {
|
||||
const goal = submitGoal.value.trim();
|
||||
const workflow = submitWorkflow.value.trim();
|
||||
if (!goal && !workflow) {
|
||||
errorMessage.value = "provide a goal or workflow id";
|
||||
return;
|
||||
}
|
||||
submitting.value = true;
|
||||
errorMessage.value = "";
|
||||
try {
|
||||
const hostId = submitHostId.value || undefined;
|
||||
const deviceId = submitDeviceId.value || undefined;
|
||||
await submitTask({
|
||||
goal: goal || undefined,
|
||||
workflow_definition_id: workflow || undefined,
|
||||
constraints: {
|
||||
driver_type: submitDriverType.value || undefined,
|
||||
capability_tags: submitCapabilityTags.value
|
||||
.split(",")
|
||||
.map((tag) => tag.trim())
|
||||
.filter(Boolean),
|
||||
target_host_id: hostId,
|
||||
target_device_id: deviceId,
|
||||
},
|
||||
});
|
||||
composerOpen.value = false;
|
||||
submitGoal.value = "";
|
||||
submitWorkflow.value = "";
|
||||
submitHostId.value = "";
|
||||
submitDeviceId.value = "";
|
||||
submitDriverType.value = "";
|
||||
submitCapabilityTags.value = "";
|
||||
offset.value = 0;
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
handleError(err, "failed to create task");
|
||||
} finally {
|
||||
submitting.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function selectTask(task: TaskListItem) {
|
||||
selectedTask.value = task;
|
||||
attempts.value = [];
|
||||
@@ -113,6 +181,11 @@ watch(pageSize, () => {
|
||||
refresh();
|
||||
});
|
||||
watch(offset, refresh);
|
||||
watch(submitHostId, () => {
|
||||
if (!availableDevices.value.some((device) => device.device_id === submitDeviceId.value)) {
|
||||
submitDeviceId.value = "";
|
||||
}
|
||||
});
|
||||
|
||||
onMounted(refresh);
|
||||
|
||||
@@ -145,6 +218,9 @@ function formatTerminalResult(attempt: TaskAttempt): string {
|
||||
<div>
|
||||
<div class="toolbar">
|
||||
<h2>Tasks</h2>
|
||||
<button v-if="props.canSubmit" @click="composerOpen = !composerOpen">
|
||||
{{ composerOpen ? "Cancel create" : "Create task" }}
|
||||
</button>
|
||||
<label>
|
||||
Status
|
||||
<select v-model="statusFilter">
|
||||
@@ -172,6 +248,29 @@ function formatTerminalResult(attempt: TaskAttempt): string {
|
||||
|
||||
<div v-if="errorMessage" class="notice error">{{ errorMessage }}</div>
|
||||
|
||||
<form v-if="composerOpen" class="panel" @submit.prevent="createTask">
|
||||
<h3>Create task</h3>
|
||||
<label>Goal <input v-model="submitGoal" placeholder="Describe the device task" /></label>
|
||||
<label>Workflow ID <input v-model="submitWorkflow" placeholder="Optional workflow reference" /></label>
|
||||
<label>
|
||||
Target Host
|
||||
<select v-model="submitHostId">
|
||||
<option value="">Any eligible host</option>
|
||||
<option v-for="host in hosts" :key="host.host_id" :value="host.host_id">{{ host.host_id }}</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Target Device
|
||||
<select v-model="submitDeviceId" :disabled="!submitHostId">
|
||||
<option value="">Any eligible device on selected host</option>
|
||||
<option v-for="device in availableDevices" :key="device.device_id" :value="device.device_id">{{ device.device_id }}</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>Driver type <input v-model="submitDriverType" placeholder="Optional, e.g. wda" /></label>
|
||||
<label>Capability tags <input v-model="submitCapabilityTags" placeholder="Optional, comma separated" /></label>
|
||||
<div class="actions"><button type="submit" :disabled="submitting">{{ submitting ? "Creating…" : "Create" }}</button></div>
|
||||
</form>
|
||||
|
||||
<div class="panel" v-if="!selectedTask">
|
||||
<table v-if="result && result.items.length">
|
||||
<thead>
|
||||
@@ -212,6 +311,9 @@ function formatTerminalResult(attempt: TaskAttempt): string {
|
||||
{{ task.assigned_device_id }}
|
||||
<span class="dim">on {{ task.assigned_host_id }}</span>
|
||||
</div>
|
||||
<div v-else-if="task.target_host_id" class="dim">
|
||||
target: {{ task.target_host_id }}<span v-if="task.target_device_id"> / {{ task.target_device_id }}</span>
|
||||
</div>
|
||||
<div v-else class="dim">unassigned</div>
|
||||
</td>
|
||||
<td>{{ task.attempt_count }}</td>
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
schema: spec-driven
|
||||
created: 2026-07-13
|
||||
@@ -0,0 +1,212 @@
|
||||
## Context
|
||||
|
||||
The Cloud Control Plane already persists scheduled tasks, pooled devices,
|
||||
enrolled Hosts, user accounts, and user sessions. Public task submission has
|
||||
only driver and capability-tag constraints, while the Console is primarily a
|
||||
read dashboard. The Host Agent uses only outbound internal routes and can
|
||||
heartbeat, claim, renew, and report work; it cannot submit a task. No durable
|
||||
policy or AI usage ledger exists.
|
||||
|
||||
`cloud-planner-proxy` is the prerequisite for enforceable AI budgets. It
|
||||
places every planner call from a Host using `AI_PLANNER_TRANSPORT=cloud` behind
|
||||
an authenticated Cloud endpoint and leaves direct provider calls supported.
|
||||
This change is deliberately scheduled after that change: a direct provider
|
||||
call cannot be authoritatively measured or stopped by the Cloud.
|
||||
|
||||
The Cloud Platform remains the durable application layer. Public/internal
|
||||
HTTP adapters, the Vue Console, and the Host Agent are outer adapters; no
|
||||
HTTP, Cloud, or LLM dependency is introduced into `core`, `driver`, `device`,
|
||||
or `tools`.
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals:**
|
||||
|
||||
- Let authorized Console users create goal or workflow tasks for an explicit
|
||||
Host and optionally an explicit Device, without a fallback target.
|
||||
- Make human task-submission permissions and target restrictions durable,
|
||||
auditable, and API-enforced.
|
||||
- Let an authenticated Host receive its effective versioned policy and create
|
||||
goal tasks that are irrevocably constrained to itself.
|
||||
- Account for provider-reported token use and enforce hard per-Host budgets
|
||||
for Cloud-proxied planner calls, including concurrent requests.
|
||||
- Display effective restrictions, budget status, and non-secret usage in the
|
||||
Console.
|
||||
|
||||
**Non-Goals:**
|
||||
|
||||
- Custom roles, tenant isolation, task-level read ACLs, invitations, or an
|
||||
external identity provider. Existing fixed roles remain the coarse access
|
||||
control layer.
|
||||
- General remote Host control, inbound connections, or allowing one Host to
|
||||
enqueue work for another Host.
|
||||
- Charging, invoice generation, provider price catalogues, prompt/screenshot
|
||||
retention, or an attempt to hard-limit direct-to-provider planner calls.
|
||||
- Cloud workflow-definition distribution. Host-self submission is goal-only
|
||||
because a workflow definition is still stored locally on each Host.
|
||||
|
||||
## Decisions
|
||||
|
||||
### D1: Use explicit target selectors inside task constraints
|
||||
|
||||
`TaskConstraints` gains an optional target object with `host_id` and optional
|
||||
`device_id`. A device target is invalid without its Host; a Host-only target
|
||||
means any eligible device on that Host; an absent target preserves broad pool
|
||||
scheduling. The existing serialized constraints field stores this additive
|
||||
shape, while API status/list responses expose it directly.
|
||||
|
||||
The scheduler applies target filtering before the named assignment strategy,
|
||||
in addition to driver type and capability tags. A missing, stale, busy, or
|
||||
temporarily incompatible selected device leaves the task queued. It never
|
||||
falls back to another target. The submission service validates a named Host
|
||||
and Device against the known pool/enrollment ownership at creation time.
|
||||
|
||||
Using only a globally unique `device_id` was rejected because repository and
|
||||
pool ownership are Host/device pairs; retaining the Host in the target makes
|
||||
authorization and audit unambiguous. Duplicating target columns outside the
|
||||
existing constraints payload was rejected for v1 because matching is already
|
||||
performed after task retrieval and there is no target-indexed query path.
|
||||
|
||||
### D2: Compose fixed role scopes with a separate restrictive user policy
|
||||
|
||||
Keep `viewer`, `operator`, and `admin` roles and their existing scope mapping.
|
||||
Add an optional `UserSubmissionPolicy` keyed by user id with a version,
|
||||
`submission_enabled`, and tri-state Host/Device allow-lists (`null` means
|
||||
unrestricted; an empty stored list means no permitted targets). A restrictive
|
||||
policy requires an explicit target within its allow-list, preventing an
|
||||
un-targeted task from escaping a user's permitted Hosts.
|
||||
|
||||
Public task submission first requires `tasks:submit`, then evaluates the
|
||||
authenticated human principal's policy. The same policy applies to an
|
||||
administrator's ordinary task submission; an administrator can change policy
|
||||
through governance administration but does not silently bypass it. Policies
|
||||
do not change task-read visibility in this increment.
|
||||
|
||||
Storing arbitrary policy JSON was rejected: it makes safe validation,
|
||||
migration, and Console editing ambiguous. Adding a custom role/permission
|
||||
model was rejected because fixed roles plus a narrow restrictive policy cover
|
||||
the requested near-term controls without lockout complexity.
|
||||
|
||||
### D3: Store versioned Host policies and synchronize them in heartbeat responses
|
||||
|
||||
`HostGovernancePolicy` is keyed by Host id and has a monotonic revision. V1
|
||||
contains an optional maximum active-task count and an optional UTC-day token
|
||||
budget. The Cloud scheduler enforces active-task limits; the budget is
|
||||
enforced only by the planner proxy. No policy record means no new limit,
|
||||
preserving existing deployments.
|
||||
|
||||
Heartbeat requests carry the last policy revision applied by the Host. A
|
||||
heartbeat response includes the effective complete policy when the revision is
|
||||
different, otherwise only confirms its current revision. The Host persists a
|
||||
safe cached policy snapshot for local status visibility, but the Cloud remains
|
||||
the authority for task assignment and AI spend. This pull-on-existing-
|
||||
heartbeat design keeps the outbound-only Host model and avoids a parallel
|
||||
long-poll channel.
|
||||
|
||||
Pushing policy through a new inbound Host listener was rejected because it
|
||||
breaks NAT deployments. Requiring a policy fetch before every planner call
|
||||
was rejected because the proxy already performs the authoritative budget check
|
||||
and would add avoidable latency.
|
||||
|
||||
### D4: Meter Cloud-proxy usage with reservation then settlement
|
||||
|
||||
After `cloud-planner-proxy` is complete, provider clients return a
|
||||
non-secret usage object together with the existing tool-call decision. It
|
||||
contains provider/model identity and provider-reported input, output, and
|
||||
total token counts when available; `AIPlanner` continues to consume only the
|
||||
decision.
|
||||
|
||||
Before the proxy invokes a provider for a Host with a token budget, it atomically
|
||||
creates a budget reservation for a conservative per-decision ceiling within the
|
||||
Host's UTC-day bucket. The Cloud configuration also applies a matching
|
||||
provider output ceiling. If used plus reserved tokens would exceed the
|
||||
budget, the endpoint rejects before invoking the provider. On a provider
|
||||
response, the reservation is settled to actual reported usage and an immutable
|
||||
usage event is recorded. If the transport outcome leaves usage unknown, the
|
||||
reservation remains until a bounded expiry cleanup rather than being released
|
||||
optimistically.
|
||||
|
||||
The proxy request carries Host execution context (task id, attempt, and lease
|
||||
when one exists). A Host-Agent-local context adapter supplies that metadata to
|
||||
the cloud client without changing `AIPlanner` or introducing Host imports into
|
||||
`runtime`. Locally initiated/non-assignment calls record Host-level usage with
|
||||
the assignment fields absent.
|
||||
|
||||
Post-hoc usage-only accounting was rejected because concurrent calls can
|
||||
overspend a limit before aggregation. Client-side budget checks were rejected
|
||||
because the Host is not the trust boundary. Direct transport is represented
|
||||
as `unmetered`, not as zero usage or a compliant hard-budget path.
|
||||
|
||||
### D5: Give Hosts a narrow self-submission operation
|
||||
|
||||
Add `POST /internal/v1/hosts/{host_id}/tasks` under existing host-scoped
|
||||
authentication. The route requires the authenticated Host to equal the URL
|
||||
Host, accepts a goal and optional local Device target, and constructs the
|
||||
Cloud-side target itself. A supplied Device must be enrolled/owned by that
|
||||
Host. Host policy can disable self-submission; human user policies do not
|
||||
apply because there is no human principal.
|
||||
|
||||
The endpoint accepts no workflow reference and no arbitrary Host target.
|
||||
Giving an enrolled Host the public `tasks:submit` scope was rejected because
|
||||
it would allow cross-Host task submission and blur service credentials with
|
||||
human authorization.
|
||||
|
||||
### D6: Keep governance administration explicit in the Console and SDK
|
||||
|
||||
Add public, scope-protected governance routes for reading/updating user
|
||||
submission policy, Host policy, and non-secret Host AI usage summaries/events.
|
||||
`governance:read` and `governance:admin` are new scopes; fixed user roles need
|
||||
no mapping change because administrators retain `*`, while scoped bearer
|
||||
automation can receive just these scopes. Console task creation renders only
|
||||
when `tasks:submit` is present. Policy and usage administration render only
|
||||
for `governance:admin`; backend scope and policy checks are authoritative.
|
||||
|
||||
Every policy change records a safe audit event with actor, target, old/new
|
||||
revision, and non-secret values. The Console clears any password inputs from
|
||||
the existing Users flow and never displays provider credentials, prompts,
|
||||
screenshots, cookies, or lease secrets.
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
- [Planner proxy is incomplete or a Host remains on direct transport] →
|
||||
Token limits are not activated for that Host; the Console labels it
|
||||
unmetered and the implementation gate prevents treating it as compliant.
|
||||
- [Provider usage is unavailable after a transport failure] → Retain the
|
||||
conservative reservation until expiry; this favors cost safety over
|
||||
temporary under-utilization.
|
||||
- [Policy migration or stale Console state changes a user's permitted target]
|
||||
→ Enforce against current Cloud policy at every submission and return a
|
||||
clear authorization failure; UI state is only advisory.
|
||||
- [Targeted Hosts/devices are offline] → Keep the task queued and visible as
|
||||
targeted rather than rerouting it; operators can edit or cancel only when a
|
||||
later lifecycle capability supplies those operations.
|
||||
- [Policy tables increase schema and operational complexity] → Use one normal
|
||||
forward migration, transactional repository operations, and the existing
|
||||
SQLite/PostgreSQL contract.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
1. Do not start implementation until `cloud-planner-proxy` has completed its
|
||||
tasks, passed strict validation and its runtime tests, and the active
|
||||
Cloud Console user-authentication change has been reconciled and verified.
|
||||
2. Add the forward schema migration for policies, audit records, usage events,
|
||||
and reservations. Existing tasks remain valid because no target and no
|
||||
policy continue to mean unrestricted behavior.
|
||||
3. Deploy the Cloud API with policy/target routes and scheduler checks, then
|
||||
deploy compatible Hosts. Hosts learn policies on their next heartbeat;
|
||||
older Hosts continue normal assignment but cannot self-submit.
|
||||
4. Enable Cloud planner transport on a pilot Host, configure a budget, and
|
||||
verify proxy reservations and settlement before assigning budgets across
|
||||
the fleet. Direct Hosts remain visibly unmetered.
|
||||
5. Deploy the Console after the API. Rollback removes policy assignments or
|
||||
returns affected Hosts to direct transport only when local provider
|
||||
credentials are available; preserve governance and usage rows for a
|
||||
forward fix rather than destructively downgrading production data.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- V1 uses UTC-day token budgets. A later change can add calendar-month or
|
||||
rolling windows without weakening the reservation/settlement contract.
|
||||
- Token cost is intentionally excluded because provider pricing and cached
|
||||
token semantics vary; usage events retain provider/model fields for a later
|
||||
pricing layer.
|
||||
@@ -0,0 +1,79 @@
|
||||
## Why
|
||||
|
||||
The Cloud Console can currently inspect distributed work but cannot direct
|
||||
that work to a selected Host or Device, govern who may submit it, or explain
|
||||
and control the AI spend generated by each Host. Now that
|
||||
`cloud-planner-proxy` is planned to centralize the LLM call path, the next
|
||||
control-plane increment can make those operational decisions enforceable at
|
||||
the Cloud boundary instead of trusting every edge process to self-govern.
|
||||
|
||||
## What Changes
|
||||
|
||||
- Add Console task creation for authorized operators, with explicit target
|
||||
selectors for a Host and optionally one of that Host's Devices. A selected
|
||||
target is a hard scheduling constraint and is never silently rerouted to a
|
||||
different Host or Device.
|
||||
- Add persisted Cloud governance policies: user submission restrictions
|
||||
(submission enablement and allowed Host/Device targets) and versioned
|
||||
per-Host operational/AI-budget policies. The API remains authoritative;
|
||||
Console visibility is not an authorization boundary.
|
||||
- Let a Host retrieve its current policy through the authenticated outbound
|
||||
protocol, with version-aware heartbeat synchronization and a safe cached
|
||||
representation for local visibility.
|
||||
- Add a Host-scoped internal task-submission operation. An authenticated
|
||||
Host can create a task only for itself and, when specified, one of its own
|
||||
enrolled Devices; it never receives general public `tasks:submit` power.
|
||||
- After `cloud-planner-proxy` is implemented and a Host uses its `cloud`
|
||||
planner transport, record provider-reported token usage per Host/task/
|
||||
attempt and enforce configured token budgets with atomic reservation and
|
||||
settlement. Direct-to-provider Hosts remain explicitly unmetered and
|
||||
cannot be represented as hard-budget-enforced.
|
||||
- Extend the Console with task-creation, user-policy, Host-policy, and
|
||||
per-Host AI-usage/budget views. Existing user lifecycle management remains
|
||||
role-gated and gains policy editing rather than a new custom-role system.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
|
||||
- `cloud-governance`: Durable, versioned user and Host policies that restrict
|
||||
task submission/targets and define Host operational and AI-budget limits.
|
||||
- `host-scoped-task-submission`: Authenticated internal operation that lets a
|
||||
Host submit tasks constrained to itself.
|
||||
- `cloud-ai-usage-governance`: Durable AI usage metering, budget reservations,
|
||||
settlement, and non-secret reporting for Cloud-proxied planner calls.
|
||||
|
||||
### Modified Capabilities
|
||||
|
||||
- `cloud-console-ui`: add governed task creation and operational governance
|
||||
views to the existing Cloud Console experience. This capability is still
|
||||
defined by active Console changes and will be reconciled before archive.
|
||||
- `platform-sdk`: extend public task submission/status and `CloudClient`
|
||||
parity with explicit task targets and safe governance-facing responses.
|
||||
- `task-scheduler`: make explicit Host/Device target selectors part of task
|
||||
constraints and enforce them during matching.
|
||||
- `host-agent-protocol`: return versioned Host policy through the outbound
|
||||
protocol and add Host-self task submission while retaining host binding.
|
||||
- `cloud-planner-proxy`: enforce Host budget reservations and emit
|
||||
provider-reported usage only after the proxy change is implemented. This
|
||||
capability is currently an active, unarchived change.
|
||||
- `agent-runtime`: carry provider-reported non-secret token usage alongside a
|
||||
tool-call decision so the Cloud proxy can meter the call without changing
|
||||
`AIPlanner` decision semantics. This capability is also active and
|
||||
unarchived.
|
||||
|
||||
## Impact
|
||||
|
||||
- `packages/cloud-platform/cloud`: governance and usage domain models,
|
||||
repository port/SQLAlchemy implementation, Alembic migration, scheduler
|
||||
matching, public/internal Pydantic contracts, policy-aware authorization,
|
||||
and proxy metering.
|
||||
- `apps/cloud-api`: router composition and configuration only; all durable
|
||||
policy and accounting behavior remains in the Cloud Platform package.
|
||||
- `apps/device-host-agent`: policy-aware client/heartbeat state and
|
||||
self-targeted task submission; no inbound Cloud connection is introduced.
|
||||
- `cloud-console/`: task composer plus user/Host policy and AI-usage views.
|
||||
- Dependencies and rollout: implementation is blocked until
|
||||
`cloud-planner-proxy` is completed, its migration is deployed, and affected
|
||||
Hosts are using `AI_PLANNER_TRANSPORT=cloud`; it also follows completion of
|
||||
the active Cloud Console user-authentication work.
|
||||
@@ -0,0 +1,50 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Pluggable dual-provider tool-calling abstraction
|
||||
The system SHALL support at least two interchangeable LLM providers
|
||||
(Anthropic native tool use and OpenAI function calling) for the AI Planner's
|
||||
decision calls, selectable via configuration, with both providers constrained
|
||||
to return exactly one tool call per request. Independently of provider
|
||||
selection, the system SHALL support at least two transports for making that
|
||||
decision call -- direct-to-provider (the tool-calling client calls the
|
||||
provider's SDK itself, using locally configured credentials) and cloud-proxy
|
||||
(the tool-calling client calls the Cloud Control Plane's planner-decision
|
||||
endpoint, which calls the provider using cloud-held credentials) -- selectable
|
||||
via configuration without requiring any change to `AIPlanner`'s own decision
|
||||
logic. A provider decision result SHALL also carry available non-secret
|
||||
provider token-usage metadata so outer Cloud adapters can meter calls without
|
||||
changing that decision logic.
|
||||
|
||||
#### Scenario: Provider selected via configuration
|
||||
- **WHEN** the AI Planner is configured with a given provider identifier
|
||||
- **THEN** it constructs and uses the tool-calling client for that provider
|
||||
without requiring any change to `AIPlanner`'s own decision logic
|
||||
|
||||
#### Scenario: Provider response resolves to a single decision
|
||||
- **WHEN** either supported provider returns a response to a tool-calling
|
||||
request
|
||||
- **THEN** the response is parsed into exactly one tool name and one arguments
|
||||
object, regardless of which provider produced it
|
||||
|
||||
#### Scenario: Transport selected via configuration
|
||||
- **WHEN** the Host Agent is configured with a given transport (direct or
|
||||
cloud-proxy)
|
||||
- **THEN** `AIPlanner` is constructed with the tool-calling client for that
|
||||
transport, and its own decision logic is unchanged regardless of which
|
||||
transport is in effect
|
||||
|
||||
#### Scenario: Direct transport remains available and default
|
||||
- **WHEN** no transport is explicitly configured
|
||||
- **THEN** the AI Planner uses the direct-to-provider transport, matching its
|
||||
behavior before the cloud-proxy transport existed
|
||||
|
||||
#### Scenario: Cloud-proxy transport resolves a decision without a local provider client
|
||||
- **WHEN** the Host Agent is configured with the cloud-proxy transport
|
||||
- **THEN** its tool-calling client sends the decision request to the Cloud
|
||||
Control Plane's planner-decision endpoint instead of constructing a local
|
||||
Anthropic or OpenAI SDK client
|
||||
|
||||
#### Scenario: Provider reports token usage
|
||||
- **WHEN** a provider response includes input or output token counts
|
||||
- **THEN** the tool-calling result makes those non-secret counts available to
|
||||
its outer adapter while `AIPlanner` selects the same single decision
|
||||
@@ -0,0 +1,52 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Cloud-proxied planner calls produce durable non-secret usage events
|
||||
After the Cloud planner proxy resolves a provider call, the system SHALL record
|
||||
a durable usage event with Host, provider/model, provider-reported token
|
||||
counts, request outcome, and known task-attempt context, while excluding raw
|
||||
prompts, screenshots, provider credentials, cookies, and lease secrets.
|
||||
|
||||
#### Scenario: Provider returns token usage
|
||||
- **WHEN** a Cloud-proxied planner call completes with provider-reported token
|
||||
counts
|
||||
- **THEN** the system stores those counts against the authenticated Host and
|
||||
returns the planner decision without exposing secret request content
|
||||
|
||||
#### Scenario: Assignment context is available
|
||||
- **WHEN** the Host makes a proxied planner call while executing a claimed
|
||||
assignment
|
||||
- **THEN** the recorded event includes that task id and attempt identifier
|
||||
|
||||
### Requirement: Host token budgets use atomic reservation and settlement
|
||||
The Cloud planner proxy SHALL atomically reserve a conservative per-call token
|
||||
amount before a provider invocation for every Host with an effective AI token
|
||||
budget using Cloud planner transport, SHALL reject over-budget calls before
|
||||
invoking the provider, and SHALL settle the reservation to provider-reported
|
||||
usage when available.
|
||||
|
||||
#### Scenario: Remaining budget permits a call
|
||||
- **WHEN** used tokens plus active reservations and the next conservative
|
||||
reservation are within the Host's UTC-day budget
|
||||
- **THEN** the proxy reserves budget, invokes the provider, and settles the
|
||||
reservation after the provider response
|
||||
|
||||
#### Scenario: Remaining budget is insufficient
|
||||
- **WHEN** the next reservation would exceed the Host's effective budget
|
||||
- **THEN** the proxy rejects the planner call without invoking the provider
|
||||
|
||||
#### Scenario: Provider outcome has unknown usage
|
||||
- **WHEN** a transport failure prevents the proxy from determining actual
|
||||
provider usage after reservation
|
||||
- **THEN** the reservation remains active until bounded expiry cleanup rather
|
||||
than being released optimistically
|
||||
|
||||
### Requirement: Direct planner transport is explicitly unmetered
|
||||
The system SHALL represent a Host using direct-to-provider planner transport as
|
||||
unmetered for Cloud token accounting and SHALL not report it as complying with
|
||||
a Cloud-enforced token budget.
|
||||
|
||||
#### Scenario: Console inspects a direct-transport Host
|
||||
- **WHEN** an authorized operator views AI usage for a Host not using Cloud
|
||||
planner transport
|
||||
- **THEN** the system reports the Host as unmetered rather than zero usage or
|
||||
hard-budget-enforced
|
||||
@@ -0,0 +1,50 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Console composes explicitly targeted tasks
|
||||
The Console SHALL provide a task-creation flow to principals with
|
||||
`tasks:submit`, exposing goal/workflow input, Host selection, Device selection
|
||||
scoped to the selected Host, and visible driver/capability constraints.
|
||||
|
||||
#### Scenario: Operator creates a Device-targeted task
|
||||
- **WHEN** an authorized operator selects a Host and one of its Devices and
|
||||
submits a valid task
|
||||
- **THEN** the Console sends the explicit target to the public task API and
|
||||
displays the resulting queued task with that target
|
||||
|
||||
#### Scenario: Restricted operator has no permitted target
|
||||
- **WHEN** the Cloud API reports that the operator's policy disallows the
|
||||
selected target or targetless submission
|
||||
- **THEN** the Console shows the authorization error and does not imply that
|
||||
the task was accepted or rerouted
|
||||
|
||||
### Requirement: Console administers governance policy within effective scopes
|
||||
The Console SHALL expose user-submission and Host-policy views only to a
|
||||
principal with `governance:admin`, and SHALL display effective non-secret
|
||||
policy revisions and mutations without treating hidden controls as security.
|
||||
|
||||
#### Scenario: Administrator changes a Host budget
|
||||
- **WHEN** a governance administrator saves a valid Host policy containing an
|
||||
AI budget or active-task limit
|
||||
- **THEN** the Console displays the returned newer policy revision and its
|
||||
effective values
|
||||
|
||||
#### Scenario: Non-governance user opens the Console
|
||||
- **WHEN** the current principal lacks `governance:admin`
|
||||
- **THEN** the Console hides governance administration navigation while the
|
||||
backend remains responsible for rejecting unauthorized requests
|
||||
|
||||
### Requirement: Console reports Host AI budget state without sensitive prompts
|
||||
The Console SHALL render per-Host budget state and non-secret token usage for
|
||||
authorized governance users, including an explicit unmetered state for Hosts
|
||||
using direct planner transport.
|
||||
|
||||
#### Scenario: Administrator views a Cloud-proxied Host
|
||||
- **WHEN** a governance administrator selects a Host with recorded proxy usage
|
||||
- **THEN** the Console displays configured budget, used/reserved/remaining
|
||||
tokens, provider/model, and non-secret event metadata
|
||||
|
||||
#### Scenario: Administrator views a direct-transport Host
|
||||
- **WHEN** a governance administrator selects a Host that does not use Cloud
|
||||
planner transport
|
||||
- **THEN** the Console labels the Host unmetered and does not display zero as
|
||||
a budget-compliant usage value
|
||||
@@ -0,0 +1,53 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: User task-submission policy is durable and authoritative
|
||||
The system SHALL persist an optional versioned submission policy for each
|
||||
human user, containing submission enablement and explicit Host/Device target
|
||||
allow-lists, and SHALL enforce that policy after scope authentication on every
|
||||
public task-submission request.
|
||||
|
||||
#### Scenario: Restricted user submits to an allowed Device
|
||||
- **WHEN** a principal with `tasks:submit` has a policy allowing a Host/Device
|
||||
target and submits a task for that exact target
|
||||
- **THEN** the system accepts the task subject to normal validation and records
|
||||
the authenticated principal as its submitter
|
||||
|
||||
#### Scenario: Restricted user omits or exceeds a target allow-list
|
||||
- **WHEN** a principal with a restrictive policy submits without a target or
|
||||
names a Host/Device outside its allow-list
|
||||
- **THEN** the system rejects the request before creating a task
|
||||
|
||||
#### Scenario: Submission is disabled
|
||||
- **WHEN** a principal with `tasks:submit` has a policy with submission
|
||||
disabled
|
||||
- **THEN** the system rejects task submission without changing the queue
|
||||
|
||||
### Requirement: Host governance policy is versioned and enforceable
|
||||
The system SHALL persist a monotonic revisioned policy for each Host with an
|
||||
optional active-task limit and optional UTC-day AI token budget, and SHALL
|
||||
enforce active-task limits during Cloud scheduling.
|
||||
|
||||
#### Scenario: Host active-task limit is reached
|
||||
- **WHEN** a Host already has its configured maximum number of assigned or
|
||||
dispatched attempts and another queued task targets that Host
|
||||
- **THEN** the scheduler leaves the later task queued until capacity becomes
|
||||
available
|
||||
|
||||
#### Scenario: Host policy is updated
|
||||
- **WHEN** an authorized governance administrator updates a Host policy
|
||||
- **THEN** the system stores a strictly newer revision and later Cloud
|
||||
authorization, scheduling, and proxy operations use the new policy
|
||||
|
||||
### Requirement: Governance mutations and reads use dedicated scopes
|
||||
The system SHALL protect governance policy and AI-usage operations with
|
||||
`governance:read` or `governance:admin` as appropriate, and SHALL record safe
|
||||
audits for policy mutations.
|
||||
|
||||
#### Scenario: Administrator changes a policy
|
||||
- **WHEN** a principal with `governance:admin` changes a user or Host policy
|
||||
- **THEN** the system records actor, target, old/new revision, timestamp, and
|
||||
non-secret policy metadata without recording credentials or request secrets
|
||||
|
||||
#### Scenario: Non-governance principal attempts a policy mutation
|
||||
- **WHEN** a principal lacking `governance:admin` updates a governance policy
|
||||
- **THEN** the system rejects the request before changing durable policy state
|
||||
@@ -0,0 +1,38 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Planner proxy enforces effective Host token budgets before provider invocation
|
||||
The Cloud planner-decision endpoint SHALL atomically reserve a conservative
|
||||
bounded token amount before invoking its provider for an authenticated Host
|
||||
using Cloud planner transport with an effective token budget, and SHALL reject
|
||||
the request without invoking the provider when the reservation would exceed
|
||||
the remaining budget.
|
||||
|
||||
#### Scenario: Budget permits a planner decision
|
||||
- **WHEN** the Host has sufficient budget after used and reserved tokens are
|
||||
considered
|
||||
- **THEN** the endpoint creates a reservation and invokes the configured
|
||||
provider exactly once
|
||||
|
||||
#### Scenario: Budget is exhausted
|
||||
- **WHEN** a next planner decision would exceed the Host's effective budget
|
||||
- **THEN** the endpoint returns a structured failure without invoking the
|
||||
configured provider
|
||||
|
||||
### Requirement: Planner proxy settles provider-reported token usage without persisting prompts
|
||||
The Cloud planner-decision endpoint SHALL settle its reservation to the
|
||||
provider-reported token usage when available and SHALL retain only non-secret
|
||||
metering metadata, never the raw prompt, screenshot, provider credentials, or
|
||||
session/lease secret.
|
||||
|
||||
#### Scenario: Provider response includes usage
|
||||
- **WHEN** the configured provider returns a valid tool-call decision and
|
||||
token-usage metadata
|
||||
- **THEN** the endpoint records and returns the decision, settles the Host's
|
||||
reservation to the reported usage, and does not durably store request text
|
||||
or screenshot bytes
|
||||
|
||||
#### Scenario: Usage is indeterminate after failure
|
||||
- **WHEN** a reservation exists but the endpoint cannot determine provider
|
||||
usage after a transport failure
|
||||
- **THEN** the reservation remains until bounded expiry cleanup rather than
|
||||
being released as unused
|
||||
@@ -0,0 +1,61 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Host identity is authenticated and bound to one host id
|
||||
The internal Host Agent API SHALL require a host-scoped bearer principal and
|
||||
SHALL reject any request that attempts to act for a `host_id` different from
|
||||
the authenticated principal's bound host.
|
||||
|
||||
#### Scenario: Host authenticates as itself
|
||||
- **WHEN** a Host Agent presents valid credentials bound to its requested
|
||||
`host_id`
|
||||
- **THEN** the internal API authorizes permitted heartbeat, claim, renewal,
|
||||
result, policy retrieval, and self-submission operations
|
||||
|
||||
#### Scenario: Host attempts to impersonate another host
|
||||
- **WHEN** valid credentials bound to host A are used on a request for host B
|
||||
- **THEN** the internal API rejects the request without reading or modifying
|
||||
host B's state
|
||||
|
||||
### Requirement: Host Agent synchronizes heartbeat and complete device snapshots
|
||||
The Host Agent SHALL periodically submit its complete local device snapshot
|
||||
and last applied policy revision to the control plane, and the control plane
|
||||
SHALL atomically refresh the host heartbeat, replace only that host's
|
||||
pooled-device records, and return the effective Host policy whenever its
|
||||
revision differs.
|
||||
|
||||
#### Scenario: Host reports devices
|
||||
- **WHEN** a Host Agent submits a valid heartbeat containing its current devices
|
||||
- **THEN** the control plane updates the host's last-seen time and exposes the
|
||||
submitted devices through the aggregated pool
|
||||
|
||||
#### Scenario: Host policy revision changed
|
||||
- **WHEN** a Host heartbeat presents a revision older than the effective
|
||||
Cloud-host policy
|
||||
- **THEN** the response includes the complete newer policy and revision for the
|
||||
Host to cache
|
||||
|
||||
#### Scenario: Host policy revision is current
|
||||
- **WHEN** a Host heartbeat presents the current effective policy revision
|
||||
- **THEN** the response confirms that revision without resending an unrelated
|
||||
policy representation
|
||||
|
||||
#### Scenario: Host reports no devices
|
||||
- **WHEN** a previously populated host submits an empty device snapshot
|
||||
- **THEN** only that host's prior device records are removed while devices
|
||||
owned by other hosts remain unchanged
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Host can submit a self-targeted goal through the outbound protocol
|
||||
The Host Agent SHALL be able to submit a goal-only task through its existing
|
||||
outbound authenticated protocol, and the control plane SHALL derive its target
|
||||
from the authenticated Host rather than accepting another Host identifier.
|
||||
|
||||
#### Scenario: Host submits a local goal
|
||||
- **WHEN** an authenticated Host submits a valid goal through its internal
|
||||
self-submission operation
|
||||
- **THEN** the control plane returns a queued task targeted to that Host
|
||||
|
||||
#### Scenario: Host submits an ineligible local Device
|
||||
- **WHEN** a Host names a Device that is not owned by its authenticated Host
|
||||
- **THEN** the control plane rejects the request without creating a task
|
||||
@@ -0,0 +1,30 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Host can submit a goal task only for itself
|
||||
The internal Host Agent API SHALL allow an authenticated Host to submit a
|
||||
goal-based task that the Cloud Control Plane irrevocably targets to that same
|
||||
Host, with an optional Device target owned by that Host.
|
||||
|
||||
#### Scenario: Host submits a goal for one of its Devices
|
||||
- **WHEN** Host A presents valid Host A credentials and submits a valid goal
|
||||
naming one of Host A's enrolled Devices
|
||||
- **THEN** the control plane creates a queued task targeted to Host A and that
|
||||
Device without granting Host A public task-submission authority
|
||||
|
||||
#### Scenario: Host omits a Device target
|
||||
- **WHEN** an authenticated Host submits a valid goal without a Device target
|
||||
- **THEN** the control plane creates a queued task targeted to that Host and
|
||||
lets the scheduler select only an eligible Device owned by it
|
||||
|
||||
### Requirement: Host self-submission preserves host isolation
|
||||
The internal Host task-submission operation SHALL reject a foreign Host,
|
||||
foreign Device, workflow reference, or disabled self-submission policy before
|
||||
creating any task.
|
||||
|
||||
#### Scenario: Host attempts a foreign target
|
||||
- **WHEN** Host A submits a request naming Host B or a Device not owned by A
|
||||
- **THEN** the control plane rejects the request and does not create a task
|
||||
|
||||
#### Scenario: Host self-submission is disabled
|
||||
- **WHEN** a Host policy disables Host self-submission
|
||||
- **THEN** the control plane rejects that Host's self-submission request
|
||||
@@ -0,0 +1,84 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Task submission and status via the SDK
|
||||
The system SHALL allow an external integrator to submit a task (goal or
|
||||
workflow reference plus constraints and an optional explicit Host/Device
|
||||
target) through the platform SDK's API, and to query that task's current
|
||||
status by id, backed by the `task-scheduler` capability. A Device target MUST
|
||||
include its owning Host, and the status representation SHALL expose any
|
||||
explicit target separately from the eventual assignment.
|
||||
|
||||
#### Scenario: Submit a task via the API
|
||||
- **WHEN** an integrator calls the task-submission endpoint with a valid goal
|
||||
and optional constraints
|
||||
- **THEN** the API returns a task id that can be used to poll status, and the
|
||||
underlying `task-scheduler` records a new `queued` `ScheduledTask`
|
||||
|
||||
#### Scenario: Submit a task for one Host and Device
|
||||
- **WHEN** a `tasks:submit` principal submits a valid target containing an
|
||||
eligible Host and Device it is permitted to use
|
||||
- **THEN** the API records that exact target and the scheduler cannot assign
|
||||
the task outside it
|
||||
|
||||
#### Scenario: Device target lacks its Host
|
||||
- **WHEN** an integrator submits a Device target without an owning Host
|
||||
- **THEN** the API rejects the request before creating a task
|
||||
|
||||
#### Scenario: Query status of a known task
|
||||
- **WHEN** an integrator requests status for a task id that exists
|
||||
- **THEN** the API returns that task's current status (`queued`, `assigned`,
|
||||
`dispatched`, `done`, or `failed`) and its explicit target when present
|
||||
|
||||
#### Scenario: Query status of an unknown task
|
||||
- **WHEN** an integrator requests status for a task id that does not exist
|
||||
- **THEN** the API returns a not-found response rather than an unhandled server
|
||||
error
|
||||
|
||||
### Requirement: Python SDK client mirrors the REST API
|
||||
The system SHALL provide a Python client (`CloudClient`) exposing methods
|
||||
corresponding to every `/v1/...` resource, user-authentication,
|
||||
user-administration, and governance route, including targeted task submission
|
||||
and read-only Host AI-usage access, so integrators do not need to
|
||||
hand-construct HTTP requests.
|
||||
|
||||
#### Scenario: Client submits a targeted task and retrieves status
|
||||
- **WHEN** a caller uses `CloudClient` to submit a task with an explicit target
|
||||
and then fetches status by the returned id
|
||||
- **THEN** the client produces the same target and lifecycle result as direct
|
||||
REST calls
|
||||
|
||||
#### Scenario: Client administers governance with a scoped bearer
|
||||
- **WHEN** a caller configures `CloudClient` with `governance:admin` and
|
||||
invokes a policy operation
|
||||
- **THEN** the client sends that authentication and returns the corresponding
|
||||
non-secret policy representation
|
||||
|
||||
### Requirement: Public API operations enforce scopes
|
||||
The public platform API SHALL require operation-specific scopes for task
|
||||
submission, task reading, pool reading, plugin reading, plugin
|
||||
administration, user administration, governance reading, and governance
|
||||
administration, regardless of whether the principal came from a bearer
|
||||
credential or user session.
|
||||
|
||||
#### Scenario: Submit principal has task scope
|
||||
- **WHEN** a bearer or user principal with `tasks:submit` calls the
|
||||
task-submission endpoint
|
||||
- **THEN** the request is authorized subject to normal task validation and
|
||||
any effective user-submission policy
|
||||
|
||||
#### Scenario: Non-admin principal attempts plugin registration
|
||||
- **WHEN** an authenticated principal without `plugins:admin` calls plugin
|
||||
registration
|
||||
- **THEN** the API rejects the request before resolving or loading the plugin
|
||||
target
|
||||
|
||||
#### Scenario: Non-admin principal attempts user administration
|
||||
- **WHEN** an authenticated principal without `users:admin` calls a
|
||||
user-administration endpoint
|
||||
- **THEN** the API rejects the request before reading or changing protected
|
||||
user state
|
||||
|
||||
#### Scenario: Principal lacks governance administration scope
|
||||
- **WHEN** an authenticated principal without `governance:admin` changes a
|
||||
user or Host policy
|
||||
- **THEN** the API rejects the request before changing policy or usage state
|
||||
@@ -0,0 +1,58 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Task submission enqueues a scheduled task
|
||||
The system SHALL allow a caller to submit a task (a goal string, or a
|
||||
reference to a `WorkflowDefinition`, plus optional device constraints:
|
||||
`driver_type`, required capability tags, and an explicit Host/Device target)
|
||||
and SHALL enqueue it as a `ScheduledTask` with status `queued`, returning a
|
||||
stable task id the caller can poll. A Device target MUST name its owning Host;
|
||||
an explicit target remains attached to the task for its entire lifecycle.
|
||||
|
||||
#### Scenario: Successful submission
|
||||
- **WHEN** a caller submits a task with a goal and no constraints
|
||||
- **THEN** the scheduler creates a `ScheduledTask` with status `queued`,
|
||||
assigns it a unique id, and returns that id to the caller without blocking
|
||||
for a device to become available
|
||||
|
||||
#### Scenario: Targeted submission
|
||||
- **WHEN** a caller submits a valid task targeted to a Host and one of its
|
||||
Devices
|
||||
- **THEN** the queued task retains that Host/Device target until it is assigned
|
||||
or reaches a terminal lifecycle state
|
||||
|
||||
#### Scenario: Queue depth limit reached
|
||||
- **WHEN** a caller submits a task while the queue already holds
|
||||
`config.max_queue_depth` queued tasks
|
||||
- **THEN** the scheduler rejects the submission with a clear error rather than
|
||||
accepting an unbounded backlog
|
||||
|
||||
### Requirement: Assignment matches a queued task to an idle, constraint-matching device
|
||||
The system SHALL assign a queued `ScheduledTask` to an idle `PooledDevice`
|
||||
(as reported by the `device-pool` capability) whose explicit target, if any,
|
||||
`driver_type`, and capability tags satisfy the task's constraints, using a
|
||||
named, registrable `AssignmentStrategy`.
|
||||
|
||||
#### Scenario: Matching idle device available
|
||||
- **WHEN** `assign()` runs and at least one idle `PooledDevice` matches the
|
||||
head-of-queue task's constraints
|
||||
- **THEN** the scheduler selects one such device via the configured
|
||||
`AssignmentStrategy`, transitions the task to status `assigned`, and records
|
||||
the chosen `device_id`/`host_id`
|
||||
|
||||
#### Scenario: Targeted Device is unavailable
|
||||
- **WHEN** `assign()` runs and the only explicitly targeted Device is busy,
|
||||
stale, absent, or otherwise ineligible
|
||||
- **THEN** the task remains `queued` and the scheduler does not assign a
|
||||
different Device or Host
|
||||
|
||||
#### Scenario: No matching device available
|
||||
- **WHEN** `assign()` runs and no idle `PooledDevice` matches the head-of-queue
|
||||
task's constraints
|
||||
- **THEN** the task remains `queued` (not failed), and `assign()` returns
|
||||
without error, ready to be retried on a later call
|
||||
|
||||
#### Scenario: Unknown assignment strategy configured
|
||||
- **WHEN** `TaskScheduler` is configured with an `AssignmentStrategy` name
|
||||
that is not registered
|
||||
- **THEN** the scheduler raises a clear configuration error at startup/first-
|
||||
assign rather than silently falling back to a default strategy
|
||||
@@ -0,0 +1,113 @@
|
||||
## 1. Preconditions and dependency reconciliation
|
||||
|
||||
- [x] 1.1 Confirm `cloud-planner-proxy` is fully implemented, passes its
|
||||
strict validation and relevant runtime tests, and its deployed schema/API
|
||||
contract is the base for this change.
|
||||
- [ ] 1.2 Complete and reconcile `cloud-console-user-authentication`, including
|
||||
the documented Console Users experience and its remaining PostgreSQL and
|
||||
HTTPS/Host-Agent verification tasks.
|
||||
- [ ] 1.3 Rebase this change's delta specs against the then-current canonical
|
||||
`platform-sdk`, `cloud-console-ui`, `cloud-planner-proxy`, and
|
||||
`agent-runtime` specifications before implementation begins.
|
||||
|
||||
## 2. Governance domain, persistence, and migration
|
||||
|
||||
- [ ] 2.1 Define target-selector, user-submission-policy, Host-governance-
|
||||
policy, token-usage-event, and token-reservation domain models with
|
||||
non-secret representations and revision semantics.
|
||||
- [ ] 2.2 Extend the Cloud repository port with transactional CRUD/query
|
||||
operations for policies, active Host capacity, budget reservations,
|
||||
settlement, expiry cleanup, and bounded usage summaries/events.
|
||||
- [ ] 2.3 Add SQLAlchemy rows, indexes, conversion helpers, and concurrency-
|
||||
safe PostgreSQL/SQLite implementations for governance policies, usage
|
||||
events, reservations, and safe policy audits.
|
||||
- [ ] 2.4 Add an Alembic forward/downgrade revision that preserves existing
|
||||
users, Hosts, devices, tasks, and attempts; advance schema readiness
|
||||
checks to the new head.
|
||||
- [ ] 2.5 Add repository and migration tests for policy revisions, null versus
|
||||
empty allow-list semantics, active-task capacity, reservation races,
|
||||
settlement, expiry cleanup, audit redaction, and SQLite/PostgreSQL
|
||||
parity.
|
||||
|
||||
## 3. Targeted scheduling and public governance API
|
||||
|
||||
- [x] 3.1 Extend task constraints, SDK request/response models, persistence
|
||||
serialization, status/list representations, and `CloudClient` for an
|
||||
optional Host/Device target; reject a Device without its Host.
|
||||
- [x] 3.2 Enforce target ownership/existence at public submission and filter
|
||||
scheduler candidates by target before strategy selection; keep an
|
||||
unavailable targeted task queued with no fallback assignment.
|
||||
- [x] 3.3 Add `governance:read` and `governance:admin` scopes and policy-aware
|
||||
public task authorization that combines `tasks:submit` with the
|
||||
authenticated human user's effective submission policy.
|
||||
- [ ] 3.4 Add bounded, non-secret public governance routes and `CloudClient`
|
||||
methods for user policy, Host policy, Host AI-budget summaries, and
|
||||
paginated usage events; audit every policy mutation.
|
||||
- [ ] 3.5 Add public API/SDK tests for targeted submission, target-policy
|
||||
denial, scope enforcement, status exposure, governance CRUD, and
|
||||
absence of secret fields in responses/audits.
|
||||
|
||||
## 4. Host policy synchronization and self-submission
|
||||
|
||||
- [x] 4.1 Extend shared internal heartbeat request/response models with the
|
||||
Host's last policy revision and a revision-aware effective-policy reply.
|
||||
- [ ] 4.2 Update Host heartbeat synchronization to persist only safe cached
|
||||
policy state and expose it through local status/history without adding an
|
||||
inbound Cloud connection.
|
||||
- [x] 4.3 Add the Host-scoped goal-only task-submission route and
|
||||
`HostAgentClient` method; derive Host targeting from authenticated
|
||||
credentials and validate any named local Device ownership.
|
||||
- [ ] 4.4 Enforce the Host policy's self-submission and active-task limits in
|
||||
the Cloud service/scheduler, not only in Host-local code.
|
||||
- [ ] 4.5 Add Cloud API and Host Agent tests for revision convergence,
|
||||
unchanged-policy replies, self-targeted task creation, foreign target
|
||||
rejection, disabled self-submission, and outbound-only compatibility.
|
||||
|
||||
## 5. Cloud-proxy usage metering and hard budget enforcement
|
||||
|
||||
- [x] 5.1 Extend the dual-provider tool-calling result with optional
|
||||
non-secret provider usage fields while preserving `AIPlanner`'s existing
|
||||
single-decision behavior and direct transport compatibility.
|
||||
- [ ] 5.2 Extend planner-proxy request context and Host-Agent-local context
|
||||
binding so Cloud-proxied calls carry known task/attempt metadata without
|
||||
importing Host or Cloud concerns into `runtime`.
|
||||
- [ ] 5.3 Add Cloud proxy preflight reservation, configured conservative
|
||||
per-call ceiling, provider invocation, actual-usage settlement, and
|
||||
bounded unknown-usage reservation expiry.
|
||||
- [ ] 5.4 Record non-secret usage events and expose accurate
|
||||
used/reserved/remaining UTC-day budget summaries; explicitly report
|
||||
direct transport as unmetered.
|
||||
- [ ] 5.5 Add provider-fake, repository concurrency, Cloud API, and Host
|
||||
Agent tests for usage parsing, over-budget rejection before provider
|
||||
invocation, concurrent reservations, settlement, timeout uncertainty,
|
||||
task-attempt attribution, and direct-transport labeling.
|
||||
|
||||
## 6. Cloud Console governance experience
|
||||
|
||||
- [x] 6.1 Add task composer API bindings and a scope-aware Console form for
|
||||
goal/workflow submission, Host/Device target selection, and target/policy
|
||||
validation errors.
|
||||
- [ ] 6.2 Complete or reconcile the admin Users view, then add user-
|
||||
submission-policy editing with safe refresh and conflict/error handling.
|
||||
- [ ] 6.3 Add Host policy administration and AI-usage/budget views, including
|
||||
revision display, unmetered direct Hosts, and no rendering of prompts,
|
||||
screenshots, provider credentials, cookies, or lease secrets.
|
||||
- [ ] 6.4 Add frontend tests for task composer scope/policy failures,
|
||||
governance navigation, policy mutations, usage rendering, unmetered
|
||||
status, CSRF writes, and retained session behavior on `403`.
|
||||
|
||||
## 7. Documentation, verification, and rollout
|
||||
|
||||
- [ ] 7.1 Update deployment and Console documentation with the dependency
|
||||
order, migration/rollback sequence, policy semantics, Cloud transport
|
||||
prerequisite, budget reservation behavior, direct-host limitation, and
|
||||
safe administrator operations.
|
||||
- [ ] 7.2 Run formatting, Ruff, compile checks, secret/redaction review, the
|
||||
complete non-integration workspace test suite, Cloud Console unit tests,
|
||||
type-check, and production build.
|
||||
- [ ] 7.3 Run real PostgreSQL migration/concurrency coverage and a Compose HTTPS
|
||||
manual flow covering user restriction, targeted scheduling, Host policy
|
||||
delivery, Host self-submission, Cloud-proxy budget exhaustion, and a
|
||||
direct Host labelled unmetered.
|
||||
- [x] 7.4 Run `openspec validate cloud-console-governance --strict` and resolve
|
||||
all proposal, design, specification, and task validation errors.
|
||||
@@ -16,6 +16,8 @@ POOL_READ_SCOPE = "pool:read"
|
||||
PLUGINS_READ_SCOPE = "plugins:read"
|
||||
PLUGINS_ADMIN_SCOPE = "plugins:admin"
|
||||
USERS_ADMIN_SCOPE = "users:admin"
|
||||
GOVERNANCE_READ_SCOPE = "governance:read"
|
||||
GOVERNANCE_ADMIN_SCOPE = "governance:admin"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -215,3 +215,29 @@ class AuthAuditRow(Base):
|
||||
outcome: Mapped[str] = mapped_column(String, nullable=False)
|
||||
correlation_id: Mapped[str | None] = mapped_column(String, nullable=True)
|
||||
metadata_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||
|
||||
|
||||
class UserSubmissionPolicyRow(Base):
|
||||
__tablename__ = "cloud_user_submission_policies"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("cloud_users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
submission_enabled: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
allowed_host_ids_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
allowed_device_targets_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
updated_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
|
||||
class HostGovernancePolicyRow(Base):
|
||||
__tablename__ = "cloud_host_governance_policies"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("host_registrations.host_id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
self_submission_enabled: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
max_active_tasks: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
daily_token_budget: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
updated_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Durable Cloud-side submission and Host governance policies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class TaskSubmissionPolicyError(PermissionError):
|
||||
"""Raised when a human user's policy disallows a task submission."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UserSubmissionPolicy:
|
||||
user_id: str
|
||||
revision: int
|
||||
submission_enabled: bool
|
||||
allowed_host_ids: tuple[str, ...] | None
|
||||
allowed_device_targets: tuple[tuple[str, str], ...] | None
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostGovernancePolicy:
|
||||
host_id: str
|
||||
revision: int
|
||||
self_submission_enabled: bool
|
||||
max_active_tasks: int | None
|
||||
daily_token_budget: int | None
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
def enforce_user_submission_policy(
|
||||
policy: UserSubmissionPolicy | None,
|
||||
*,
|
||||
target_host_id: str | None,
|
||||
target_device_id: str | None,
|
||||
) -> None:
|
||||
if policy is None:
|
||||
return
|
||||
if not policy.submission_enabled:
|
||||
raise TaskSubmissionPolicyError("task submission is disabled for this user")
|
||||
restricted = (
|
||||
policy.allowed_host_ids is not None
|
||||
or policy.allowed_device_targets is not None
|
||||
)
|
||||
if not restricted:
|
||||
return
|
||||
if target_host_id is None:
|
||||
raise TaskSubmissionPolicyError("an explicit permitted target is required")
|
||||
if (
|
||||
policy.allowed_host_ids is not None
|
||||
and target_host_id not in policy.allowed_host_ids
|
||||
):
|
||||
raise TaskSubmissionPolicyError("target host is not permitted")
|
||||
if policy.allowed_device_targets is not None:
|
||||
if target_device_id is None or (
|
||||
target_host_id,
|
||||
target_device_id,
|
||||
) not in policy.allowed_device_targets:
|
||||
raise TaskSubmissionPolicyError("target device is not permitted")
|
||||
@@ -25,8 +25,11 @@ from cloud.internal_api.models import (
|
||||
DeviceEnrollmentResponse,
|
||||
HeartbeatRequest,
|
||||
HeartbeatResponse,
|
||||
HostGovernancePolicyModel,
|
||||
HostEnrollmentRequest,
|
||||
HostEnrollmentResponse,
|
||||
HostTaskSubmissionRequest,
|
||||
HostTaskSubmissionResponse,
|
||||
LeaseRenewalRequest,
|
||||
LeaseRenewalResponse,
|
||||
PlannerDecisionError,
|
||||
@@ -47,6 +50,7 @@ from runtime.tool_specs import ToolSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cloud.pool import DevicePool
|
||||
from cloud.scheduler import TaskScheduler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -60,6 +64,7 @@ def create_internal_router(
|
||||
lease_duration_seconds: float = 60.0,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
planner_client_factory: Callable[[], ToolCallingClient] | None = None,
|
||||
scheduler: TaskScheduler | None = None,
|
||||
) -> APIRouter:
|
||||
if claim_poll_interval_seconds <= 0:
|
||||
raise ValueError("claim_poll_interval_seconds must be greater than zero")
|
||||
@@ -166,12 +171,71 @@ def create_internal_router(
|
||||
address=payload.address,
|
||||
allow_device_takeover=allow_device_takeover,
|
||||
)
|
||||
policy = pool.store.get_host_governance_policy(host_id)
|
||||
policy_revision = policy.revision if policy is not None else 0
|
||||
return HeartbeatResponse(
|
||||
host_id=host_id,
|
||||
accepted_devices=len(devices),
|
||||
received_at=utc_now(),
|
||||
policy_revision=policy_revision,
|
||||
policy=(
|
||||
HostGovernancePolicyModel(
|
||||
revision=policy.revision,
|
||||
self_submission_enabled=policy.self_submission_enabled,
|
||||
max_active_tasks=policy.max_active_tasks,
|
||||
daily_token_budget=policy.daily_token_budget,
|
||||
)
|
||||
if policy is not None and payload.policy_revision != policy.revision
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/hosts/{host_id}/tasks",
|
||||
response_model=HostTaskSubmissionResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def submit_host_task(
|
||||
host_id: str,
|
||||
payload: HostTaskSubmissionRequest,
|
||||
request: Request,
|
||||
) -> HostTaskSubmissionResponse:
|
||||
authorize_host(request, host_id)
|
||||
if payload.host_id != host_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="task host_id must match the request path",
|
||||
)
|
||||
if scheduler is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="task submission is unavailable",
|
||||
)
|
||||
policy = pool.store.get_host_governance_policy(host_id)
|
||||
if policy is not None and not policy.self_submission_enabled:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Host self-submission is disabled",
|
||||
)
|
||||
if payload.device_id is not None and not any(
|
||||
device.host_id == host_id and device.device_id == payload.device_id
|
||||
for device in pool.list_devices()
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="target device is not owned by this Host",
|
||||
)
|
||||
from cloud.scheduler import TaskConstraints
|
||||
|
||||
task_id = scheduler.submit(
|
||||
goal=payload.goal,
|
||||
constraints=TaskConstraints(
|
||||
target_host_id=host_id,
|
||||
target_device_id=payload.device_id,
|
||||
),
|
||||
)
|
||||
return HostTaskSubmissionResponse(task_id=task_id)
|
||||
|
||||
@router.post(
|
||||
"/hosts/{host_id}/assignments/claim",
|
||||
response_model=ClaimResponse,
|
||||
@@ -285,6 +349,7 @@ def create_internal_router(
|
||||
@router.post(
|
||||
"/hosts/{host_id}/planner/decide",
|
||||
response_model=PlannerDecisionResponse,
|
||||
response_model_exclude_none=True,
|
||||
responses={
|
||||
status.HTTP_502_BAD_GATEWAY: {"model": PlannerDecisionError},
|
||||
},
|
||||
@@ -354,6 +419,9 @@ def create_internal_router(
|
||||
return PlannerDecisionResponse(
|
||||
tool_name=decision.tool_name,
|
||||
arguments=dict(decision.arguments),
|
||||
input_tokens=(decision.usage.input_tokens if decision.usage else None),
|
||||
output_tokens=(decision.usage.output_tokens if decision.usage else None),
|
||||
total_tokens=(decision.usage.total_tokens if decision.usage else None),
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
@@ -38,12 +38,22 @@ class HeartbeatRequest(BaseModel):
|
||||
host_id: str = Field(min_length=1)
|
||||
address: str | None = None
|
||||
devices: list[DeviceSnapshotModel] = Field(default_factory=list)
|
||||
policy_revision: int = Field(default=0, ge=0)
|
||||
|
||||
|
||||
class HostGovernancePolicyModel(BaseModel):
|
||||
revision: int = Field(ge=1)
|
||||
self_submission_enabled: bool
|
||||
max_active_tasks: int | None = None
|
||||
daily_token_budget: int | None = None
|
||||
|
||||
|
||||
class HeartbeatResponse(BaseModel):
|
||||
host_id: str
|
||||
accepted_devices: int
|
||||
received_at: datetime
|
||||
policy_revision: int = Field(default=0, ge=0)
|
||||
policy: HostGovernancePolicyModel | None = None
|
||||
|
||||
|
||||
class ClaimRequest(BaseModel):
|
||||
@@ -93,6 +103,16 @@ class TerminalResultResponse(BaseModel):
|
||||
status: Literal["recorded", "already_recorded"]
|
||||
|
||||
|
||||
class HostTaskSubmissionRequest(BaseModel):
|
||||
host_id: str = Field(min_length=1)
|
||||
goal: str = Field(min_length=1)
|
||||
device_id: str | None = Field(default=None, min_length=1)
|
||||
|
||||
|
||||
class HostTaskSubmissionResponse(BaseModel):
|
||||
task_id: str
|
||||
|
||||
|
||||
class StaleLeaseConflict(BaseModel):
|
||||
code: Literal["stale_lease"] = "stale_lease"
|
||||
detail: str
|
||||
@@ -116,6 +136,9 @@ class PlannerDecisionRequest(BaseModel):
|
||||
class PlannerDecisionResponse(BaseModel):
|
||||
tool_name: str
|
||||
arguments: dict[str, Any] = Field(default_factory=dict)
|
||||
input_tokens: int | None = Field(default=None, ge=0)
|
||||
output_tokens: int | None = Field(default=None, ge=0)
|
||||
total_tokens: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class PlannerDecisionError(BaseModel):
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Add durable Cloud governance policies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0004_cloud_governance"
|
||||
down_revision = "0003_cloud_user_authentication"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
tables = set(inspector.get_table_names())
|
||||
if "cloud_user_submission_policies" not in tables:
|
||||
op.create_table(
|
||||
"cloud_user_submission_policies",
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.String(),
|
||||
sa.ForeignKey("cloud_users.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("submission_enabled", sa.Integer(), nullable=False),
|
||||
sa.Column("allowed_host_ids_json", sa.Text(), nullable=True),
|
||||
sa.Column("allowed_device_targets_json", sa.Text(), nullable=True),
|
||||
sa.Column("updated_at", sa.String(), nullable=False),
|
||||
)
|
||||
if "cloud_host_governance_policies" not in tables:
|
||||
op.create_table(
|
||||
"cloud_host_governance_policies",
|
||||
sa.Column(
|
||||
"host_id",
|
||||
sa.String(),
|
||||
sa.ForeignKey("host_registrations.host_id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("self_submission_enabled", sa.Integer(), nullable=False),
|
||||
sa.Column("max_active_tasks", sa.Integer(), nullable=True),
|
||||
sa.Column("daily_token_budget", sa.Integer(), nullable=True),
|
||||
sa.Column("updated_at", sa.String(), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
tables = set(inspector.get_table_names())
|
||||
if "cloud_host_governance_policies" in tables:
|
||||
op.drop_table("cloud_host_governance_policies")
|
||||
if "cloud_user_submission_policies" in tables:
|
||||
op.drop_table("cloud_user_submission_policies")
|
||||
@@ -15,6 +15,7 @@ if TYPE_CHECKING:
|
||||
UserAccount,
|
||||
UserSession,
|
||||
)
|
||||
from cloud.governance import HostGovernancePolicy, UserSubmissionPolicy
|
||||
|
||||
|
||||
AttemptStatus = Literal["assigned", "dispatched", "done", "failed", "expired"]
|
||||
@@ -283,6 +284,30 @@ class CloudRepository(Protocol):
|
||||
|
||||
def cleanup_auth_state(self, *, now: datetime, limit: int) -> int: ...
|
||||
|
||||
def get_user_submission_policy(self, user_id: str) -> UserSubmissionPolicy | None: ...
|
||||
|
||||
def upsert_user_submission_policy(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
submission_enabled: bool,
|
||||
allowed_host_ids: tuple[str, ...] | None,
|
||||
allowed_device_targets: tuple[tuple[str, str], ...] | None,
|
||||
updated_at: datetime,
|
||||
) -> UserSubmissionPolicy: ...
|
||||
|
||||
def get_host_governance_policy(self, host_id: str) -> HostGovernancePolicy | None: ...
|
||||
|
||||
def upsert_host_governance_policy(
|
||||
self,
|
||||
*,
|
||||
host_id: str,
|
||||
self_submission_enabled: bool,
|
||||
max_active_tasks: int | None,
|
||||
daily_token_budget: int | None,
|
||||
updated_at: datetime,
|
||||
) -> HostGovernancePolicy: ...
|
||||
|
||||
def list_reserved_device_ids(self, *, now: datetime) -> set[str]: ...
|
||||
|
||||
def assign_task(
|
||||
|
||||
@@ -31,6 +31,8 @@ class TaskConstraints:
|
||||
|
||||
driver_type: str | None = None
|
||||
capability_tags: list[str] = field(default_factory=list)
|
||||
target_host_id: str | None = None
|
||||
target_device_id: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -195,6 +197,10 @@ class TaskScheduler:
|
||||
|
||||
|
||||
def _matches(device: "PooledDevice", constraints: TaskConstraints) -> bool:
|
||||
if constraints.target_host_id and device.host_id != constraints.target_host_id:
|
||||
return False
|
||||
if constraints.target_device_id and device.device_id != constraints.target_device_id:
|
||||
return False
|
||||
if constraints.driver_type and device.driver_type != constraints.driver_type:
|
||||
return False
|
||||
if constraints.capability_tags:
|
||||
|
||||
@@ -9,7 +9,7 @@ from alembic.runtime.migration import MigrationContext
|
||||
from cloud.database import create_database_engine, normalize_database_url
|
||||
|
||||
|
||||
HEAD_REVISION = "0003_cloud_user_authentication"
|
||||
HEAD_REVISION = "0004_cloud_governance"
|
||||
|
||||
|
||||
class SchemaVersionError(RuntimeError):
|
||||
|
||||
@@ -22,6 +22,7 @@ from cloud.auth import (
|
||||
NullAuthProvider,
|
||||
Principal,
|
||||
)
|
||||
from cloud.governance import TaskSubmissionPolicyError, enforce_user_submission_policy
|
||||
from cloud.sdk.models import (
|
||||
DeviceResponse,
|
||||
ErrorResponse,
|
||||
@@ -94,8 +95,21 @@ def create_cloud_router(
|
||||
payload: TaskSubmissionRequest,
|
||||
request: Request,
|
||||
) -> TaskSubmissionResponse:
|
||||
_authorize(request, TASKS_SUBMIT_SCOPE)
|
||||
task_constraints = _build_constraints(payload.constraints)
|
||||
principal = _authorize(request, TASKS_SUBMIT_SCOPE)
|
||||
try:
|
||||
task_constraints = _build_constraints(payload.constraints)
|
||||
_validate_task_target(pool, task_constraints)
|
||||
_enforce_user_policy(principal, scheduler.store, task_constraints)
|
||||
except TaskSubmissionPolicyError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
try:
|
||||
task_id = scheduler.submit(
|
||||
goal=payload.goal,
|
||||
@@ -128,6 +142,8 @@ def create_cloud_router(
|
||||
attempt_count=task.attempt_count,
|
||||
lease_expires_at=task.lease_expires_at,
|
||||
failure_reason=task.failure_reason,
|
||||
target_host_id=task.constraints.target_host_id,
|
||||
target_device_id=task.constraints.target_device_id,
|
||||
)
|
||||
|
||||
@router.get("/tasks", response_model=TaskListResponse)
|
||||
@@ -158,6 +174,8 @@ def create_cloud_router(
|
||||
assigned_host_id=task.assigned_host_id,
|
||||
attempt_count=task.attempt_count,
|
||||
failure_reason=task.failure_reason,
|
||||
target_host_id=task.constraints.target_host_id,
|
||||
target_device_id=task.constraints.target_device_id,
|
||||
created_at=task.created_at,
|
||||
)
|
||||
for task in tasks
|
||||
@@ -303,4 +321,35 @@ def _build_constraints(model):
|
||||
return TaskConstraints(
|
||||
driver_type=model.driver_type,
|
||||
capability_tags=list(model.capability_tags),
|
||||
target_host_id=model.target_host_id,
|
||||
target_device_id=model.target_device_id,
|
||||
)
|
||||
|
||||
|
||||
def _validate_task_target(pool, constraints) -> None:
|
||||
if constraints.target_device_id and not constraints.target_host_id:
|
||||
raise ValueError("target_device_id requires target_host_id")
|
||||
if constraints.target_host_id is None:
|
||||
return
|
||||
if not any(host.host_id == constraints.target_host_id for host in pool.list_hosts()):
|
||||
raise ValueError(f"target host {constraints.target_host_id!r} is not known")
|
||||
if constraints.target_device_id is not None and not any(
|
||||
device.host_id == constraints.target_host_id
|
||||
and device.device_id == constraints.target_device_id
|
||||
for device in pool.list_devices()
|
||||
):
|
||||
raise ValueError(
|
||||
f"target device {constraints.target_device_id!r} is not owned by "
|
||||
f"host {constraints.target_host_id!r}"
|
||||
)
|
||||
|
||||
|
||||
def _enforce_user_policy(principal, store, constraints) -> None:
|
||||
if not principal.id.startswith("user:"):
|
||||
return
|
||||
policy = store.get_user_submission_policy(principal.id.removeprefix("user:"))
|
||||
enforce_user_submission_policy(
|
||||
policy,
|
||||
target_host_id=constraints.target_host_id,
|
||||
target_device_id=constraints.target_device_id,
|
||||
)
|
||||
|
||||
@@ -72,15 +72,24 @@ class CloudClient:
|
||||
workflow_definition_id: str | None = None,
|
||||
driver_type: str | None = None,
|
||||
capability_tags: list[str] | None = None,
|
||||
target_host_id: str | None = None,
|
||||
target_device_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"goal": goal,
|
||||
"workflow_definition_id": workflow_definition_id,
|
||||
}
|
||||
if driver_type is not None or capability_tags is not None:
|
||||
if (
|
||||
driver_type is not None
|
||||
or capability_tags is not None
|
||||
or target_host_id is not None
|
||||
or target_device_id is not None
|
||||
):
|
||||
payload["constraints"] = {
|
||||
"driver_type": driver_type,
|
||||
"capability_tags": list(capability_tags or []),
|
||||
"target_host_id": target_host_id,
|
||||
"target_device_id": target_device_id,
|
||||
}
|
||||
resp = self._request("POST", "/tasks", json=payload)
|
||||
return resp.json()
|
||||
@@ -209,6 +218,28 @@ class CloudClient:
|
||||
def revoke_user_sessions(self, user_id: str) -> None:
|
||||
self._request("DELETE", f"/users/{user_id}/sessions")
|
||||
|
||||
# ------------------------------------------------------------- governance
|
||||
|
||||
def get_user_submission_policy(self, user_id: str) -> dict[str, Any]:
|
||||
return self._request("GET", f"/users/{user_id}/submission-policy").json()
|
||||
|
||||
def update_user_submission_policy(
|
||||
self, user_id: str, **policy: Any
|
||||
) -> dict[str, Any]:
|
||||
return self._request(
|
||||
"PUT", f"/users/{user_id}/submission-policy", json=policy
|
||||
).json()
|
||||
|
||||
def get_host_governance_policy(self, host_id: str) -> dict[str, Any]:
|
||||
return self._request("GET", f"/hosts/{host_id}/governance-policy").json()
|
||||
|
||||
def update_host_governance_policy(
|
||||
self, host_id: str, **policy: Any
|
||||
) -> dict[str, Any]:
|
||||
return self._request(
|
||||
"PUT", f"/hosts/{host_id}/governance-policy", json=policy
|
||||
).json()
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request, status
|
||||
|
||||
from cloud.auth import AuthProvider, GOVERNANCE_ADMIN_SCOPE, GOVERNANCE_READ_SCOPE
|
||||
from cloud.observability import current_correlation_id
|
||||
from cloud.sdk.models import (
|
||||
HostGovernancePolicyRequest,
|
||||
HostGovernancePolicyResponse,
|
||||
UserSubmissionPolicyRequest,
|
||||
UserSubmissionPolicyResponse,
|
||||
)
|
||||
from cloud.user_auth import AuthAuditEvent
|
||||
from core.models import utc_now
|
||||
|
||||
|
||||
def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIRouter:
|
||||
router = APIRouter(prefix="/v1", tags=["cloud-governance"])
|
||||
|
||||
def authorize(request: Request, required_scope: str):
|
||||
principal = auth_provider.authenticate(request)
|
||||
if principal is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="unauthorized",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
if principal.must_change_password or not principal.has_scope(required_scope):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"missing required scope: {required_scope}",
|
||||
)
|
||||
return principal
|
||||
|
||||
@router.get(
|
||||
"/users/{user_id}/submission-policy",
|
||||
response_model=UserSubmissionPolicyResponse,
|
||||
)
|
||||
def get_user_policy(user_id: str, request: Request) -> UserSubmissionPolicyResponse:
|
||||
authorize(request, GOVERNANCE_READ_SCOPE)
|
||||
policy = repository.get_user_submission_policy(user_id)
|
||||
if policy is None:
|
||||
raise HTTPException(status_code=404, detail="submission policy not found")
|
||||
return UserSubmissionPolicyResponse(
|
||||
user_id=policy.user_id,
|
||||
revision=policy.revision,
|
||||
submission_enabled=policy.submission_enabled,
|
||||
allowed_host_ids=list(policy.allowed_host_ids)
|
||||
if policy.allowed_host_ids is not None
|
||||
else None,
|
||||
allowed_device_targets=[
|
||||
{"host_id": host_id, "device_id": device_id}
|
||||
for host_id, device_id in policy.allowed_device_targets or ()
|
||||
]
|
||||
if policy.allowed_device_targets is not None
|
||||
else None,
|
||||
updated_at=policy.updated_at,
|
||||
)
|
||||
|
||||
@router.put(
|
||||
"/users/{user_id}/submission-policy",
|
||||
response_model=UserSubmissionPolicyResponse,
|
||||
)
|
||||
def put_user_policy(
|
||||
user_id: str,
|
||||
payload: UserSubmissionPolicyRequest,
|
||||
request: Request,
|
||||
) -> UserSubmissionPolicyResponse:
|
||||
principal = authorize(request, GOVERNANCE_ADMIN_SCOPE)
|
||||
try:
|
||||
policy = repository.upsert_user_submission_policy(
|
||||
user_id=user_id,
|
||||
submission_enabled=payload.submission_enabled,
|
||||
allowed_host_ids=_unique_strings(payload.allowed_host_ids),
|
||||
allowed_device_targets=(
|
||||
tuple((item.host_id, item.device_id) for item in payload.allowed_device_targets)
|
||||
if payload.allowed_device_targets is not None
|
||||
else None
|
||||
),
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="user not found") from exc
|
||||
_audit(repository, principal.id, user_id, "user_submission_policy_update")
|
||||
return UserSubmissionPolicyResponse(
|
||||
user_id=policy.user_id,
|
||||
revision=policy.revision,
|
||||
submission_enabled=policy.submission_enabled,
|
||||
allowed_host_ids=list(policy.allowed_host_ids)
|
||||
if policy.allowed_host_ids is not None
|
||||
else None,
|
||||
allowed_device_targets=[
|
||||
{"host_id": host_id, "device_id": device_id}
|
||||
for host_id, device_id in policy.allowed_device_targets or ()
|
||||
]
|
||||
if policy.allowed_device_targets is not None
|
||||
else None,
|
||||
updated_at=policy.updated_at,
|
||||
)
|
||||
|
||||
@router.get(
|
||||
"/hosts/{host_id}/governance-policy",
|
||||
response_model=HostGovernancePolicyResponse,
|
||||
)
|
||||
def get_host_policy(host_id: str, request: Request) -> HostGovernancePolicyResponse:
|
||||
authorize(request, GOVERNANCE_READ_SCOPE)
|
||||
policy = repository.get_host_governance_policy(host_id)
|
||||
if policy is None:
|
||||
raise HTTPException(status_code=404, detail="Host policy not found")
|
||||
return _host_response(policy)
|
||||
|
||||
@router.put(
|
||||
"/hosts/{host_id}/governance-policy",
|
||||
response_model=HostGovernancePolicyResponse,
|
||||
)
|
||||
def put_host_policy(
|
||||
host_id: str,
|
||||
payload: HostGovernancePolicyRequest,
|
||||
request: Request,
|
||||
) -> HostGovernancePolicyResponse:
|
||||
principal = authorize(request, GOVERNANCE_ADMIN_SCOPE)
|
||||
try:
|
||||
policy = repository.upsert_host_governance_policy(
|
||||
host_id=host_id,
|
||||
self_submission_enabled=payload.self_submission_enabled,
|
||||
max_active_tasks=payload.max_active_tasks,
|
||||
daily_token_budget=payload.daily_token_budget,
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="Host not found") from exc
|
||||
_audit(repository, principal.id, host_id, "host_governance_policy_update")
|
||||
return _host_response(policy)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _unique_strings(values: list[str] | None) -> tuple[str, ...] | None:
|
||||
if values is None:
|
||||
return None
|
||||
return tuple(dict.fromkeys(value for value in values if value))
|
||||
|
||||
|
||||
def _host_response(policy) -> HostGovernancePolicyResponse:
|
||||
return HostGovernancePolicyResponse(
|
||||
host_id=policy.host_id,
|
||||
revision=policy.revision,
|
||||
self_submission_enabled=policy.self_submission_enabled,
|
||||
max_active_tasks=policy.max_active_tasks,
|
||||
daily_token_budget=policy.daily_token_budget,
|
||||
updated_at=policy.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def _audit(repository, actor_id: str, target: str, action: str) -> None:
|
||||
repository.record_auth_audit(
|
||||
AuthAuditEvent(
|
||||
id=uuid4().hex,
|
||||
occurred_at=utc_now(),
|
||||
actor_principal_id=actor_id,
|
||||
target_user_id=target if action.startswith("user_") else None,
|
||||
action=action,
|
||||
outcome="success",
|
||||
correlation_id=current_correlation_id(),
|
||||
metadata={"target": target},
|
||||
)
|
||||
)
|
||||
@@ -11,6 +11,8 @@ from pydantic import BaseModel, Field
|
||||
class TaskConstraintsModel(BaseModel):
|
||||
driver_type: str | None = None
|
||||
capability_tags: list[str] = Field(default_factory=list)
|
||||
target_host_id: str | None = Field(default=None, min_length=1)
|
||||
target_device_id: str | None = Field(default=None, min_length=1)
|
||||
|
||||
|
||||
class TaskSubmissionRequest(BaseModel):
|
||||
@@ -33,6 +35,8 @@ class TaskStatusResponse(BaseModel):
|
||||
attempt_count: int = 0
|
||||
lease_expires_at: datetime | None = None
|
||||
failure_reason: str | None = None
|
||||
target_host_id: str | None = None
|
||||
target_device_id: str | None = None
|
||||
|
||||
|
||||
class TaskListItem(BaseModel):
|
||||
@@ -44,6 +48,8 @@ class TaskListItem(BaseModel):
|
||||
assigned_host_id: str | None = None
|
||||
attempt_count: int = 0
|
||||
failure_reason: str | None = None
|
||||
target_host_id: str | None = None
|
||||
target_device_id: str | None = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@@ -145,3 +151,32 @@ class PasswordResetRequest(BaseModel):
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
detail: str
|
||||
|
||||
|
||||
class DeviceTargetModel(BaseModel):
|
||||
host_id: str = Field(min_length=1)
|
||||
device_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class UserSubmissionPolicyRequest(BaseModel):
|
||||
submission_enabled: bool = True
|
||||
allowed_host_ids: list[str] | None = None
|
||||
allowed_device_targets: list[DeviceTargetModel] | None = None
|
||||
|
||||
|
||||
class UserSubmissionPolicyResponse(UserSubmissionPolicyRequest):
|
||||
user_id: str
|
||||
revision: int
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class HostGovernancePolicyRequest(BaseModel):
|
||||
self_submission_enabled: bool = True
|
||||
max_active_tasks: int | None = Field(default=None, ge=1)
|
||||
daily_token_budget: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
class HostGovernancePolicyResponse(HostGovernancePolicyRequest):
|
||||
host_id: str
|
||||
revision: int
|
||||
updated_at: datetime
|
||||
|
||||
@@ -19,9 +19,11 @@ from cloud.db_models import (
|
||||
ScheduledTaskRow,
|
||||
TaskAttemptRow,
|
||||
AuthAuditRow,
|
||||
HostGovernancePolicyRow,
|
||||
LoginThrottleRow,
|
||||
UserRow,
|
||||
UserSessionRow,
|
||||
UserSubmissionPolicyRow,
|
||||
)
|
||||
from cloud.observability import current_correlation_id
|
||||
from core.models import utc_now
|
||||
@@ -807,6 +809,86 @@ class SQLAlchemyCloudRepository:
|
||||
removed += len(stale_throttles)
|
||||
return removed
|
||||
|
||||
# -------------------------------------------------------------- governance
|
||||
|
||||
def get_user_submission_policy(self, user_id: str) -> Any | None:
|
||||
with self._sessions() as session:
|
||||
row = session.get(UserSubmissionPolicyRow, user_id)
|
||||
return _user_submission_policy_from_row(row) if row is not None else None
|
||||
|
||||
def upsert_user_submission_policy(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
submission_enabled: bool,
|
||||
allowed_host_ids: tuple[str, ...] | None,
|
||||
allowed_device_targets: tuple[tuple[str, str], ...] | None,
|
||||
updated_at: datetime,
|
||||
) -> Any:
|
||||
with self._sessions.begin() as session:
|
||||
if session.get(UserRow, user_id) is None:
|
||||
raise KeyError(f"unknown user {user_id!r}")
|
||||
row = session.get(UserSubmissionPolicyRow, user_id)
|
||||
if row is None:
|
||||
row = UserSubmissionPolicyRow(
|
||||
user_id=user_id,
|
||||
revision=1,
|
||||
submission_enabled=1 if submission_enabled else 0,
|
||||
allowed_host_ids_json=_dump_optional_list(allowed_host_ids),
|
||||
allowed_device_targets_json=_dump_optional_list(
|
||||
allowed_device_targets
|
||||
),
|
||||
updated_at=_iso(updated_at),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.revision += 1
|
||||
row.submission_enabled = 1 if submission_enabled else 0
|
||||
row.allowed_host_ids_json = _dump_optional_list(allowed_host_ids)
|
||||
row.allowed_device_targets_json = _dump_optional_list(
|
||||
allowed_device_targets
|
||||
)
|
||||
row.updated_at = _iso(updated_at)
|
||||
session.flush()
|
||||
return _user_submission_policy_from_row(row)
|
||||
|
||||
def get_host_governance_policy(self, host_id: str) -> Any | None:
|
||||
with self._sessions() as session:
|
||||
row = session.get(HostGovernancePolicyRow, host_id)
|
||||
return _host_governance_policy_from_row(row) if row is not None else None
|
||||
|
||||
def upsert_host_governance_policy(
|
||||
self,
|
||||
*,
|
||||
host_id: str,
|
||||
self_submission_enabled: bool,
|
||||
max_active_tasks: int | None,
|
||||
daily_token_budget: int | None,
|
||||
updated_at: datetime,
|
||||
) -> Any:
|
||||
with self._sessions.begin() as session:
|
||||
if session.get(HostRow, host_id) is None:
|
||||
raise KeyError(f"unknown host {host_id!r}")
|
||||
row = session.get(HostGovernancePolicyRow, host_id)
|
||||
if row is None:
|
||||
row = HostGovernancePolicyRow(
|
||||
host_id=host_id,
|
||||
revision=1,
|
||||
self_submission_enabled=1 if self_submission_enabled else 0,
|
||||
max_active_tasks=max_active_tasks,
|
||||
daily_token_budget=daily_token_budget,
|
||||
updated_at=_iso(updated_at),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.revision += 1
|
||||
row.self_submission_enabled = 1 if self_submission_enabled else 0
|
||||
row.max_active_tasks = max_active_tasks
|
||||
row.daily_token_budget = daily_token_budget
|
||||
row.updated_at = _iso(updated_at)
|
||||
session.flush()
|
||||
return _host_governance_policy_from_row(row)
|
||||
|
||||
def list_reserved_device_ids(self, *, now: datetime) -> set[str]:
|
||||
with self._sessions() as session:
|
||||
device_ids = session.scalars(
|
||||
@@ -1218,6 +1300,8 @@ def _task_from_row(row: ScheduledTaskRow) -> Any:
|
||||
constraints=TaskConstraints(
|
||||
driver_type=constraints_data.get("driver_type"),
|
||||
capability_tags=list(constraints_data.get("capability_tags") or []),
|
||||
target_host_id=constraints_data.get("target_host_id"),
|
||||
target_device_id=constraints_data.get("target_device_id"),
|
||||
),
|
||||
status=row.status,
|
||||
assigned_device_id=row.assigned_device_id,
|
||||
@@ -1232,6 +1316,66 @@ def _task_from_row(row: ScheduledTaskRow) -> Any:
|
||||
)
|
||||
|
||||
|
||||
def _dump_optional_list(value: tuple[Any, ...] | None) -> str | None:
|
||||
return json.dumps(value, ensure_ascii=False) if value is not None else None
|
||||
|
||||
|
||||
def _load_optional_string_tuple(value: str | None) -> tuple[str, ...] | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
parsed = []
|
||||
return tuple(item for item in parsed if isinstance(item, str))
|
||||
|
||||
|
||||
def _load_optional_device_targets(
|
||||
value: str | None,
|
||||
) -> tuple[tuple[str, str], ...] | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
parsed = []
|
||||
return tuple(
|
||||
(item[0], item[1])
|
||||
for item in parsed
|
||||
if isinstance(item, list | tuple)
|
||||
and len(item) == 2
|
||||
and all(isinstance(part, str) for part in item)
|
||||
)
|
||||
|
||||
|
||||
def _user_submission_policy_from_row(row: UserSubmissionPolicyRow) -> Any:
|
||||
from cloud.governance import UserSubmissionPolicy
|
||||
|
||||
return UserSubmissionPolicy(
|
||||
user_id=row.user_id,
|
||||
revision=row.revision,
|
||||
submission_enabled=bool(row.submission_enabled),
|
||||
allowed_host_ids=_load_optional_string_tuple(row.allowed_host_ids_json),
|
||||
allowed_device_targets=_load_optional_device_targets(
|
||||
row.allowed_device_targets_json
|
||||
),
|
||||
updated_at=_parse_dt(row.updated_at) or utc_now(),
|
||||
)
|
||||
|
||||
|
||||
def _host_governance_policy_from_row(row: HostGovernancePolicyRow) -> Any:
|
||||
from cloud.governance import HostGovernancePolicy
|
||||
|
||||
return HostGovernancePolicy(
|
||||
host_id=row.host_id,
|
||||
revision=row.revision,
|
||||
self_submission_enabled=bool(row.self_submission_enabled),
|
||||
max_active_tasks=row.max_active_tasks,
|
||||
daily_token_budget=row.daily_token_budget,
|
||||
updated_at=_parse_dt(row.updated_at) or utc_now(),
|
||||
)
|
||||
|
||||
|
||||
def _task_attempt_from_row(row: TaskAttemptRow) -> Any:
|
||||
from cloud.repository import TaskAttemptRecord
|
||||
|
||||
|
||||
@@ -13,10 +13,18 @@ class ToolCallUnavailable(Exception):
|
||||
"""Internal signal for expected tool-calling transport/response failures."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolCallUsage:
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolCallDecision:
|
||||
tool_name: str
|
||||
arguments: dict[str, Any]
|
||||
usage: ToolCallUsage | None = None
|
||||
|
||||
|
||||
class ToolCallingClient(Protocol):
|
||||
@@ -234,7 +242,11 @@ def _decision_from_anthropic_response(response: Any) -> ToolCallDecision:
|
||||
name = _value(block, "name")
|
||||
arguments = _value(block, "input")
|
||||
if isinstance(name, str) and isinstance(arguments, dict):
|
||||
return ToolCallDecision(tool_name=name, arguments=arguments)
|
||||
return ToolCallDecision(
|
||||
tool_name=name,
|
||||
arguments=arguments,
|
||||
usage=_anthropic_usage(response),
|
||||
)
|
||||
raise ValueError("anthropic response did not include a tool_use block")
|
||||
|
||||
|
||||
@@ -278,7 +290,40 @@ def _decision_from_openai_response(response: Any) -> ToolCallDecision:
|
||||
if not isinstance(name, str):
|
||||
raise ValueError("openai tool call missing a function name")
|
||||
arguments = _decode_openai_arguments(_value(function, "arguments"))
|
||||
return ToolCallDecision(tool_name=name, arguments=arguments)
|
||||
return ToolCallDecision(
|
||||
tool_name=name,
|
||||
arguments=arguments,
|
||||
usage=_openai_usage(response),
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_usage(response: Any) -> ToolCallUsage | None:
|
||||
usage = _value(response, "usage")
|
||||
input_tokens = _integer_value(usage, "input_tokens")
|
||||
output_tokens = _integer_value(usage, "output_tokens")
|
||||
if input_tokens is None and output_tokens is None:
|
||||
return None
|
||||
return ToolCallUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=(input_tokens or 0) + (output_tokens or 0),
|
||||
)
|
||||
|
||||
|
||||
def _openai_usage(response: Any) -> ToolCallUsage | None:
|
||||
usage = _value(response, "usage")
|
||||
input_tokens = _integer_value(usage, "prompt_tokens")
|
||||
output_tokens = _integer_value(usage, "completion_tokens")
|
||||
total_tokens = _integer_value(usage, "total_tokens")
|
||||
if input_tokens is None and output_tokens is None and total_tokens is None:
|
||||
return None
|
||||
return ToolCallUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens
|
||||
if total_tokens is not None
|
||||
else (input_tokens or 0) + (output_tokens or 0),
|
||||
)
|
||||
|
||||
|
||||
def _decode_openai_arguments(raw_arguments: Any) -> dict[str, Any]:
|
||||
@@ -296,3 +341,8 @@ def _value(source: Any, key: str) -> Any:
|
||||
return source.get(key)
|
||||
value = getattr(source, key, None)
|
||||
return None if callable(value) else value
|
||||
|
||||
|
||||
def _integer_value(source: Any, key: str) -> int | None:
|
||||
value = _value(source, key)
|
||||
return value if isinstance(value, int) and value >= 0 else None
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from cloud.auth import Principal
|
||||
from cloud.config import CloudConfig
|
||||
from cloud.database import CloudDatabase
|
||||
from cloud.governance import enforce_user_submission_policy
|
||||
from cloud.plugins import PluginRegistry
|
||||
from cloud.pool import DevicePool
|
||||
from cloud.sdk.api import create_cloud_router
|
||||
from cloud.sdk.governance_api import create_governance_router
|
||||
from cloud.scheduler import TaskScheduler
|
||||
from cloud.user_auth import UserAccount
|
||||
from core.models import Device
|
||||
|
||||
|
||||
pytest.importorskip("fastapi")
|
||||
from fastapi import FastAPI # noqa: E402
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
class _PrincipalProvider:
|
||||
def __init__(self, principal: Principal) -> None:
|
||||
self.principal = principal
|
||||
|
||||
def authenticate(self, _request: object) -> Principal:
|
||||
return self.principal
|
||||
|
||||
|
||||
def _config() -> CloudConfig:
|
||||
return CloudConfig(max_queue_depth=20)
|
||||
|
||||
|
||||
def _add_user(repository, user_id: str = "user-a") -> None:
|
||||
now = datetime.now(UTC)
|
||||
repository.create_user(
|
||||
UserAccount(
|
||||
id=user_id,
|
||||
username=user_id,
|
||||
username_normalized=user_id,
|
||||
display_name=user_id,
|
||||
role="operator",
|
||||
enabled=True,
|
||||
must_change_password=False,
|
||||
authentication_version=1,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
password_hash="not-used-by-this-test",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_user_submission_policy_requires_permitted_explicit_target(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'governance.sqlite3').as_posix()}")
|
||||
try:
|
||||
_add_user(database.repository)
|
||||
database.repository.upsert_user_submission_policy(
|
||||
user_id="user-a",
|
||||
submission_enabled=True,
|
||||
allowed_host_ids=("host-a",),
|
||||
allowed_device_targets=(("host-a", "device-a"),),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
pool = DevicePool(database.repository, _config())
|
||||
pool.sync_host_devices(
|
||||
"host-a",
|
||||
[Device(id="device-a", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||
)
|
||||
scheduler = TaskScheduler(pool, database.repository, _config())
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
create_cloud_router(
|
||||
pool=pool,
|
||||
scheduler=scheduler,
|
||||
plugin_registry=PluginRegistry(database.repository),
|
||||
auth_provider=_PrincipalProvider(
|
||||
Principal(id="user:user-a", scopes=frozenset({"tasks:submit"}))
|
||||
),
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
|
||||
denied = client.post("/v1/tasks", json={"goal": "unscoped"})
|
||||
assert denied.status_code == 403
|
||||
allowed = client.post(
|
||||
"/v1/tasks",
|
||||
json={
|
||||
"goal": "scoped",
|
||||
"constraints": {
|
||||
"target_host_id": "host-a",
|
||||
"target_device_id": "device-a",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert allowed.status_code == 201, allowed.text
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_governance_routes_persist_revisioned_policies(tmp_path) -> None:
|
||||
database = CloudDatabase(f"sqlite:///{(tmp_path / 'governance-api.sqlite3').as_posix()}")
|
||||
try:
|
||||
_add_user(database.repository)
|
||||
database.repository.upsert_host(
|
||||
"host-a", address=None, last_seen_at=datetime.now(UTC)
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
create_governance_router(
|
||||
repository=database.repository,
|
||||
auth_provider=_PrincipalProvider(
|
||||
Principal(id="admin", scopes=frozenset({"governance:admin", "governance:read"}))
|
||||
),
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
|
||||
user_policy = client.put(
|
||||
"/v1/users/user-a/submission-policy",
|
||||
json={"submission_enabled": False, "allowed_host_ids": []},
|
||||
)
|
||||
assert user_policy.status_code == 200, user_policy.text
|
||||
assert user_policy.json()["revision"] == 1
|
||||
host_policy = client.put(
|
||||
"/v1/hosts/host-a/governance-policy",
|
||||
json={"max_active_tasks": 2, "daily_token_budget": 1000},
|
||||
)
|
||||
assert host_policy.status_code == 200, host_policy.text
|
||||
assert host_policy.json()["revision"] == 1
|
||||
reread = client.get("/v1/hosts/host-a/governance-policy")
|
||||
assert reread.json()["daily_token_budget"] == 1000
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_absent_policy_preserves_existing_submission_behavior() -> None:
|
||||
enforce_user_submission_policy(None, target_host_id=None, target_device_id=None)
|
||||
@@ -267,6 +267,70 @@ def test_submit_with_constraints(tmp_path) -> None:
|
||||
assert status["status"] == "queued"
|
||||
|
||||
|
||||
def test_submit_with_explicit_target_is_listed_and_not_rerouted(tmp_path) -> None:
|
||||
app, pool, scheduler, _ = _build_app(tmp_path)
|
||||
pool.sync_host_devices(
|
||||
"host-a",
|
||||
[Device(id="device-a", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||
)
|
||||
pool.sync_host_devices(
|
||||
"host-b",
|
||||
[Device(id="device-b", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||
)
|
||||
client = _client_for(app)
|
||||
|
||||
response = client.post(
|
||||
"/v1/tasks",
|
||||
json={
|
||||
"goal": "target b",
|
||||
"constraints": {
|
||||
"target_host_id": "host-b",
|
||||
"target_device_id": "device-b",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
task_id = response.json()["task_id"]
|
||||
|
||||
scheduler.assign()
|
||||
task = client.get(f"/v1/tasks/{task_id}").json()
|
||||
assert task["target_host_id"] == "host-b"
|
||||
assert task["target_device_id"] == "device-b"
|
||||
assert task["assigned_host_id"] == "host-b"
|
||||
assert task["assigned_device_id"] == "device-b"
|
||||
listed = client.get("/v1/tasks").json()["items"]
|
||||
assert next(item for item in listed if item["id"] == task_id)["target_host_id"] == "host-b"
|
||||
|
||||
|
||||
def test_submit_rejects_incomplete_or_foreign_target(tmp_path) -> None:
|
||||
app, pool, _, _ = _build_app(tmp_path)
|
||||
pool.sync_host_devices(
|
||||
"host-a",
|
||||
[Device(id="device-a", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||
)
|
||||
client = _client_for(app)
|
||||
|
||||
missing_host = client.post(
|
||||
"/v1/tasks",
|
||||
json={
|
||||
"goal": "invalid",
|
||||
"constraints": {"target_device_id": "device-a"},
|
||||
},
|
||||
)
|
||||
assert missing_host.status_code == 400
|
||||
foreign_device = client.post(
|
||||
"/v1/tasks",
|
||||
json={
|
||||
"goal": "invalid",
|
||||
"constraints": {
|
||||
"target_host_id": "host-a",
|
||||
"target_device_id": "unknown",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert foreign_device.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "path", "payload", "required_scope"),
|
||||
[
|
||||
|
||||
@@ -10,7 +10,7 @@ from cloud.auth import BearerCredential, ConfiguredBearerAuthProvider
|
||||
from cloud.config import CloudConfig
|
||||
from cloud.internal_api.api import create_internal_router
|
||||
from cloud.pool import DevicePool, PooledDevice
|
||||
from cloud.scheduler import ScheduledTask, TaskConstraints
|
||||
from cloud.scheduler import ScheduledTask, TaskConstraints, TaskScheduler
|
||||
from cloud.store import CloudStore
|
||||
|
||||
|
||||
@@ -34,7 +34,13 @@ def _build_client(tmp_path) -> tuple[TestClient, DevicePool]:
|
||||
]
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(create_internal_router(pool=pool, auth_provider=auth_provider))
|
||||
app.include_router(
|
||||
create_internal_router(
|
||||
pool=pool,
|
||||
auth_provider=auth_provider,
|
||||
scheduler=TaskScheduler(pool, pool.store, CloudConfig(stale_after_seconds=60)),
|
||||
)
|
||||
)
|
||||
return TestClient(app), pool
|
||||
|
||||
|
||||
@@ -196,6 +202,36 @@ def test_host_token_cannot_submit_heartbeat_for_another_host(tmp_path) -> None:
|
||||
assert pool.store.get_host("host-b") is None
|
||||
|
||||
|
||||
def test_heartbeat_and_self_submission_preserve_host_isolation(tmp_path) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
headers = {"Authorization": "Bearer token-a"}
|
||||
heartbeat = client.put(
|
||||
"/internal/v1/hosts/host-a/heartbeat",
|
||||
headers=headers,
|
||||
json=_heartbeat_payload("host-a", "device-a"),
|
||||
)
|
||||
assert heartbeat.status_code == 200
|
||||
assert heartbeat.json()["policy_revision"] == 0
|
||||
assert heartbeat.json()["policy"] is None
|
||||
|
||||
created = client.post(
|
||||
"/internal/v1/hosts/host-a/tasks",
|
||||
headers=headers,
|
||||
json={"host_id": "host-a", "goal": "local work", "device_id": "device-a"},
|
||||
)
|
||||
assert created.status_code == 201, created.text
|
||||
task = pool.store.get_task(created.json()["task_id"])
|
||||
assert task is not None
|
||||
assert task.constraints.target_host_id == "host-a"
|
||||
assert task.constraints.target_device_id == "device-a"
|
||||
foreign = client.post(
|
||||
"/internal/v1/hosts/host-b/tasks",
|
||||
headers=headers,
|
||||
json={"host_id": "host-b", "goal": "forbidden"},
|
||||
)
|
||||
assert foreign.status_code == 403
|
||||
|
||||
|
||||
def test_long_poll_claim_returns_at_most_one_owned_assignment(tmp_path) -> None:
|
||||
client, pool = _build_client(tmp_path)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
@@ -225,3 +225,42 @@ def test_capability_tag_constraint_filters_candidates(tmp_path) -> None:
|
||||
assignments = scheduler.assign()
|
||||
assert [a.task_id for a in assignments] == [task_id]
|
||||
assert assignments[0].device_id == "dev-2"
|
||||
|
||||
|
||||
def test_explicit_host_and_device_target_is_a_hard_constraint(tmp_path) -> None:
|
||||
pool = _pool_with_devices(tmp_path, _device("host-a-device"), host_id="host-a")
|
||||
pool.sync_host_devices("host-b", [_device("host-b-device")])
|
||||
scheduler = TaskScheduler(pool, pool.store, _config())
|
||||
|
||||
task_id = scheduler.submit(
|
||||
goal="target host b",
|
||||
constraints=TaskConstraints(
|
||||
target_host_id="host-b",
|
||||
target_device_id="host-b-device",
|
||||
),
|
||||
)
|
||||
|
||||
assignments = scheduler.assign()
|
||||
|
||||
assert [(item.host_id, item.device_id) for item in assignments] == [
|
||||
("host-b", "host-b-device")
|
||||
]
|
||||
task = pool.store.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.constraints.target_host_id == "host-b"
|
||||
assert task.constraints.target_device_id == "host-b-device"
|
||||
|
||||
|
||||
def test_unavailable_explicit_target_is_never_rerouted(tmp_path) -> None:
|
||||
pool = _pool_with_devices(tmp_path, _device("host-a-device"), host_id="host-a")
|
||||
pool.sync_host_devices("host-b", [_device("host-b-device", status="busy")])
|
||||
scheduler = TaskScheduler(pool, pool.store, _config())
|
||||
task_id = scheduler.submit(
|
||||
goal="wait for host b",
|
||||
constraints=TaskConstraints(target_host_id="host-b"),
|
||||
)
|
||||
|
||||
assert scheduler.assign() == []
|
||||
task = pool.store.get_task(task_id)
|
||||
assert task is not None
|
||||
assert task.status == "queued"
|
||||
|
||||
Reference in New Issue
Block a user