This commit is contained in:
@@ -43,6 +43,7 @@ from cloud.pool import DevicePool
|
|||||||
from cloud.scheduler import TaskScheduler
|
from cloud.scheduler import TaskScheduler
|
||||||
from cloud.schema import require_current_schema
|
from cloud.schema import require_current_schema
|
||||||
from cloud.sdk.api import create_cloud_router
|
from cloud.sdk.api import create_cloud_router
|
||||||
|
from cloud.sdk.governance_api import create_governance_router
|
||||||
from cloud.sdk.user_api import create_user_auth_router
|
from cloud.sdk.user_api import create_user_auth_router
|
||||||
from cloud.user_auth import USER_CSRF_COOKIE, USER_SESSION_COOKIE, UserAuthService, UserAuthSettings
|
from cloud.user_auth import USER_CSRF_COOKIE, USER_SESSION_COOKIE, UserAuthService, UserAuthSettings
|
||||||
from core.models import utc_now
|
from core.models import utc_now
|
||||||
@@ -288,11 +289,18 @@ def create_app(
|
|||||||
config=control_config,
|
config=control_config,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
app.include_router(
|
||||||
|
create_governance_router(
|
||||||
|
repository=repository,
|
||||||
|
auth_provider=auth_provider,
|
||||||
|
)
|
||||||
|
)
|
||||||
app.include_router(
|
app.include_router(
|
||||||
create_internal_router(
|
create_internal_router(
|
||||||
pool=pool,
|
pool=pool,
|
||||||
auth_provider=auth_provider,
|
auth_provider=auth_provider,
|
||||||
lease_duration_seconds=control_config.lease_duration_seconds,
|
lease_duration_seconds=control_config.lease_duration_seconds,
|
||||||
|
scheduler=scheduler,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from cloud.internal_api.models import (
|
|||||||
DeviceSnapshotModel,
|
DeviceSnapshotModel,
|
||||||
HeartbeatResponse,
|
HeartbeatResponse,
|
||||||
HostEnrollmentResponse,
|
HostEnrollmentResponse,
|
||||||
|
HostTaskSubmissionResponse,
|
||||||
LeaseRenewalResponse,
|
LeaseRenewalResponse,
|
||||||
TerminalResultResponse,
|
TerminalResultResponse,
|
||||||
)
|
)
|
||||||
@@ -146,6 +147,7 @@ class HostAgentClient:
|
|||||||
devices: list[DeviceSnapshotModel],
|
devices: list[DeviceSnapshotModel],
|
||||||
*,
|
*,
|
||||||
address: str | None = None,
|
address: str | None = None,
|
||||||
|
policy_revision: int = 0,
|
||||||
) -> HeartbeatResponse:
|
) -> HeartbeatResponse:
|
||||||
response = await self._request(
|
response = await self._request(
|
||||||
"PUT",
|
"PUT",
|
||||||
@@ -154,10 +156,28 @@ class HostAgentClient:
|
|||||||
"host_id": self.config.host_id,
|
"host_id": self.config.host_id,
|
||||||
"address": address,
|
"address": address,
|
||||||
"devices": [device.model_dump(mode="json") for device in devices],
|
"devices": [device.model_dump(mode="json") for device in devices],
|
||||||
|
"policy_revision": policy_revision,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return HeartbeatResponse.model_validate(response.json())
|
return HeartbeatResponse.model_validate(response.json())
|
||||||
|
|
||||||
|
async def submit_self_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
goal: str,
|
||||||
|
device_id: str | None = None,
|
||||||
|
) -> HostTaskSubmissionResponse:
|
||||||
|
response = await self._request(
|
||||||
|
"POST",
|
||||||
|
f"/internal/v1/hosts/{self.config.host_id}/tasks",
|
||||||
|
json={
|
||||||
|
"host_id": self.config.host_id,
|
||||||
|
"goal": goal,
|
||||||
|
"device_id": device_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return HostTaskSubmissionResponse.model_validate(response.json())
|
||||||
|
|
||||||
async def claim(self) -> AssignmentModel | None:
|
async def claim(self) -> AssignmentModel | None:
|
||||||
response = await self._request(
|
response = await self._request(
|
||||||
"POST",
|
"POST",
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import httpx
|
|||||||
|
|
||||||
from cloud.internal_api.models import PlannerDecisionError, PlannerDecisionResponse
|
from cloud.internal_api.models import PlannerDecisionError, PlannerDecisionResponse
|
||||||
from host_agent.config import HostAgentConfig
|
from host_agent.config import HostAgentConfig
|
||||||
from runtime.tool_calling_client import ToolCallDecision, ToolCallUnavailable
|
from runtime.tool_calling_client import ToolCallDecision, ToolCallUnavailable, ToolCallUsage
|
||||||
from runtime.tool_specs import ToolSpec
|
from runtime.tool_specs import ToolSpec
|
||||||
|
|
||||||
|
|
||||||
@@ -85,6 +85,22 @@ class CloudProxyToolCallingClient:
|
|||||||
return ToolCallDecision(
|
return ToolCallDecision(
|
||||||
tool_name=decoded.tool_name,
|
tool_name=decoded.tool_name,
|
||||||
arguments=dict(decoded.arguments),
|
arguments=dict(decoded.arguments),
|
||||||
|
usage=(
|
||||||
|
ToolCallUsage(
|
||||||
|
input_tokens=decoded.input_tokens,
|
||||||
|
output_tokens=decoded.output_tokens,
|
||||||
|
total_tokens=decoded.total_tokens,
|
||||||
|
)
|
||||||
|
if any(
|
||||||
|
value is not None
|
||||||
|
for value in (
|
||||||
|
decoded.input_tokens,
|
||||||
|
decoded.output_tokens,
|
||||||
|
decoded.total_tokens,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
raise ToolCallUnavailable(_error_detail(response))
|
raise ToolCallUnavailable(_error_detail(response))
|
||||||
|
|
||||||
|
|||||||
@@ -45,13 +45,19 @@ class HeartbeatSynchronizer:
|
|||||||
self._sleep = sleep
|
self._sleep = sleep
|
||||||
self.status_tracker = status_tracker
|
self.status_tracker = status_tracker
|
||||||
self.on_sync = on_sync
|
self.on_sync = on_sync
|
||||||
|
self.policy_revision = 0
|
||||||
|
self.policy = None
|
||||||
|
|
||||||
async def sync_once(self) -> HeartbeatResponse:
|
async def sync_once(self) -> HeartbeatResponse:
|
||||||
snapshot = build_device_snapshot(self.manager)
|
snapshot = build_device_snapshot(self.manager)
|
||||||
response = await self.client.heartbeat(
|
response = await self.client.heartbeat(
|
||||||
snapshot,
|
snapshot,
|
||||||
address=self.address,
|
address=self.address,
|
||||||
|
policy_revision=self.policy_revision,
|
||||||
)
|
)
|
||||||
|
self.policy_revision = response.policy_revision
|
||||||
|
if response.policy is not None:
|
||||||
|
self.policy = response.policy
|
||||||
if self.status_tracker is not None:
|
if self.status_tracker is not None:
|
||||||
self.status_tracker.mark_heartbeat(ok=True, device_count=len(snapshot))
|
self.status_tracker.mark_heartbeat(ok=True, device_count=len(snapshot))
|
||||||
if self.on_sync is not None:
|
if self.on_sync is not None:
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ def test_heartbeat_synchronizer_runs_at_configured_interval_until_stopped() -> N
|
|||||||
calls: list[list[str]] = []
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
class FakeClient:
|
class FakeClient:
|
||||||
async def heartbeat(self, devices, *, address=None):
|
async def heartbeat(self, devices, *, address=None, policy_revision=0):
|
||||||
calls.append([device.device_id for device in devices])
|
calls.append([device.device_id for device in devices])
|
||||||
return HeartbeatResponse(
|
return HeartbeatResponse(
|
||||||
host_id="host-a",
|
host_id="host-a",
|
||||||
@@ -96,7 +96,7 @@ def test_sync_once_notifies_status_tracker_and_on_sync_with_device_count() -> No
|
|||||||
)
|
)
|
||||||
|
|
||||||
class FakeClient:
|
class FakeClient:
|
||||||
async def heartbeat(self, devices, *, address=None):
|
async def heartbeat(self, devices, *, address=None, policy_revision=0):
|
||||||
return HeartbeatResponse(
|
return HeartbeatResponse(
|
||||||
host_id="host-a",
|
host_id="host-a",
|
||||||
accepted_devices=len(devices),
|
accepted_devices=len(devices),
|
||||||
|
|||||||
@@ -32,6 +32,11 @@ const canAdminPlugins = computed(
|
|||||||
currentUser.value?.scopes.includes("*") ||
|
currentUser.value?.scopes.includes("*") ||
|
||||||
currentUser.value?.scopes.includes("plugins:admin"),
|
currentUser.value?.scopes.includes("plugins:admin"),
|
||||||
);
|
);
|
||||||
|
const canSubmitTasks = computed(
|
||||||
|
() =>
|
||||||
|
currentUser.value?.scopes.includes("*") ||
|
||||||
|
currentUser.value?.scopes.includes("tasks:submit"),
|
||||||
|
);
|
||||||
const isAuthenticated = computed(() => currentUser.value !== null);
|
const isAuthenticated = computed(() => currentUser.value !== null);
|
||||||
const currentUserLabel = computed(() =>
|
const currentUserLabel = computed(() =>
|
||||||
currentUser.value ? `${currentUser.value.display_name} (${currentUser.value.role})` : "",
|
currentUser.value ? `${currentUser.value.display_name} (${currentUser.value.role})` : "",
|
||||||
@@ -128,7 +133,7 @@ const activeComponent = computed(() => {
|
|||||||
</nav>
|
</nav>
|
||||||
<main class="app-main">
|
<main class="app-main">
|
||||||
<PluginsView v-if="activeView === 'plugins'" :can-admin="canAdminPlugins" />
|
<PluginsView v-if="activeView === 'plugins'" :can-admin="canAdminPlugins" />
|
||||||
<component v-else :is="activeComponent" />
|
<component v-else :is="activeComponent" :can-submit="canSubmitTasks" />
|
||||||
</main>
|
</main>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import type {
|
|||||||
PluginRegistrationPayload,
|
PluginRegistrationPayload,
|
||||||
TaskAttempt,
|
TaskAttempt,
|
||||||
TaskListResponse,
|
TaskListResponse,
|
||||||
|
TaskSubmissionPayload,
|
||||||
TaskStatus,
|
TaskStatus,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
@@ -135,6 +136,13 @@ export function getTaskAttempts(taskId: string): Promise<TaskAttempt[]> {
|
|||||||
return request<TaskAttempt[]>(`/v1/tasks/${encodeURIComponent(taskId)}/attempts`);
|
return request<TaskAttempt[]>(`/v1/tasks/${encodeURIComponent(taskId)}/attempts`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function submitTask(payload: TaskSubmissionPayload): Promise<{ task_id: string }> {
|
||||||
|
return request<{ task_id: string }>("/v1/tasks", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function listDevices(): Promise<DeviceRecord[]> {
|
export function listDevices(): Promise<DeviceRecord[]> {
|
||||||
return request<DeviceRecord[]>("/v1/devices");
|
return request<DeviceRecord[]>("/v1/devices");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,9 +14,22 @@ export interface TaskListItem {
|
|||||||
assigned_host_id: string | null;
|
assigned_host_id: string | null;
|
||||||
attempt_count: number;
|
attempt_count: number;
|
||||||
failure_reason: string | null;
|
failure_reason: string | null;
|
||||||
|
target_host_id: string | null;
|
||||||
|
target_device_id: string | null;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TaskSubmissionPayload {
|
||||||
|
goal?: string;
|
||||||
|
workflow_definition_id?: string;
|
||||||
|
constraints?: {
|
||||||
|
driver_type?: string;
|
||||||
|
capability_tags?: string[];
|
||||||
|
target_host_id?: string;
|
||||||
|
target_device_id?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface TaskListResponse {
|
export interface TaskListResponse {
|
||||||
items: TaskListItem[];
|
items: TaskListItem[];
|
||||||
total: number;
|
total: number;
|
||||||
|
|||||||
@@ -1,18 +1,25 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, ref, watch } from "vue";
|
import { computed, onMounted, ref, watch } from "vue";
|
||||||
import { LoaderCircle, RefreshCw } from "@lucide/vue";
|
import { LoaderCircle, RefreshCw } from "@lucide/vue";
|
||||||
import {
|
import {
|
||||||
CloudApiError,
|
CloudApiError,
|
||||||
getTaskAttempts,
|
getTaskAttempts,
|
||||||
listTasks,
|
listTasks,
|
||||||
|
listDevices,
|
||||||
|
listHosts,
|
||||||
|
submitTask,
|
||||||
} from "../api";
|
} from "../api";
|
||||||
import type {
|
import type {
|
||||||
TaskAttempt,
|
TaskAttempt,
|
||||||
TaskListItem,
|
TaskListItem,
|
||||||
TaskListResponse,
|
TaskListResponse,
|
||||||
TaskStatus,
|
TaskStatus,
|
||||||
|
DeviceRecord,
|
||||||
|
HostRecord,
|
||||||
} from "../types";
|
} from "../types";
|
||||||
|
|
||||||
|
const props = defineProps<{ canSubmit: boolean }>();
|
||||||
|
|
||||||
const STATUSES: TaskStatus[] = [
|
const STATUSES: TaskStatus[] = [
|
||||||
"queued",
|
"queued",
|
||||||
"assigned",
|
"assigned",
|
||||||
@@ -31,16 +38,36 @@ const selectedTask = ref<TaskListItem | null>(null);
|
|||||||
const attempts = ref<TaskAttempt[]>([]);
|
const attempts = ref<TaskAttempt[]>([]);
|
||||||
const attemptsLoading = ref(false);
|
const attemptsLoading = ref(false);
|
||||||
const attemptsError = ref("");
|
const attemptsError = ref("");
|
||||||
|
const composerOpen = ref(false);
|
||||||
|
const submitGoal = ref("");
|
||||||
|
const submitWorkflow = ref("");
|
||||||
|
const submitHostId = ref("");
|
||||||
|
const submitDeviceId = ref("");
|
||||||
|
const submitDriverType = ref("");
|
||||||
|
const submitCapabilityTags = ref("");
|
||||||
|
const submitting = ref(false);
|
||||||
|
const hosts = ref<HostRecord[]>([]);
|
||||||
|
const devices = ref<DeviceRecord[]>([]);
|
||||||
|
const availableDevices = computed(() =>
|
||||||
|
devices.value.filter((device) => device.host_id === submitHostId.value),
|
||||||
|
);
|
||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
loading.value = true;
|
loading.value = true;
|
||||||
errorMessage.value = "";
|
errorMessage.value = "";
|
||||||
try {
|
try {
|
||||||
result.value = await listTasks({
|
const [tasks, loadedHosts, loadedDevices] = await Promise.all([
|
||||||
|
listTasks({
|
||||||
status: statusFilter.value === "" ? undefined : statusFilter.value,
|
status: statusFilter.value === "" ? undefined : statusFilter.value,
|
||||||
limit: pageSize.value,
|
limit: pageSize.value,
|
||||||
offset: offset.value,
|
offset: offset.value,
|
||||||
});
|
}),
|
||||||
|
listHosts(),
|
||||||
|
listDevices(),
|
||||||
|
]);
|
||||||
|
result.value = tasks;
|
||||||
|
hosts.value = loadedHosts;
|
||||||
|
devices.value = loadedDevices;
|
||||||
if (selectedTask.value) {
|
if (selectedTask.value) {
|
||||||
const stillPresent = result.value.items.find(
|
const stillPresent = result.value.items.find(
|
||||||
(item) => item.id === selectedTask.value?.id,
|
(item) => item.id === selectedTask.value?.id,
|
||||||
@@ -57,6 +84,47 @@ async function refresh() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function createTask() {
|
||||||
|
const goal = submitGoal.value.trim();
|
||||||
|
const workflow = submitWorkflow.value.trim();
|
||||||
|
if (!goal && !workflow) {
|
||||||
|
errorMessage.value = "provide a goal or workflow id";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
submitting.value = true;
|
||||||
|
errorMessage.value = "";
|
||||||
|
try {
|
||||||
|
const hostId = submitHostId.value || undefined;
|
||||||
|
const deviceId = submitDeviceId.value || undefined;
|
||||||
|
await submitTask({
|
||||||
|
goal: goal || undefined,
|
||||||
|
workflow_definition_id: workflow || undefined,
|
||||||
|
constraints: {
|
||||||
|
driver_type: submitDriverType.value || undefined,
|
||||||
|
capability_tags: submitCapabilityTags.value
|
||||||
|
.split(",")
|
||||||
|
.map((tag) => tag.trim())
|
||||||
|
.filter(Boolean),
|
||||||
|
target_host_id: hostId,
|
||||||
|
target_device_id: deviceId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
composerOpen.value = false;
|
||||||
|
submitGoal.value = "";
|
||||||
|
submitWorkflow.value = "";
|
||||||
|
submitHostId.value = "";
|
||||||
|
submitDeviceId.value = "";
|
||||||
|
submitDriverType.value = "";
|
||||||
|
submitCapabilityTags.value = "";
|
||||||
|
offset.value = 0;
|
||||||
|
await refresh();
|
||||||
|
} catch (err) {
|
||||||
|
handleError(err, "failed to create task");
|
||||||
|
} finally {
|
||||||
|
submitting.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function selectTask(task: TaskListItem) {
|
async function selectTask(task: TaskListItem) {
|
||||||
selectedTask.value = task;
|
selectedTask.value = task;
|
||||||
attempts.value = [];
|
attempts.value = [];
|
||||||
@@ -113,6 +181,11 @@ watch(pageSize, () => {
|
|||||||
refresh();
|
refresh();
|
||||||
});
|
});
|
||||||
watch(offset, refresh);
|
watch(offset, refresh);
|
||||||
|
watch(submitHostId, () => {
|
||||||
|
if (!availableDevices.value.some((device) => device.device_id === submitDeviceId.value)) {
|
||||||
|
submitDeviceId.value = "";
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
onMounted(refresh);
|
onMounted(refresh);
|
||||||
|
|
||||||
@@ -145,6 +218,9 @@ function formatTerminalResult(attempt: TaskAttempt): string {
|
|||||||
<div>
|
<div>
|
||||||
<div class="toolbar">
|
<div class="toolbar">
|
||||||
<h2>Tasks</h2>
|
<h2>Tasks</h2>
|
||||||
|
<button v-if="props.canSubmit" @click="composerOpen = !composerOpen">
|
||||||
|
{{ composerOpen ? "Cancel create" : "Create task" }}
|
||||||
|
</button>
|
||||||
<label>
|
<label>
|
||||||
Status
|
Status
|
||||||
<select v-model="statusFilter">
|
<select v-model="statusFilter">
|
||||||
@@ -172,6 +248,29 @@ function formatTerminalResult(attempt: TaskAttempt): string {
|
|||||||
|
|
||||||
<div v-if="errorMessage" class="notice error">{{ errorMessage }}</div>
|
<div v-if="errorMessage" class="notice error">{{ errorMessage }}</div>
|
||||||
|
|
||||||
|
<form v-if="composerOpen" class="panel" @submit.prevent="createTask">
|
||||||
|
<h3>Create task</h3>
|
||||||
|
<label>Goal <input v-model="submitGoal" placeholder="Describe the device task" /></label>
|
||||||
|
<label>Workflow ID <input v-model="submitWorkflow" placeholder="Optional workflow reference" /></label>
|
||||||
|
<label>
|
||||||
|
Target Host
|
||||||
|
<select v-model="submitHostId">
|
||||||
|
<option value="">Any eligible host</option>
|
||||||
|
<option v-for="host in hosts" :key="host.host_id" :value="host.host_id">{{ host.host_id }}</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
Target Device
|
||||||
|
<select v-model="submitDeviceId" :disabled="!submitHostId">
|
||||||
|
<option value="">Any eligible device on selected host</option>
|
||||||
|
<option v-for="device in availableDevices" :key="device.device_id" :value="device.device_id">{{ device.device_id }}</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label>Driver type <input v-model="submitDriverType" placeholder="Optional, e.g. wda" /></label>
|
||||||
|
<label>Capability tags <input v-model="submitCapabilityTags" placeholder="Optional, comma separated" /></label>
|
||||||
|
<div class="actions"><button type="submit" :disabled="submitting">{{ submitting ? "Creating…" : "Create" }}</button></div>
|
||||||
|
</form>
|
||||||
|
|
||||||
<div class="panel" v-if="!selectedTask">
|
<div class="panel" v-if="!selectedTask">
|
||||||
<table v-if="result && result.items.length">
|
<table v-if="result && result.items.length">
|
||||||
<thead>
|
<thead>
|
||||||
@@ -212,6 +311,9 @@ function formatTerminalResult(attempt: TaskAttempt): string {
|
|||||||
{{ task.assigned_device_id }}
|
{{ task.assigned_device_id }}
|
||||||
<span class="dim">on {{ task.assigned_host_id }}</span>
|
<span class="dim">on {{ task.assigned_host_id }}</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div v-else-if="task.target_host_id" class="dim">
|
||||||
|
target: {{ task.target_host_id }}<span v-if="task.target_device_id"> / {{ task.target_device_id }}</span>
|
||||||
|
</div>
|
||||||
<div v-else class="dim">unassigned</div>
|
<div v-else class="dim">unassigned</div>
|
||||||
</td>
|
</td>
|
||||||
<td>{{ task.attempt_count }}</td>
|
<td>{{ task.attempt_count }}</td>
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
schema: spec-driven
|
||||||
|
created: 2026-07-13
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
## Context
|
||||||
|
|
||||||
|
The Cloud Control Plane already persists scheduled tasks, pooled devices,
|
||||||
|
enrolled Hosts, user accounts, and user sessions. Public task submission has
|
||||||
|
only driver and capability-tag constraints, while the Console is primarily a
|
||||||
|
read dashboard. The Host Agent uses only outbound internal routes and can
|
||||||
|
heartbeat, claim, renew, and report work; it cannot submit a task. No durable
|
||||||
|
policy or AI usage ledger exists.
|
||||||
|
|
||||||
|
`cloud-planner-proxy` is the prerequisite for enforceable AI budgets. It
|
||||||
|
places every planner call from a Host using `AI_PLANNER_TRANSPORT=cloud` behind
|
||||||
|
an authenticated Cloud endpoint and leaves direct provider calls supported.
|
||||||
|
This change is deliberately scheduled after that change: a direct provider
|
||||||
|
call cannot be authoritatively measured or stopped by the Cloud.
|
||||||
|
|
||||||
|
The Cloud Platform remains the durable application layer. Public/internal
|
||||||
|
HTTP adapters, the Vue Console, and the Host Agent are outer adapters; no
|
||||||
|
HTTP, Cloud, or LLM dependency is introduced into `core`, `driver`, `device`,
|
||||||
|
or `tools`.
|
||||||
|
|
||||||
|
## Goals / Non-Goals
|
||||||
|
|
||||||
|
**Goals:**
|
||||||
|
|
||||||
|
- Let authorized Console users create goal or workflow tasks for an explicit
|
||||||
|
Host and optionally an explicit Device, without a fallback target.
|
||||||
|
- Make human task-submission permissions and target restrictions durable,
|
||||||
|
auditable, and API-enforced.
|
||||||
|
- Let an authenticated Host receive its effective versioned policy and create
|
||||||
|
goal tasks that are irrevocably constrained to itself.
|
||||||
|
- Account for provider-reported token use and enforce hard per-Host budgets
|
||||||
|
for Cloud-proxied planner calls, including concurrent requests.
|
||||||
|
- Display effective restrictions, budget status, and non-secret usage in the
|
||||||
|
Console.
|
||||||
|
|
||||||
|
**Non-Goals:**
|
||||||
|
|
||||||
|
- Custom roles, tenant isolation, task-level read ACLs, invitations, or an
|
||||||
|
external identity provider. Existing fixed roles remain the coarse access
|
||||||
|
control layer.
|
||||||
|
- General remote Host control, inbound connections, or allowing one Host to
|
||||||
|
enqueue work for another Host.
|
||||||
|
- Charging, invoice generation, provider price catalogues, prompt/screenshot
|
||||||
|
retention, or an attempt to hard-limit direct-to-provider planner calls.
|
||||||
|
- Cloud workflow-definition distribution. Host-self submission is goal-only
|
||||||
|
because a workflow definition is still stored locally on each Host.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
### D1: Use explicit target selectors inside task constraints
|
||||||
|
|
||||||
|
`TaskConstraints` gains an optional target object with `host_id` and optional
|
||||||
|
`device_id`. A device target is invalid without its Host; a Host-only target
|
||||||
|
means any eligible device on that Host; an absent target preserves broad pool
|
||||||
|
scheduling. The existing serialized constraints field stores this additive
|
||||||
|
shape, while API status/list responses expose it directly.
|
||||||
|
|
||||||
|
The scheduler applies target filtering before the named assignment strategy,
|
||||||
|
in addition to driver type and capability tags. A missing, stale, busy, or
|
||||||
|
temporarily incompatible selected device leaves the task queued. It never
|
||||||
|
falls back to another target. The submission service validates a named Host
|
||||||
|
and Device against the known pool/enrollment ownership at creation time.
|
||||||
|
|
||||||
|
Using only a globally unique `device_id` was rejected because repository and
|
||||||
|
pool ownership are Host/device pairs; retaining the Host in the target makes
|
||||||
|
authorization and audit unambiguous. Duplicating target columns outside the
|
||||||
|
existing constraints payload was rejected for v1 because matching is already
|
||||||
|
performed after task retrieval and there is no target-indexed query path.
|
||||||
|
|
||||||
|
### D2: Compose fixed role scopes with a separate restrictive user policy
|
||||||
|
|
||||||
|
Keep `viewer`, `operator`, and `admin` roles and their existing scope mapping.
|
||||||
|
Add an optional `UserSubmissionPolicy` keyed by user id with a version,
|
||||||
|
`submission_enabled`, and tri-state Host/Device allow-lists (`null` means
|
||||||
|
unrestricted; an empty stored list means no permitted targets). A restrictive
|
||||||
|
policy requires an explicit target within its allow-list, preventing an
|
||||||
|
un-targeted task from escaping a user's permitted Hosts.
|
||||||
|
|
||||||
|
Public task submission first requires `tasks:submit`, then evaluates the
|
||||||
|
authenticated human principal's policy. The same policy applies to an
|
||||||
|
administrator's ordinary task submission; an administrator can change policy
|
||||||
|
through governance administration but does not silently bypass it. Policies
|
||||||
|
do not change task-read visibility in this increment.
|
||||||
|
|
||||||
|
Storing arbitrary policy JSON was rejected: it makes safe validation,
|
||||||
|
migration, and Console editing ambiguous. Adding a custom role/permission
|
||||||
|
model was rejected because fixed roles plus a narrow restrictive policy cover
|
||||||
|
the requested near-term controls without lockout complexity.
|
||||||
|
|
||||||
|
### D3: Store versioned Host policies and synchronize them in heartbeat responses
|
||||||
|
|
||||||
|
`HostGovernancePolicy` is keyed by Host id and has a monotonic revision. V1
|
||||||
|
contains an optional maximum active-task count and an optional UTC-day token
|
||||||
|
budget. The Cloud scheduler enforces active-task limits; the budget is
|
||||||
|
enforced only by the planner proxy. No policy record means no new limit,
|
||||||
|
preserving existing deployments.
|
||||||
|
|
||||||
|
Heartbeat requests carry the last policy revision applied by the Host. A
|
||||||
|
heartbeat response includes the effective complete policy when the revision is
|
||||||
|
different, otherwise only confirms its current revision. The Host persists a
|
||||||
|
safe cached policy snapshot for local status visibility, but the Cloud remains
|
||||||
|
the authority for task assignment and AI spend. This pull-on-existing-
|
||||||
|
heartbeat design keeps the outbound-only Host model and avoids a parallel
|
||||||
|
long-poll channel.
|
||||||
|
|
||||||
|
Pushing policy through a new inbound Host listener was rejected because it
|
||||||
|
breaks NAT deployments. Requiring a policy fetch before every planner call
|
||||||
|
was rejected because the proxy already performs the authoritative budget check
|
||||||
|
and would add avoidable latency.
|
||||||
|
|
||||||
|
### D4: Meter Cloud-proxy usage with reservation then settlement
|
||||||
|
|
||||||
|
After `cloud-planner-proxy` is complete, provider clients return a
|
||||||
|
non-secret usage object together with the existing tool-call decision. It
|
||||||
|
contains provider/model identity and provider-reported input, output, and
|
||||||
|
total token counts when available; `AIPlanner` continues to consume only the
|
||||||
|
decision.
|
||||||
|
|
||||||
|
Before the proxy invokes a provider for a Host with a token budget, it atomically
|
||||||
|
creates a budget reservation for a conservative per-decision ceiling within the
|
||||||
|
Host's UTC-day bucket. The Cloud configuration also applies a matching
|
||||||
|
provider output ceiling. If used plus reserved tokens would exceed the
|
||||||
|
budget, the endpoint rejects before invoking the provider. On a provider
|
||||||
|
response, the reservation is settled to actual reported usage and an immutable
|
||||||
|
usage event is recorded. If the transport outcome leaves usage unknown, the
|
||||||
|
reservation remains until a bounded expiry cleanup rather than being released
|
||||||
|
optimistically.
|
||||||
|
|
||||||
|
The proxy request carries Host execution context (task id, attempt, and lease
|
||||||
|
when one exists). A Host-Agent-local context adapter supplies that metadata to
|
||||||
|
the cloud client without changing `AIPlanner` or introducing Host imports into
|
||||||
|
`runtime`. Locally initiated/non-assignment calls record Host-level usage with
|
||||||
|
the assignment fields absent.
|
||||||
|
|
||||||
|
Post-hoc usage-only accounting was rejected because concurrent calls can
|
||||||
|
overspend a limit before aggregation. Client-side budget checks were rejected
|
||||||
|
because the Host is not the trust boundary. Direct transport is represented
|
||||||
|
as `unmetered`, not as zero usage or a compliant hard-budget path.
|
||||||
|
|
||||||
|
### D5: Give Hosts a narrow self-submission operation
|
||||||
|
|
||||||
|
Add `POST /internal/v1/hosts/{host_id}/tasks` under existing host-scoped
|
||||||
|
authentication. The route requires the authenticated Host to equal the URL
|
||||||
|
Host, accepts a goal and optional local Device target, and constructs the
|
||||||
|
Cloud-side target itself. A supplied Device must be enrolled/owned by that
|
||||||
|
Host. Host policy can disable self-submission; human user policies do not
|
||||||
|
apply because there is no human principal.
|
||||||
|
|
||||||
|
The endpoint accepts no workflow reference and no arbitrary Host target.
|
||||||
|
Giving an enrolled Host the public `tasks:submit` scope was rejected because
|
||||||
|
it would allow cross-Host task submission and blur service credentials with
|
||||||
|
human authorization.
|
||||||
|
|
||||||
|
### D6: Keep governance administration explicit in the Console and SDK
|
||||||
|
|
||||||
|
Add public, scope-protected governance routes for reading/updating user
|
||||||
|
submission policy, Host policy, and non-secret Host AI usage summaries/events.
|
||||||
|
`governance:read` and `governance:admin` are new scopes; fixed user roles need
|
||||||
|
no mapping change because administrators retain `*`, while scoped bearer
|
||||||
|
automation can receive just these scopes. Console task creation renders only
|
||||||
|
when `tasks:submit` is present. Policy and usage administration render only
|
||||||
|
for `governance:admin`; backend scope and policy checks are authoritative.
|
||||||
|
|
||||||
|
Every policy change records a safe audit event with actor, target, old/new
|
||||||
|
revision, and non-secret values. The Console clears any password inputs from
|
||||||
|
the existing Users flow and never displays provider credentials, prompts,
|
||||||
|
screenshots, cookies, or lease secrets.
|
||||||
|
|
||||||
|
## Risks / Trade-offs
|
||||||
|
|
||||||
|
- [Planner proxy is incomplete or a Host remains on direct transport] →
|
||||||
|
Token limits are not activated for that Host; the Console labels it
|
||||||
|
unmetered and the implementation gate prevents treating it as compliant.
|
||||||
|
- [Provider usage is unavailable after a transport failure] → Retain the
|
||||||
|
conservative reservation until expiry; this favors cost safety over
|
||||||
|
temporary under-utilization.
|
||||||
|
- [Policy migration or stale Console state changes a user's permitted target]
|
||||||
|
→ Enforce against current Cloud policy at every submission and return a
|
||||||
|
clear authorization failure; UI state is only advisory.
|
||||||
|
- [Targeted Hosts/devices are offline] → Keep the task queued and visible as
|
||||||
|
targeted rather than rerouting it; operators can edit or cancel only when a
|
||||||
|
later lifecycle capability supplies those operations.
|
||||||
|
- [Policy tables increase schema and operational complexity] → Use one normal
|
||||||
|
forward migration, transactional repository operations, and the existing
|
||||||
|
SQLite/PostgreSQL contract.
|
||||||
|
|
||||||
|
## Migration Plan
|
||||||
|
|
||||||
|
1. Do not start implementation until `cloud-planner-proxy` has completed its
|
||||||
|
tasks, passed strict validation and its runtime tests, and the active
|
||||||
|
Cloud Console user-authentication change has been reconciled and verified.
|
||||||
|
2. Add the forward schema migration for policies, audit records, usage events,
|
||||||
|
and reservations. Existing tasks remain valid because no target and no
|
||||||
|
policy continue to mean unrestricted behavior.
|
||||||
|
3. Deploy the Cloud API with policy/target routes and scheduler checks, then
|
||||||
|
deploy compatible Hosts. Hosts learn policies on their next heartbeat;
|
||||||
|
older Hosts continue normal assignment but cannot self-submit.
|
||||||
|
4. Enable Cloud planner transport on a pilot Host, configure a budget, and
|
||||||
|
verify proxy reservations and settlement before assigning budgets across
|
||||||
|
the fleet. Direct Hosts remain visibly unmetered.
|
||||||
|
5. Deploy the Console after the API. Rollback removes policy assignments or
|
||||||
|
returns affected Hosts to direct transport only when local provider
|
||||||
|
credentials are available; preserve governance and usage rows for a
|
||||||
|
forward fix rather than destructively downgrading production data.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
- V1 uses UTC-day token budgets. A later change can add calendar-month or
|
||||||
|
rolling windows without weakening the reservation/settlement contract.
|
||||||
|
- Token cost is intentionally excluded because provider pricing and cached
|
||||||
|
token semantics vary; usage events retain provider/model fields for a later
|
||||||
|
pricing layer.
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
## Why
|
||||||
|
|
||||||
|
The Cloud Console can currently inspect distributed work but cannot direct
|
||||||
|
that work to a selected Host or Device, govern who may submit it, or explain
|
||||||
|
and control the AI spend generated by each Host. Now that
|
||||||
|
`cloud-planner-proxy` is planned to centralize the LLM call path, the next
|
||||||
|
control-plane increment can make those operational decisions enforceable at
|
||||||
|
the Cloud boundary instead of trusting every edge process to self-govern.
|
||||||
|
|
||||||
|
## What Changes
|
||||||
|
|
||||||
|
- Add Console task creation for authorized operators, with explicit target
|
||||||
|
selectors for a Host and optionally one of that Host's Devices. A selected
|
||||||
|
target is a hard scheduling constraint and is never silently rerouted to a
|
||||||
|
different Host or Device.
|
||||||
|
- Add persisted Cloud governance policies: user submission restrictions
|
||||||
|
(submission enablement and allowed Host/Device targets) and versioned
|
||||||
|
per-Host operational/AI-budget policies. The API remains authoritative;
|
||||||
|
Console visibility is not an authorization boundary.
|
||||||
|
- Let a Host retrieve its current policy through the authenticated outbound
|
||||||
|
protocol, with version-aware heartbeat synchronization and a safe cached
|
||||||
|
representation for local visibility.
|
||||||
|
- Add a Host-scoped internal task-submission operation. An authenticated
|
||||||
|
Host can create a task only for itself and, when specified, one of its own
|
||||||
|
enrolled Devices; it never receives general public `tasks:submit` power.
|
||||||
|
- After `cloud-planner-proxy` is implemented and a Host uses its `cloud`
|
||||||
|
planner transport, record provider-reported token usage per Host/task/
|
||||||
|
attempt and enforce configured token budgets with atomic reservation and
|
||||||
|
settlement. Direct-to-provider Hosts remain explicitly unmetered and
|
||||||
|
cannot be represented as hard-budget-enforced.
|
||||||
|
- Extend the Console with task-creation, user-policy, Host-policy, and
|
||||||
|
per-Host AI-usage/budget views. Existing user lifecycle management remains
|
||||||
|
role-gated and gains policy editing rather than a new custom-role system.
|
||||||
|
|
||||||
|
## Capabilities
|
||||||
|
|
||||||
|
### New Capabilities
|
||||||
|
|
||||||
|
- `cloud-governance`: Durable, versioned user and Host policies that restrict
|
||||||
|
task submission/targets and define Host operational and AI-budget limits.
|
||||||
|
- `host-scoped-task-submission`: Authenticated internal operation that lets a
|
||||||
|
Host submit tasks constrained to itself.
|
||||||
|
- `cloud-ai-usage-governance`: Durable AI usage metering, budget reservations,
|
||||||
|
settlement, and non-secret reporting for Cloud-proxied planner calls.
|
||||||
|
|
||||||
|
### Modified Capabilities
|
||||||
|
|
||||||
|
- `cloud-console-ui`: add governed task creation and operational governance
|
||||||
|
views to the existing Cloud Console experience. This capability is still
|
||||||
|
defined by active Console changes and will be reconciled before archive.
|
||||||
|
- `platform-sdk`: extend public task submission/status and `CloudClient`
|
||||||
|
parity with explicit task targets and safe governance-facing responses.
|
||||||
|
- `task-scheduler`: make explicit Host/Device target selectors part of task
|
||||||
|
constraints and enforce them during matching.
|
||||||
|
- `host-agent-protocol`: return versioned Host policy through the outbound
|
||||||
|
protocol and add Host-self task submission while retaining host binding.
|
||||||
|
- `cloud-planner-proxy`: enforce Host budget reservations and emit
|
||||||
|
provider-reported usage only after the proxy change is implemented. This
|
||||||
|
capability is currently an active, unarchived change.
|
||||||
|
- `agent-runtime`: carry provider-reported non-secret token usage alongside a
|
||||||
|
tool-call decision so the Cloud proxy can meter the call without changing
|
||||||
|
`AIPlanner` decision semantics. This capability is also active and
|
||||||
|
unarchived.
|
||||||
|
|
||||||
|
## Impact
|
||||||
|
|
||||||
|
- `packages/cloud-platform/cloud`: governance and usage domain models,
|
||||||
|
repository port/SQLAlchemy implementation, Alembic migration, scheduler
|
||||||
|
matching, public/internal Pydantic contracts, policy-aware authorization,
|
||||||
|
and proxy metering.
|
||||||
|
- `apps/cloud-api`: router composition and configuration only; all durable
|
||||||
|
policy and accounting behavior remains in the Cloud Platform package.
|
||||||
|
- `apps/device-host-agent`: policy-aware client/heartbeat state and
|
||||||
|
self-targeted task submission; no inbound Cloud connection is introduced.
|
||||||
|
- `cloud-console/`: task composer plus user/Host policy and AI-usage views.
|
||||||
|
- Dependencies and rollout: implementation is blocked until
|
||||||
|
`cloud-planner-proxy` is completed, its migration is deployed, and affected
|
||||||
|
Hosts are using `AI_PLANNER_TRANSPORT=cloud`; it also follows completion of
|
||||||
|
the active Cloud Console user-authentication work.
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
## MODIFIED Requirements
|
||||||
|
|
||||||
|
### Requirement: Pluggable dual-provider tool-calling abstraction
|
||||||
|
The system SHALL support at least two interchangeable LLM providers
|
||||||
|
(Anthropic native tool use and OpenAI function calling) for the AI Planner's
|
||||||
|
decision calls, selectable via configuration, with both providers constrained
|
||||||
|
to return exactly one tool call per request. Independently of provider
|
||||||
|
selection, the system SHALL support at least two transports for making that
|
||||||
|
decision call -- direct-to-provider (the tool-calling client calls the
|
||||||
|
provider's SDK itself, using locally configured credentials) and cloud-proxy
|
||||||
|
(the tool-calling client calls the Cloud Control Plane's planner-decision
|
||||||
|
endpoint, which calls the provider using cloud-held credentials) -- selectable
|
||||||
|
via configuration without requiring any change to `AIPlanner`'s own decision
|
||||||
|
logic. A provider decision result SHALL also carry available non-secret
|
||||||
|
provider token-usage metadata so outer Cloud adapters can meter calls without
|
||||||
|
changing that decision logic.
|
||||||
|
|
||||||
|
#### Scenario: Provider selected via configuration
|
||||||
|
- **WHEN** the AI Planner is configured with a given provider identifier
|
||||||
|
- **THEN** it constructs and uses the tool-calling client for that provider
|
||||||
|
without requiring any change to `AIPlanner`'s own decision logic
|
||||||
|
|
||||||
|
#### Scenario: Provider response resolves to a single decision
|
||||||
|
- **WHEN** either supported provider returns a response to a tool-calling
|
||||||
|
request
|
||||||
|
- **THEN** the response is parsed into exactly one tool name and one arguments
|
||||||
|
object, regardless of which provider produced it
|
||||||
|
|
||||||
|
#### Scenario: Transport selected via configuration
|
||||||
|
- **WHEN** the Host Agent is configured with a given transport (direct or
|
||||||
|
cloud-proxy)
|
||||||
|
- **THEN** `AIPlanner` is constructed with the tool-calling client for that
|
||||||
|
transport, and its own decision logic is unchanged regardless of which
|
||||||
|
transport is in effect
|
||||||
|
|
||||||
|
#### Scenario: Direct transport remains available and default
|
||||||
|
- **WHEN** no transport is explicitly configured
|
||||||
|
- **THEN** the AI Planner uses the direct-to-provider transport, matching its
|
||||||
|
behavior before the cloud-proxy transport existed
|
||||||
|
|
||||||
|
#### Scenario: Cloud-proxy transport resolves a decision without a local provider client
|
||||||
|
- **WHEN** the Host Agent is configured with the cloud-proxy transport
|
||||||
|
- **THEN** its tool-calling client sends the decision request to the Cloud
|
||||||
|
Control Plane's planner-decision endpoint instead of constructing a local
|
||||||
|
Anthropic or OpenAI SDK client
|
||||||
|
|
||||||
|
#### Scenario: Provider reports token usage
|
||||||
|
- **WHEN** a provider response includes input or output token counts
|
||||||
|
- **THEN** the tool-calling result makes those non-secret counts available to
|
||||||
|
its outer adapter while `AIPlanner` selects the same single decision
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
## ADDED Requirements
|
||||||
|
|
||||||
|
### Requirement: Cloud-proxied planner calls produce durable non-secret usage events
|
||||||
|
After the Cloud planner proxy resolves a provider call, the system SHALL record
|
||||||
|
a durable usage event with Host, provider/model, provider-reported token
|
||||||
|
counts, request outcome, and known task-attempt context, while excluding raw
|
||||||
|
prompts, screenshots, provider credentials, cookies, and lease secrets.
|
||||||
|
|
||||||
|
#### Scenario: Provider returns token usage
|
||||||
|
- **WHEN** a Cloud-proxied planner call completes with provider-reported token
|
||||||
|
counts
|
||||||
|
- **THEN** the system stores those counts against the authenticated Host and
|
||||||
|
returns the planner decision without exposing secret request content
|
||||||
|
|
||||||
|
#### Scenario: Assignment context is available
|
||||||
|
- **WHEN** the Host makes a proxied planner call while executing a claimed
|
||||||
|
assignment
|
||||||
|
- **THEN** the recorded event includes that task id and attempt identifier
|
||||||
|
|
||||||
|
### Requirement: Host token budgets use atomic reservation and settlement
|
||||||
|
The Cloud planner proxy SHALL atomically reserve a conservative per-call token
|
||||||
|
amount before a provider invocation for every Host with an effective AI token
|
||||||
|
budget using Cloud planner transport, SHALL reject over-budget calls before
|
||||||
|
invoking the provider, and SHALL settle the reservation to provider-reported
|
||||||
|
usage when available.
|
||||||
|
|
||||||
|
#### Scenario: Remaining budget permits a call
|
||||||
|
- **WHEN** used tokens plus active reservations and the next conservative
|
||||||
|
reservation are within the Host's UTC-day budget
|
||||||
|
- **THEN** the proxy reserves budget, invokes the provider, and settles the
|
||||||
|
reservation after the provider response
|
||||||
|
|
||||||
|
#### Scenario: Remaining budget is insufficient
|
||||||
|
- **WHEN** the next reservation would exceed the Host's effective budget
|
||||||
|
- **THEN** the proxy rejects the planner call without invoking the provider
|
||||||
|
|
||||||
|
#### Scenario: Provider outcome has unknown usage
|
||||||
|
- **WHEN** a transport failure prevents the proxy from determining actual
|
||||||
|
provider usage after reservation
|
||||||
|
- **THEN** the reservation remains active until bounded expiry cleanup rather
|
||||||
|
than being released optimistically
|
||||||
|
|
||||||
|
### Requirement: Direct planner transport is explicitly unmetered
|
||||||
|
The system SHALL represent a Host using direct-to-provider planner transport as
|
||||||
|
unmetered for Cloud token accounting and SHALL not report it as complying with
|
||||||
|
a Cloud-enforced token budget.
|
||||||
|
|
||||||
|
#### Scenario: Console inspects a direct-transport Host
|
||||||
|
- **WHEN** an authorized operator views AI usage for a Host not using Cloud
|
||||||
|
planner transport
|
||||||
|
- **THEN** the system reports the Host as unmetered rather than zero usage or
|
||||||
|
hard-budget-enforced
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
## ADDED Requirements
|
||||||
|
|
||||||
|
### Requirement: Console composes explicitly targeted tasks
|
||||||
|
The Console SHALL provide a task-creation flow to principals with
|
||||||
|
`tasks:submit`, exposing goal/workflow input, Host selection, Device selection
|
||||||
|
scoped to the selected Host, and visible driver/capability constraints.
|
||||||
|
|
||||||
|
#### Scenario: Operator creates a Device-targeted task
|
||||||
|
- **WHEN** an authorized operator selects a Host and one of its Devices and
|
||||||
|
submits a valid task
|
||||||
|
- **THEN** the Console sends the explicit target to the public task API and
|
||||||
|
displays the resulting queued task with that target
|
||||||
|
|
||||||
|
#### Scenario: Restricted operator has no permitted target
|
||||||
|
- **WHEN** the Cloud API reports that the operator's policy disallows the
|
||||||
|
selected target or targetless submission
|
||||||
|
- **THEN** the Console shows the authorization error and does not imply that
|
||||||
|
the task was accepted or rerouted
|
||||||
|
|
||||||
|
### Requirement: Console administers governance policy within effective scopes
|
||||||
|
The Console SHALL expose user-submission and Host-policy views only to a
|
||||||
|
principal with `governance:admin`, and SHALL display effective non-secret
|
||||||
|
policy revisions and mutations without treating hidden controls as security.
|
||||||
|
|
||||||
|
#### Scenario: Administrator changes a Host budget
|
||||||
|
- **WHEN** a governance administrator saves a valid Host policy containing an
|
||||||
|
AI budget or active-task limit
|
||||||
|
- **THEN** the Console displays the returned newer policy revision and its
|
||||||
|
effective values
|
||||||
|
|
||||||
|
#### Scenario: Non-governance user opens the Console
|
||||||
|
- **WHEN** the current principal lacks `governance:admin`
|
||||||
|
- **THEN** the Console hides governance administration navigation while the
|
||||||
|
backend remains responsible for rejecting unauthorized requests
|
||||||
|
|
||||||
|
### Requirement: Console reports Host AI budget state without sensitive prompts
|
||||||
|
The Console SHALL render per-Host budget state and non-secret token usage for
|
||||||
|
authorized governance users, including an explicit unmetered state for Hosts
|
||||||
|
using direct planner transport.
|
||||||
|
|
||||||
|
#### Scenario: Administrator views a Cloud-proxied Host
|
||||||
|
- **WHEN** a governance administrator selects a Host with recorded proxy usage
|
||||||
|
- **THEN** the Console displays configured budget, used/reserved/remaining
|
||||||
|
tokens, provider/model, and non-secret event metadata
|
||||||
|
|
||||||
|
#### Scenario: Administrator views a direct-transport Host
|
||||||
|
- **WHEN** a governance administrator selects a Host that does not use Cloud
|
||||||
|
planner transport
|
||||||
|
- **THEN** the Console labels the Host unmetered and does not display zero as
|
||||||
|
a budget-compliant usage value
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
## ADDED Requirements
|
||||||
|
|
||||||
|
### Requirement: User task-submission policy is durable and authoritative
|
||||||
|
The system SHALL persist an optional versioned submission policy for each
|
||||||
|
human user, containing submission enablement and explicit Host/Device target
|
||||||
|
allow-lists, and SHALL enforce that policy after scope authentication on every
|
||||||
|
public task-submission request.
|
||||||
|
|
||||||
|
#### Scenario: Restricted user submits to an allowed Device
|
||||||
|
- **WHEN** a principal with `tasks:submit` has a policy allowing a Host/Device
|
||||||
|
target and submits a task for that exact target
|
||||||
|
- **THEN** the system accepts the task subject to normal validation and records
|
||||||
|
the authenticated principal as its submitter
|
||||||
|
|
||||||
|
#### Scenario: Restricted user omits or exceeds a target allow-list
|
||||||
|
- **WHEN** a principal with a restrictive policy submits without a target or
|
||||||
|
names a Host/Device outside its allow-list
|
||||||
|
- **THEN** the system rejects the request before creating a task
|
||||||
|
|
||||||
|
#### Scenario: Submission is disabled
|
||||||
|
- **WHEN** a principal with `tasks:submit` has a policy with submission
|
||||||
|
disabled
|
||||||
|
- **THEN** the system rejects task submission without changing the queue
|
||||||
|
|
||||||
|
### Requirement: Host governance policy is versioned and enforceable
|
||||||
|
The system SHALL persist a monotonic revisioned policy for each Host with an
|
||||||
|
optional active-task limit and optional UTC-day AI token budget, and SHALL
|
||||||
|
enforce active-task limits during Cloud scheduling.
|
||||||
|
|
||||||
|
#### Scenario: Host active-task limit is reached
|
||||||
|
- **WHEN** a Host already has its configured maximum number of assigned or
|
||||||
|
dispatched attempts and another queued task targets that Host
|
||||||
|
- **THEN** the scheduler leaves the later task queued until capacity becomes
|
||||||
|
available
|
||||||
|
|
||||||
|
#### Scenario: Host policy is updated
|
||||||
|
- **WHEN** an authorized governance administrator updates a Host policy
|
||||||
|
- **THEN** the system stores a strictly newer revision and later Cloud
|
||||||
|
authorization, scheduling, and proxy operations use the new policy
|
||||||
|
|
||||||
|
### Requirement: Governance mutations and reads use dedicated scopes
|
||||||
|
The system SHALL protect governance policy and AI-usage operations with
|
||||||
|
`governance:read` or `governance:admin` as appropriate, and SHALL record safe
|
||||||
|
audits for policy mutations.
|
||||||
|
|
||||||
|
#### Scenario: Administrator changes a policy
|
||||||
|
- **WHEN** a principal with `governance:admin` changes a user or Host policy
|
||||||
|
- **THEN** the system records actor, target, old/new revision, timestamp, and
|
||||||
|
non-secret policy metadata without recording credentials or request secrets
|
||||||
|
|
||||||
|
#### Scenario: Non-governance principal attempts a policy mutation
|
||||||
|
- **WHEN** a principal lacking `governance:admin` updates a governance policy
|
||||||
|
- **THEN** the system rejects the request before changing durable policy state
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
## ADDED Requirements
|
||||||
|
|
||||||
|
### Requirement: Planner proxy enforces effective Host token budgets before provider invocation
|
||||||
|
The Cloud planner-decision endpoint SHALL atomically reserve a conservative
|
||||||
|
bounded token amount before invoking its provider for an authenticated Host
|
||||||
|
using Cloud planner transport with an effective token budget, and SHALL reject
|
||||||
|
the request without invoking the provider when the reservation would exceed
|
||||||
|
the remaining budget.
|
||||||
|
|
||||||
|
#### Scenario: Budget permits a planner decision
|
||||||
|
- **WHEN** the Host has sufficient budget after used and reserved tokens are
|
||||||
|
considered
|
||||||
|
- **THEN** the endpoint creates a reservation and invokes the configured
|
||||||
|
provider exactly once
|
||||||
|
|
||||||
|
#### Scenario: Budget is exhausted
|
||||||
|
- **WHEN** a next planner decision would exceed the Host's effective budget
|
||||||
|
- **THEN** the endpoint returns a structured failure without invoking the
|
||||||
|
configured provider
|
||||||
|
|
||||||
|
### Requirement: Planner proxy settles provider-reported token usage without persisting prompts
|
||||||
|
The Cloud planner-decision endpoint SHALL settle its reservation to the
|
||||||
|
provider-reported token usage when available and SHALL retain only non-secret
|
||||||
|
metering metadata, never the raw prompt, screenshot, provider credentials, or
|
||||||
|
session/lease secret.
|
||||||
|
|
||||||
|
#### Scenario: Provider response includes usage
|
||||||
|
- **WHEN** the configured provider returns a valid tool-call decision and
|
||||||
|
token-usage metadata
|
||||||
|
- **THEN** the endpoint records and returns the decision, settles the Host's
|
||||||
|
reservation to the reported usage, and does not durably store request text
|
||||||
|
or screenshot bytes
|
||||||
|
|
||||||
|
#### Scenario: Usage is indeterminate after failure
|
||||||
|
- **WHEN** a reservation exists but the endpoint cannot determine provider
|
||||||
|
usage after a transport failure
|
||||||
|
- **THEN** the reservation remains until bounded expiry cleanup rather than
|
||||||
|
being released as unused
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
## MODIFIED Requirements
|
||||||
|
|
||||||
|
### Requirement: Host identity is authenticated and bound to one host id
|
||||||
|
The internal Host Agent API SHALL require a host-scoped bearer principal and
|
||||||
|
SHALL reject any request that attempts to act for a `host_id` different from
|
||||||
|
the authenticated principal's bound host.
|
||||||
|
|
||||||
|
#### Scenario: Host authenticates as itself
|
||||||
|
- **WHEN** a Host Agent presents valid credentials bound to its requested
|
||||||
|
`host_id`
|
||||||
|
- **THEN** the internal API authorizes permitted heartbeat, claim, renewal,
|
||||||
|
result, policy retrieval, and self-submission operations
|
||||||
|
|
||||||
|
#### Scenario: Host attempts to impersonate another host
|
||||||
|
- **WHEN** valid credentials bound to host A are used on a request for host B
|
||||||
|
- **THEN** the internal API rejects the request without reading or modifying
|
||||||
|
host B's state
|
||||||
|
|
||||||
|
### Requirement: Host Agent synchronizes heartbeat and complete device snapshots
|
||||||
|
The Host Agent SHALL periodically submit its complete local device snapshot
|
||||||
|
and last applied policy revision to the control plane, and the control plane
|
||||||
|
SHALL atomically refresh the host heartbeat, replace only that host's
|
||||||
|
pooled-device records, and return the effective Host policy whenever its
|
||||||
|
revision differs.
|
||||||
|
|
||||||
|
#### Scenario: Host reports devices
|
||||||
|
- **WHEN** a Host Agent submits a valid heartbeat containing its current devices
|
||||||
|
- **THEN** the control plane updates the host's last-seen time and exposes the
|
||||||
|
submitted devices through the aggregated pool
|
||||||
|
|
||||||
|
#### Scenario: Host policy revision changed
|
||||||
|
- **WHEN** a Host heartbeat presents a revision older than the effective
|
||||||
|
Cloud-host policy
|
||||||
|
- **THEN** the response includes the complete newer policy and revision for the
|
||||||
|
Host to cache
|
||||||
|
|
||||||
|
#### Scenario: Host policy revision is current
|
||||||
|
- **WHEN** a Host heartbeat presents the current effective policy revision
|
||||||
|
- **THEN** the response confirms that revision without resending an unrelated
|
||||||
|
policy representation
|
||||||
|
|
||||||
|
#### Scenario: Host reports no devices
|
||||||
|
- **WHEN** a previously populated host submits an empty device snapshot
|
||||||
|
- **THEN** only that host's prior device records are removed while devices
|
||||||
|
owned by other hosts remain unchanged
|
||||||
|
|
||||||
|
## ADDED Requirements
|
||||||
|
|
||||||
|
### Requirement: Host can submit a self-targeted goal through the outbound protocol
|
||||||
|
The Host Agent SHALL be able to submit a goal-only task through its existing
|
||||||
|
outbound authenticated protocol, and the control plane SHALL derive its target
|
||||||
|
from the authenticated Host rather than accepting another Host identifier.
|
||||||
|
|
||||||
|
#### Scenario: Host submits a local goal
|
||||||
|
- **WHEN** an authenticated Host submits a valid goal through its internal
|
||||||
|
self-submission operation
|
||||||
|
- **THEN** the control plane returns a queued task targeted to that Host
|
||||||
|
|
||||||
|
#### Scenario: Host submits an ineligible local Device
|
||||||
|
- **WHEN** a Host names a Device that is not owned by its authenticated Host
|
||||||
|
- **THEN** the control plane rejects the request without creating a task
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
## ADDED Requirements
|
||||||
|
|
||||||
|
### Requirement: Host can submit a goal task only for itself
|
||||||
|
The internal Host Agent API SHALL allow an authenticated Host to submit a
|
||||||
|
goal-based task that the Cloud Control Plane irrevocably targets to that same
|
||||||
|
Host, with an optional Device target owned by that Host.
|
||||||
|
|
||||||
|
#### Scenario: Host submits a goal for one of its Devices
|
||||||
|
- **WHEN** Host A presents valid Host A credentials and submits a valid goal
|
||||||
|
naming one of Host A's enrolled Devices
|
||||||
|
- **THEN** the control plane creates a queued task targeted to Host A and that
|
||||||
|
Device without granting Host A public task-submission authority
|
||||||
|
|
||||||
|
#### Scenario: Host omits a Device target
|
||||||
|
- **WHEN** an authenticated Host submits a valid goal without a Device target
|
||||||
|
- **THEN** the control plane creates a queued task targeted to that Host and
|
||||||
|
lets the scheduler select only an eligible Device owned by it
|
||||||
|
|
||||||
|
### Requirement: Host self-submission preserves host isolation
|
||||||
|
The internal Host task-submission operation SHALL reject a foreign Host,
|
||||||
|
foreign Device, workflow reference, or disabled self-submission policy before
|
||||||
|
creating any task.
|
||||||
|
|
||||||
|
#### Scenario: Host attempts a foreign target
|
||||||
|
- **WHEN** Host A submits a request naming Host B or a Device not owned by A
|
||||||
|
- **THEN** the control plane rejects the request and does not create a task
|
||||||
|
|
||||||
|
#### Scenario: Host self-submission is disabled
|
||||||
|
- **WHEN** a Host policy disables Host self-submission
|
||||||
|
- **THEN** the control plane rejects that Host's self-submission request
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
## MODIFIED Requirements
|
||||||
|
|
||||||
|
### Requirement: Task submission and status via the SDK
|
||||||
|
The system SHALL allow an external integrator to submit a task (goal or
|
||||||
|
workflow reference plus constraints and an optional explicit Host/Device
|
||||||
|
target) through the platform SDK's API, and to query that task's current
|
||||||
|
status by id, backed by the `task-scheduler` capability. A Device target MUST
|
||||||
|
include its owning Host, and the status representation SHALL expose any
|
||||||
|
explicit target separately from the eventual assignment.
|
||||||
|
|
||||||
|
#### Scenario: Submit a task via the API
|
||||||
|
- **WHEN** an integrator calls the task-submission endpoint with a valid goal
|
||||||
|
and optional constraints
|
||||||
|
- **THEN** the API returns a task id that can be used to poll status, and the
|
||||||
|
underlying `task-scheduler` records a new `queued` `ScheduledTask`
|
||||||
|
|
||||||
|
#### Scenario: Submit a task for one Host and Device
|
||||||
|
- **WHEN** a `tasks:submit` principal submits a valid target containing an
|
||||||
|
eligible Host and Device it is permitted to use
|
||||||
|
- **THEN** the API records that exact target and the scheduler cannot assign
|
||||||
|
the task outside it
|
||||||
|
|
||||||
|
#### Scenario: Device target lacks its Host
|
||||||
|
- **WHEN** an integrator submits a Device target without an owning Host
|
||||||
|
- **THEN** the API rejects the request before creating a task
|
||||||
|
|
||||||
|
#### Scenario: Query status of a known task
|
||||||
|
- **WHEN** an integrator requests status for a task id that exists
|
||||||
|
- **THEN** the API returns that task's current status (`queued`, `assigned`,
|
||||||
|
`dispatched`, `done`, or `failed`) and its explicit target when present
|
||||||
|
|
||||||
|
#### Scenario: Query status of an unknown task
|
||||||
|
- **WHEN** an integrator requests status for a task id that does not exist
|
||||||
|
- **THEN** the API returns a not-found response rather than an unhandled server
|
||||||
|
error
|
||||||
|
|
||||||
|
### Requirement: Python SDK client mirrors the REST API
|
||||||
|
The system SHALL provide a Python client (`CloudClient`) exposing methods
|
||||||
|
corresponding to every `/v1/...` resource, user-authentication,
|
||||||
|
user-administration, and governance route, including targeted task submission
|
||||||
|
and read-only Host AI-usage access, so integrators do not need to
|
||||||
|
hand-construct HTTP requests.
|
||||||
|
|
||||||
|
#### Scenario: Client submits a targeted task and retrieves status
|
||||||
|
- **WHEN** a caller uses `CloudClient` to submit a task with an explicit target
|
||||||
|
and then fetches status by the returned id
|
||||||
|
- **THEN** the client produces the same target and lifecycle result as direct
|
||||||
|
REST calls
|
||||||
|
|
||||||
|
#### Scenario: Client administers governance with a scoped bearer
|
||||||
|
- **WHEN** a caller configures `CloudClient` with `governance:admin` and
|
||||||
|
invokes a policy operation
|
||||||
|
- **THEN** the client sends that authentication and returns the corresponding
|
||||||
|
non-secret policy representation
|
||||||
|
|
||||||
|
### Requirement: Public API operations enforce scopes
|
||||||
|
The public platform API SHALL require operation-specific scopes for task
|
||||||
|
submission, task reading, pool reading, plugin reading, plugin
|
||||||
|
administration, user administration, governance reading, and governance
|
||||||
|
administration, regardless of whether the principal came from a bearer
|
||||||
|
credential or user session.
|
||||||
|
|
||||||
|
#### Scenario: Submit principal has task scope
|
||||||
|
- **WHEN** a bearer or user principal with `tasks:submit` calls the
|
||||||
|
task-submission endpoint
|
||||||
|
- **THEN** the request is authorized subject to normal task validation and
|
||||||
|
any effective user-submission policy
|
||||||
|
|
||||||
|
#### Scenario: Non-admin principal attempts plugin registration
|
||||||
|
- **WHEN** an authenticated principal without `plugins:admin` calls plugin
|
||||||
|
registration
|
||||||
|
- **THEN** the API rejects the request before resolving or loading the plugin
|
||||||
|
target
|
||||||
|
|
||||||
|
#### Scenario: Non-admin principal attempts user administration
|
||||||
|
- **WHEN** an authenticated principal without `users:admin` calls a
|
||||||
|
user-administration endpoint
|
||||||
|
- **THEN** the API rejects the request before reading or changing protected
|
||||||
|
user state
|
||||||
|
|
||||||
|
#### Scenario: Principal lacks governance administration scope
|
||||||
|
- **WHEN** an authenticated principal without `governance:admin` changes a
|
||||||
|
user or Host policy
|
||||||
|
- **THEN** the API rejects the request before changing policy or usage state
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
## MODIFIED Requirements
|
||||||
|
|
||||||
|
### Requirement: Task submission enqueues a scheduled task
|
||||||
|
The system SHALL allow a caller to submit a task (a goal string, or a
|
||||||
|
reference to a `WorkflowDefinition`, plus optional device constraints:
|
||||||
|
`driver_type`, required capability tags, and an explicit Host/Device target)
|
||||||
|
and SHALL enqueue it as a `ScheduledTask` with status `queued`, returning a
|
||||||
|
stable task id the caller can poll. A Device target MUST name its owning Host;
|
||||||
|
an explicit target remains attached to the task for its entire lifecycle.
|
||||||
|
|
||||||
|
#### Scenario: Successful submission
|
||||||
|
- **WHEN** a caller submits a task with a goal and no constraints
|
||||||
|
- **THEN** the scheduler creates a `ScheduledTask` with status `queued`,
|
||||||
|
assigns it a unique id, and returns that id to the caller without blocking
|
||||||
|
for a device to become available
|
||||||
|
|
||||||
|
#### Scenario: Targeted submission
|
||||||
|
- **WHEN** a caller submits a valid task targeted to a Host and one of its
|
||||||
|
Devices
|
||||||
|
- **THEN** the queued task retains that Host/Device target until it is assigned
|
||||||
|
or reaches a terminal lifecycle state
|
||||||
|
|
||||||
|
#### Scenario: Queue depth limit reached
|
||||||
|
- **WHEN** a caller submits a task while the queue already holds
|
||||||
|
`config.max_queue_depth` queued tasks
|
||||||
|
- **THEN** the scheduler rejects the submission with a clear error rather than
|
||||||
|
accepting an unbounded backlog
|
||||||
|
|
||||||
|
### Requirement: Assignment matches a queued task to an idle, constraint-matching device
|
||||||
|
The system SHALL assign a queued `ScheduledTask` to an idle `PooledDevice`
|
||||||
|
(as reported by the `device-pool` capability) whose explicit target, if any,
|
||||||
|
`driver_type`, and capability tags satisfy the task's constraints, using a
|
||||||
|
named, registrable `AssignmentStrategy`.
|
||||||
|
|
||||||
|
#### Scenario: Matching idle device available
|
||||||
|
- **WHEN** `assign()` runs and at least one idle `PooledDevice` matches the
|
||||||
|
head-of-queue task's constraints
|
||||||
|
- **THEN** the scheduler selects one such device via the configured
|
||||||
|
`AssignmentStrategy`, transitions the task to status `assigned`, and records
|
||||||
|
the chosen `device_id`/`host_id`
|
||||||
|
|
||||||
|
#### Scenario: Targeted Device is unavailable
|
||||||
|
- **WHEN** `assign()` runs and the only explicitly targeted Device is busy,
|
||||||
|
stale, absent, or otherwise ineligible
|
||||||
|
- **THEN** the task remains `queued` and the scheduler does not assign a
|
||||||
|
different Device or Host
|
||||||
|
|
||||||
|
#### Scenario: No matching device available
|
||||||
|
- **WHEN** `assign()` runs and no idle `PooledDevice` matches the head-of-queue
|
||||||
|
task's constraints
|
||||||
|
- **THEN** the task remains `queued` (not failed), and `assign()` returns
|
||||||
|
without error, ready to be retried on a later call
|
||||||
|
|
||||||
|
#### Scenario: Unknown assignment strategy configured
|
||||||
|
- **WHEN** `TaskScheduler` is configured with an `AssignmentStrategy` name
|
||||||
|
that is not registered
|
||||||
|
- **THEN** the scheduler raises a clear configuration error at startup/first-
|
||||||
|
assign rather than silently falling back to a default strategy
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
## 1. Preconditions and dependency reconciliation
|
||||||
|
|
||||||
|
- [x] 1.1 Confirm `cloud-planner-proxy` is fully implemented, passes its
|
||||||
|
strict validation and relevant runtime tests, and its deployed schema/API
|
||||||
|
contract is the base for this change.
|
||||||
|
- [ ] 1.2 Complete and reconcile `cloud-console-user-authentication`, including
|
||||||
|
the documented Console Users experience and its remaining PostgreSQL and
|
||||||
|
HTTPS/Host-Agent verification tasks.
|
||||||
|
- [ ] 1.3 Rebase this change's delta specs against the then-current canonical
|
||||||
|
`platform-sdk`, `cloud-console-ui`, `cloud-planner-proxy`, and
|
||||||
|
`agent-runtime` specifications before implementation begins.
|
||||||
|
|
||||||
|
## 2. Governance domain, persistence, and migration
|
||||||
|
|
||||||
|
- [ ] 2.1 Define target-selector, user-submission-policy, Host-governance-
|
||||||
|
policy, token-usage-event, and token-reservation domain models with
|
||||||
|
non-secret representations and revision semantics.
|
||||||
|
- [ ] 2.2 Extend the Cloud repository port with transactional CRUD/query
|
||||||
|
operations for policies, active Host capacity, budget reservations,
|
||||||
|
settlement, expiry cleanup, and bounded usage summaries/events.
|
||||||
|
- [ ] 2.3 Add SQLAlchemy rows, indexes, conversion helpers, and concurrency-
|
||||||
|
safe PostgreSQL/SQLite implementations for governance policies, usage
|
||||||
|
events, reservations, and safe policy audits.
|
||||||
|
- [ ] 2.4 Add an Alembic forward/downgrade revision that preserves existing
|
||||||
|
users, Hosts, devices, tasks, and attempts; advance schema readiness
|
||||||
|
checks to the new head.
|
||||||
|
- [ ] 2.5 Add repository and migration tests for policy revisions, null versus
|
||||||
|
empty allow-list semantics, active-task capacity, reservation races,
|
||||||
|
settlement, expiry cleanup, audit redaction, and SQLite/PostgreSQL
|
||||||
|
parity.
|
||||||
|
|
||||||
|
## 3. Targeted scheduling and public governance API
|
||||||
|
|
||||||
|
- [x] 3.1 Extend task constraints, SDK request/response models, persistence
|
||||||
|
serialization, status/list representations, and `CloudClient` for an
|
||||||
|
optional Host/Device target; reject a Device without its Host.
|
||||||
|
- [x] 3.2 Enforce target ownership/existence at public submission and filter
|
||||||
|
scheduler candidates by target before strategy selection; keep an
|
||||||
|
unavailable targeted task queued with no fallback assignment.
|
||||||
|
- [x] 3.3 Add `governance:read` and `governance:admin` scopes and policy-aware
|
||||||
|
public task authorization that combines `tasks:submit` with the
|
||||||
|
authenticated human user's effective submission policy.
|
||||||
|
- [ ] 3.4 Add bounded, non-secret public governance routes and `CloudClient`
|
||||||
|
methods for user policy, Host policy, Host AI-budget summaries, and
|
||||||
|
paginated usage events; audit every policy mutation.
|
||||||
|
- [ ] 3.5 Add public API/SDK tests for targeted submission, target-policy
|
||||||
|
denial, scope enforcement, status exposure, governance CRUD, and
|
||||||
|
absence of secret fields in responses/audits.
|
||||||
|
|
||||||
|
## 4. Host policy synchronization and self-submission
|
||||||
|
|
||||||
|
- [x] 4.1 Extend shared internal heartbeat request/response models with the
|
||||||
|
Host's last policy revision and a revision-aware effective-policy reply.
|
||||||
|
- [ ] 4.2 Update Host heartbeat synchronization to persist only safe cached
|
||||||
|
policy state and expose it through local status/history without adding an
|
||||||
|
inbound Cloud connection.
|
||||||
|
- [x] 4.3 Add the Host-scoped goal-only task-submission route and
|
||||||
|
`HostAgentClient` method; derive Host targeting from authenticated
|
||||||
|
credentials and validate any named local Device ownership.
|
||||||
|
- [ ] 4.4 Enforce the Host policy's self-submission and active-task limits in
|
||||||
|
the Cloud service/scheduler, not only in Host-local code.
|
||||||
|
- [ ] 4.5 Add Cloud API and Host Agent tests for revision convergence,
|
||||||
|
unchanged-policy replies, self-targeted task creation, foreign target
|
||||||
|
rejection, disabled self-submission, and outbound-only compatibility.
|
||||||
|
|
||||||
|
## 5. Cloud-proxy usage metering and hard budget enforcement
|
||||||
|
|
||||||
|
- [x] 5.1 Extend the dual-provider tool-calling result with optional
|
||||||
|
non-secret provider usage fields while preserving `AIPlanner`'s existing
|
||||||
|
single-decision behavior and direct transport compatibility.
|
||||||
|
- [ ] 5.2 Extend planner-proxy request context and Host-Agent-local context
|
||||||
|
binding so Cloud-proxied calls carry known task/attempt metadata without
|
||||||
|
importing Host or Cloud concerns into `runtime`.
|
||||||
|
- [ ] 5.3 Add Cloud proxy preflight reservation, configured conservative
|
||||||
|
per-call ceiling, provider invocation, actual-usage settlement, and
|
||||||
|
bounded unknown-usage reservation expiry.
|
||||||
|
- [ ] 5.4 Record non-secret usage events and expose accurate
|
||||||
|
used/reserved/remaining UTC-day budget summaries; explicitly report
|
||||||
|
direct transport as unmetered.
|
||||||
|
- [ ] 5.5 Add provider-fake, repository concurrency, Cloud API, and Host
|
||||||
|
Agent tests for usage parsing, over-budget rejection before provider
|
||||||
|
invocation, concurrent reservations, settlement, timeout uncertainty,
|
||||||
|
task-attempt attribution, and direct-transport labeling.
|
||||||
|
|
||||||
|
## 6. Cloud Console governance experience
|
||||||
|
|
||||||
|
- [x] 6.1 Add task composer API bindings and a scope-aware Console form for
|
||||||
|
goal/workflow submission, Host/Device target selection, and target/policy
|
||||||
|
validation errors.
|
||||||
|
- [ ] 6.2 Complete or reconcile the admin Users view, then add user-
|
||||||
|
submission-policy editing with safe refresh and conflict/error handling.
|
||||||
|
- [ ] 6.3 Add Host policy administration and AI-usage/budget views, including
|
||||||
|
revision display, unmetered direct Hosts, and no rendering of prompts,
|
||||||
|
screenshots, provider credentials, cookies, or lease secrets.
|
||||||
|
- [ ] 6.4 Add frontend tests for task composer scope/policy failures,
|
||||||
|
governance navigation, policy mutations, usage rendering, unmetered
|
||||||
|
status, CSRF writes, and retained session behavior on `403`.
|
||||||
|
|
||||||
|
## 7. Documentation, verification, and rollout
|
||||||
|
|
||||||
|
- [ ] 7.1 Update deployment and Console documentation with the dependency
|
||||||
|
order, migration/rollback sequence, policy semantics, Cloud transport
|
||||||
|
prerequisite, budget reservation behavior, direct-host limitation, and
|
||||||
|
safe administrator operations.
|
||||||
|
- [ ] 7.2 Run formatting, Ruff, compile checks, secret/redaction review, the
|
||||||
|
complete non-integration workspace test suite, Cloud Console unit tests,
|
||||||
|
type-check, and production build.
|
||||||
|
- [ ] 7.3 Run real PostgreSQL migration/concurrency coverage and a Compose HTTPS
|
||||||
|
manual flow covering user restriction, targeted scheduling, Host policy
|
||||||
|
delivery, Host self-submission, Cloud-proxy budget exhaustion, and a
|
||||||
|
direct Host labelled unmetered.
|
||||||
|
- [x] 7.4 Run `openspec validate cloud-console-governance --strict` and resolve
|
||||||
|
all proposal, design, specification, and task validation errors.
|
||||||
@@ -16,6 +16,8 @@ POOL_READ_SCOPE = "pool:read"
|
|||||||
PLUGINS_READ_SCOPE = "plugins:read"
|
PLUGINS_READ_SCOPE = "plugins:read"
|
||||||
PLUGINS_ADMIN_SCOPE = "plugins:admin"
|
PLUGINS_ADMIN_SCOPE = "plugins:admin"
|
||||||
USERS_ADMIN_SCOPE = "users:admin"
|
USERS_ADMIN_SCOPE = "users:admin"
|
||||||
|
GOVERNANCE_READ_SCOPE = "governance:read"
|
||||||
|
GOVERNANCE_ADMIN_SCOPE = "governance:admin"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|||||||
@@ -215,3 +215,29 @@ class AuthAuditRow(Base):
|
|||||||
outcome: Mapped[str] = mapped_column(String, nullable=False)
|
outcome: Mapped[str] = mapped_column(String, nullable=False)
|
||||||
correlation_id: Mapped[str | None] = mapped_column(String, nullable=True)
|
correlation_id: Mapped[str | None] = mapped_column(String, nullable=True)
|
||||||
metadata_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
metadata_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||||
|
|
||||||
|
|
||||||
|
class UserSubmissionPolicyRow(Base):
|
||||||
|
__tablename__ = "cloud_user_submission_policies"
|
||||||
|
|
||||||
|
user_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("cloud_users.id", ondelete="CASCADE"), primary_key=True
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
submission_enabled: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
allowed_host_ids_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
allowed_device_targets_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
updated_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class HostGovernancePolicyRow(Base):
|
||||||
|
__tablename__ = "cloud_host_governance_policies"
|
||||||
|
|
||||||
|
host_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("host_registrations.host_id", ondelete="CASCADE"), primary_key=True
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
self_submission_enabled: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
max_active_tasks: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
daily_token_budget: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
updated_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
"""Durable Cloud-side submission and Host governance policies."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
class TaskSubmissionPolicyError(PermissionError):
|
||||||
|
"""Raised when a human user's policy disallows a task submission."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UserSubmissionPolicy:
|
||||||
|
user_id: str
|
||||||
|
revision: int
|
||||||
|
submission_enabled: bool
|
||||||
|
allowed_host_ids: tuple[str, ...] | None
|
||||||
|
allowed_device_targets: tuple[tuple[str, str], ...] | None
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class HostGovernancePolicy:
|
||||||
|
host_id: str
|
||||||
|
revision: int
|
||||||
|
self_submission_enabled: bool
|
||||||
|
max_active_tasks: int | None
|
||||||
|
daily_token_budget: int | None
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
def enforce_user_submission_policy(
|
||||||
|
policy: UserSubmissionPolicy | None,
|
||||||
|
*,
|
||||||
|
target_host_id: str | None,
|
||||||
|
target_device_id: str | None,
|
||||||
|
) -> None:
|
||||||
|
if policy is None:
|
||||||
|
return
|
||||||
|
if not policy.submission_enabled:
|
||||||
|
raise TaskSubmissionPolicyError("task submission is disabled for this user")
|
||||||
|
restricted = (
|
||||||
|
policy.allowed_host_ids is not None
|
||||||
|
or policy.allowed_device_targets is not None
|
||||||
|
)
|
||||||
|
if not restricted:
|
||||||
|
return
|
||||||
|
if target_host_id is None:
|
||||||
|
raise TaskSubmissionPolicyError("an explicit permitted target is required")
|
||||||
|
if (
|
||||||
|
policy.allowed_host_ids is not None
|
||||||
|
and target_host_id not in policy.allowed_host_ids
|
||||||
|
):
|
||||||
|
raise TaskSubmissionPolicyError("target host is not permitted")
|
||||||
|
if policy.allowed_device_targets is not None:
|
||||||
|
if target_device_id is None or (
|
||||||
|
target_host_id,
|
||||||
|
target_device_id,
|
||||||
|
) not in policy.allowed_device_targets:
|
||||||
|
raise TaskSubmissionPolicyError("target device is not permitted")
|
||||||
@@ -25,8 +25,11 @@ from cloud.internal_api.models import (
|
|||||||
DeviceEnrollmentResponse,
|
DeviceEnrollmentResponse,
|
||||||
HeartbeatRequest,
|
HeartbeatRequest,
|
||||||
HeartbeatResponse,
|
HeartbeatResponse,
|
||||||
|
HostGovernancePolicyModel,
|
||||||
HostEnrollmentRequest,
|
HostEnrollmentRequest,
|
||||||
HostEnrollmentResponse,
|
HostEnrollmentResponse,
|
||||||
|
HostTaskSubmissionRequest,
|
||||||
|
HostTaskSubmissionResponse,
|
||||||
LeaseRenewalRequest,
|
LeaseRenewalRequest,
|
||||||
LeaseRenewalResponse,
|
LeaseRenewalResponse,
|
||||||
PlannerDecisionError,
|
PlannerDecisionError,
|
||||||
@@ -47,6 +50,7 @@ from runtime.tool_specs import ToolSpec
|
|||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from cloud.pool import DevicePool
|
from cloud.pool import DevicePool
|
||||||
|
from cloud.scheduler import TaskScheduler
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -60,6 +64,7 @@ def create_internal_router(
|
|||||||
lease_duration_seconds: float = 60.0,
|
lease_duration_seconds: float = 60.0,
|
||||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||||
planner_client_factory: Callable[[], ToolCallingClient] | None = None,
|
planner_client_factory: Callable[[], ToolCallingClient] | None = None,
|
||||||
|
scheduler: TaskScheduler | None = None,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
if claim_poll_interval_seconds <= 0:
|
if claim_poll_interval_seconds <= 0:
|
||||||
raise ValueError("claim_poll_interval_seconds must be greater than zero")
|
raise ValueError("claim_poll_interval_seconds must be greater than zero")
|
||||||
@@ -166,12 +171,71 @@ def create_internal_router(
|
|||||||
address=payload.address,
|
address=payload.address,
|
||||||
allow_device_takeover=allow_device_takeover,
|
allow_device_takeover=allow_device_takeover,
|
||||||
)
|
)
|
||||||
|
policy = pool.store.get_host_governance_policy(host_id)
|
||||||
|
policy_revision = policy.revision if policy is not None else 0
|
||||||
return HeartbeatResponse(
|
return HeartbeatResponse(
|
||||||
host_id=host_id,
|
host_id=host_id,
|
||||||
accepted_devices=len(devices),
|
accepted_devices=len(devices),
|
||||||
received_at=utc_now(),
|
received_at=utc_now(),
|
||||||
|
policy_revision=policy_revision,
|
||||||
|
policy=(
|
||||||
|
HostGovernancePolicyModel(
|
||||||
|
revision=policy.revision,
|
||||||
|
self_submission_enabled=policy.self_submission_enabled,
|
||||||
|
max_active_tasks=policy.max_active_tasks,
|
||||||
|
daily_token_budget=policy.daily_token_budget,
|
||||||
|
)
|
||||||
|
if policy is not None and payload.policy_revision != policy.revision
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/hosts/{host_id}/tasks",
|
||||||
|
response_model=HostTaskSubmissionResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def submit_host_task(
|
||||||
|
host_id: str,
|
||||||
|
payload: HostTaskSubmissionRequest,
|
||||||
|
request: Request,
|
||||||
|
) -> HostTaskSubmissionResponse:
|
||||||
|
authorize_host(request, host_id)
|
||||||
|
if payload.host_id != host_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="task host_id must match the request path",
|
||||||
|
)
|
||||||
|
if scheduler is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
|
detail="task submission is unavailable",
|
||||||
|
)
|
||||||
|
policy = pool.store.get_host_governance_policy(host_id)
|
||||||
|
if policy is not None and not policy.self_submission_enabled:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail="Host self-submission is disabled",
|
||||||
|
)
|
||||||
|
if payload.device_id is not None and not any(
|
||||||
|
device.host_id == host_id and device.device_id == payload.device_id
|
||||||
|
for device in pool.list_devices()
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="target device is not owned by this Host",
|
||||||
|
)
|
||||||
|
from cloud.scheduler import TaskConstraints
|
||||||
|
|
||||||
|
task_id = scheduler.submit(
|
||||||
|
goal=payload.goal,
|
||||||
|
constraints=TaskConstraints(
|
||||||
|
target_host_id=host_id,
|
||||||
|
target_device_id=payload.device_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return HostTaskSubmissionResponse(task_id=task_id)
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/hosts/{host_id}/assignments/claim",
|
"/hosts/{host_id}/assignments/claim",
|
||||||
response_model=ClaimResponse,
|
response_model=ClaimResponse,
|
||||||
@@ -285,6 +349,7 @@ def create_internal_router(
|
|||||||
@router.post(
|
@router.post(
|
||||||
"/hosts/{host_id}/planner/decide",
|
"/hosts/{host_id}/planner/decide",
|
||||||
response_model=PlannerDecisionResponse,
|
response_model=PlannerDecisionResponse,
|
||||||
|
response_model_exclude_none=True,
|
||||||
responses={
|
responses={
|
||||||
status.HTTP_502_BAD_GATEWAY: {"model": PlannerDecisionError},
|
status.HTTP_502_BAD_GATEWAY: {"model": PlannerDecisionError},
|
||||||
},
|
},
|
||||||
@@ -354,6 +419,9 @@ def create_internal_router(
|
|||||||
return PlannerDecisionResponse(
|
return PlannerDecisionResponse(
|
||||||
tool_name=decision.tool_name,
|
tool_name=decision.tool_name,
|
||||||
arguments=dict(decision.arguments),
|
arguments=dict(decision.arguments),
|
||||||
|
input_tokens=(decision.usage.input_tokens if decision.usage else None),
|
||||||
|
output_tokens=(decision.usage.output_tokens if decision.usage else None),
|
||||||
|
total_tokens=(decision.usage.total_tokens if decision.usage else None),
|
||||||
)
|
)
|
||||||
|
|
||||||
return router
|
return router
|
||||||
|
|||||||
@@ -38,12 +38,22 @@ class HeartbeatRequest(BaseModel):
|
|||||||
host_id: str = Field(min_length=1)
|
host_id: str = Field(min_length=1)
|
||||||
address: str | None = None
|
address: str | None = None
|
||||||
devices: list[DeviceSnapshotModel] = Field(default_factory=list)
|
devices: list[DeviceSnapshotModel] = Field(default_factory=list)
|
||||||
|
policy_revision: int = Field(default=0, ge=0)
|
||||||
|
|
||||||
|
|
||||||
|
class HostGovernancePolicyModel(BaseModel):
|
||||||
|
revision: int = Field(ge=1)
|
||||||
|
self_submission_enabled: bool
|
||||||
|
max_active_tasks: int | None = None
|
||||||
|
daily_token_budget: int | None = None
|
||||||
|
|
||||||
|
|
||||||
class HeartbeatResponse(BaseModel):
|
class HeartbeatResponse(BaseModel):
|
||||||
host_id: str
|
host_id: str
|
||||||
accepted_devices: int
|
accepted_devices: int
|
||||||
received_at: datetime
|
received_at: datetime
|
||||||
|
policy_revision: int = Field(default=0, ge=0)
|
||||||
|
policy: HostGovernancePolicyModel | None = None
|
||||||
|
|
||||||
|
|
||||||
class ClaimRequest(BaseModel):
|
class ClaimRequest(BaseModel):
|
||||||
@@ -93,6 +103,16 @@ class TerminalResultResponse(BaseModel):
|
|||||||
status: Literal["recorded", "already_recorded"]
|
status: Literal["recorded", "already_recorded"]
|
||||||
|
|
||||||
|
|
||||||
|
class HostTaskSubmissionRequest(BaseModel):
|
||||||
|
host_id: str = Field(min_length=1)
|
||||||
|
goal: str = Field(min_length=1)
|
||||||
|
device_id: str | None = Field(default=None, min_length=1)
|
||||||
|
|
||||||
|
|
||||||
|
class HostTaskSubmissionResponse(BaseModel):
|
||||||
|
task_id: str
|
||||||
|
|
||||||
|
|
||||||
class StaleLeaseConflict(BaseModel):
|
class StaleLeaseConflict(BaseModel):
|
||||||
code: Literal["stale_lease"] = "stale_lease"
|
code: Literal["stale_lease"] = "stale_lease"
|
||||||
detail: str
|
detail: str
|
||||||
@@ -116,6 +136,9 @@ class PlannerDecisionRequest(BaseModel):
|
|||||||
class PlannerDecisionResponse(BaseModel):
|
class PlannerDecisionResponse(BaseModel):
|
||||||
tool_name: str
|
tool_name: str
|
||||||
arguments: dict[str, Any] = Field(default_factory=dict)
|
arguments: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
input_tokens: int | None = Field(default=None, ge=0)
|
||||||
|
output_tokens: int | None = Field(default=None, ge=0)
|
||||||
|
total_tokens: int | None = Field(default=None, ge=0)
|
||||||
|
|
||||||
|
|
||||||
class PlannerDecisionError(BaseModel):
|
class PlannerDecisionError(BaseModel):
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""Add durable Cloud governance policies."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "0004_cloud_governance"
|
||||||
|
down_revision = "0003_cloud_user_authentication"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
tables = set(inspector.get_table_names())
|
||||||
|
if "cloud_user_submission_policies" not in tables:
|
||||||
|
op.create_table(
|
||||||
|
"cloud_user_submission_policies",
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.String(),
|
||||||
|
sa.ForeignKey("cloud_users.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
),
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("submission_enabled", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("allowed_host_ids_json", sa.Text(), nullable=True),
|
||||||
|
sa.Column("allowed_device_targets_json", sa.Text(), nullable=True),
|
||||||
|
sa.Column("updated_at", sa.String(), nullable=False),
|
||||||
|
)
|
||||||
|
if "cloud_host_governance_policies" not in tables:
|
||||||
|
op.create_table(
|
||||||
|
"cloud_host_governance_policies",
|
||||||
|
sa.Column(
|
||||||
|
"host_id",
|
||||||
|
sa.String(),
|
||||||
|
sa.ForeignKey("host_registrations.host_id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
),
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("self_submission_enabled", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("max_active_tasks", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("daily_token_budget", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("updated_at", sa.String(), nullable=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
tables = set(inspector.get_table_names())
|
||||||
|
if "cloud_host_governance_policies" in tables:
|
||||||
|
op.drop_table("cloud_host_governance_policies")
|
||||||
|
if "cloud_user_submission_policies" in tables:
|
||||||
|
op.drop_table("cloud_user_submission_policies")
|
||||||
@@ -15,6 +15,7 @@ if TYPE_CHECKING:
|
|||||||
UserAccount,
|
UserAccount,
|
||||||
UserSession,
|
UserSession,
|
||||||
)
|
)
|
||||||
|
from cloud.governance import HostGovernancePolicy, UserSubmissionPolicy
|
||||||
|
|
||||||
|
|
||||||
AttemptStatus = Literal["assigned", "dispatched", "done", "failed", "expired"]
|
AttemptStatus = Literal["assigned", "dispatched", "done", "failed", "expired"]
|
||||||
@@ -283,6 +284,30 @@ class CloudRepository(Protocol):
|
|||||||
|
|
||||||
def cleanup_auth_state(self, *, now: datetime, limit: int) -> int: ...
|
def cleanup_auth_state(self, *, now: datetime, limit: int) -> int: ...
|
||||||
|
|
||||||
|
def get_user_submission_policy(self, user_id: str) -> UserSubmissionPolicy | None: ...
|
||||||
|
|
||||||
|
def upsert_user_submission_policy(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
user_id: str,
|
||||||
|
submission_enabled: bool,
|
||||||
|
allowed_host_ids: tuple[str, ...] | None,
|
||||||
|
allowed_device_targets: tuple[tuple[str, str], ...] | None,
|
||||||
|
updated_at: datetime,
|
||||||
|
) -> UserSubmissionPolicy: ...
|
||||||
|
|
||||||
|
def get_host_governance_policy(self, host_id: str) -> HostGovernancePolicy | None: ...
|
||||||
|
|
||||||
|
def upsert_host_governance_policy(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
host_id: str,
|
||||||
|
self_submission_enabled: bool,
|
||||||
|
max_active_tasks: int | None,
|
||||||
|
daily_token_budget: int | None,
|
||||||
|
updated_at: datetime,
|
||||||
|
) -> HostGovernancePolicy: ...
|
||||||
|
|
||||||
def list_reserved_device_ids(self, *, now: datetime) -> set[str]: ...
|
def list_reserved_device_ids(self, *, now: datetime) -> set[str]: ...
|
||||||
|
|
||||||
def assign_task(
|
def assign_task(
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ class TaskConstraints:
|
|||||||
|
|
||||||
driver_type: str | None = None
|
driver_type: str | None = None
|
||||||
capability_tags: list[str] = field(default_factory=list)
|
capability_tags: list[str] = field(default_factory=list)
|
||||||
|
target_host_id: str | None = None
|
||||||
|
target_device_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -195,6 +197,10 @@ class TaskScheduler:
|
|||||||
|
|
||||||
|
|
||||||
def _matches(device: "PooledDevice", constraints: TaskConstraints) -> bool:
|
def _matches(device: "PooledDevice", constraints: TaskConstraints) -> bool:
|
||||||
|
if constraints.target_host_id and device.host_id != constraints.target_host_id:
|
||||||
|
return False
|
||||||
|
if constraints.target_device_id and device.device_id != constraints.target_device_id:
|
||||||
|
return False
|
||||||
if constraints.driver_type and device.driver_type != constraints.driver_type:
|
if constraints.driver_type and device.driver_type != constraints.driver_type:
|
||||||
return False
|
return False
|
||||||
if constraints.capability_tags:
|
if constraints.capability_tags:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from alembic.runtime.migration import MigrationContext
|
|||||||
from cloud.database import create_database_engine, normalize_database_url
|
from cloud.database import create_database_engine, normalize_database_url
|
||||||
|
|
||||||
|
|
||||||
HEAD_REVISION = "0003_cloud_user_authentication"
|
HEAD_REVISION = "0004_cloud_governance"
|
||||||
|
|
||||||
|
|
||||||
class SchemaVersionError(RuntimeError):
|
class SchemaVersionError(RuntimeError):
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from cloud.auth import (
|
|||||||
NullAuthProvider,
|
NullAuthProvider,
|
||||||
Principal,
|
Principal,
|
||||||
)
|
)
|
||||||
|
from cloud.governance import TaskSubmissionPolicyError, enforce_user_submission_policy
|
||||||
from cloud.sdk.models import (
|
from cloud.sdk.models import (
|
||||||
DeviceResponse,
|
DeviceResponse,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
@@ -94,8 +95,21 @@ def create_cloud_router(
|
|||||||
payload: TaskSubmissionRequest,
|
payload: TaskSubmissionRequest,
|
||||||
request: Request,
|
request: Request,
|
||||||
) -> TaskSubmissionResponse:
|
) -> TaskSubmissionResponse:
|
||||||
_authorize(request, TASKS_SUBMIT_SCOPE)
|
principal = _authorize(request, TASKS_SUBMIT_SCOPE)
|
||||||
task_constraints = _build_constraints(payload.constraints)
|
try:
|
||||||
|
task_constraints = _build_constraints(payload.constraints)
|
||||||
|
_validate_task_target(pool, task_constraints)
|
||||||
|
_enforce_user_policy(principal, scheduler.store, task_constraints)
|
||||||
|
except TaskSubmissionPolicyError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
try:
|
try:
|
||||||
task_id = scheduler.submit(
|
task_id = scheduler.submit(
|
||||||
goal=payload.goal,
|
goal=payload.goal,
|
||||||
@@ -128,6 +142,8 @@ def create_cloud_router(
|
|||||||
attempt_count=task.attempt_count,
|
attempt_count=task.attempt_count,
|
||||||
lease_expires_at=task.lease_expires_at,
|
lease_expires_at=task.lease_expires_at,
|
||||||
failure_reason=task.failure_reason,
|
failure_reason=task.failure_reason,
|
||||||
|
target_host_id=task.constraints.target_host_id,
|
||||||
|
target_device_id=task.constraints.target_device_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
@router.get("/tasks", response_model=TaskListResponse)
|
@router.get("/tasks", response_model=TaskListResponse)
|
||||||
@@ -158,6 +174,8 @@ def create_cloud_router(
|
|||||||
assigned_host_id=task.assigned_host_id,
|
assigned_host_id=task.assigned_host_id,
|
||||||
attempt_count=task.attempt_count,
|
attempt_count=task.attempt_count,
|
||||||
failure_reason=task.failure_reason,
|
failure_reason=task.failure_reason,
|
||||||
|
target_host_id=task.constraints.target_host_id,
|
||||||
|
target_device_id=task.constraints.target_device_id,
|
||||||
created_at=task.created_at,
|
created_at=task.created_at,
|
||||||
)
|
)
|
||||||
for task in tasks
|
for task in tasks
|
||||||
@@ -303,4 +321,35 @@ def _build_constraints(model):
|
|||||||
return TaskConstraints(
|
return TaskConstraints(
|
||||||
driver_type=model.driver_type,
|
driver_type=model.driver_type,
|
||||||
capability_tags=list(model.capability_tags),
|
capability_tags=list(model.capability_tags),
|
||||||
|
target_host_id=model.target_host_id,
|
||||||
|
target_device_id=model.target_device_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_task_target(pool, constraints) -> None:
|
||||||
|
if constraints.target_device_id and not constraints.target_host_id:
|
||||||
|
raise ValueError("target_device_id requires target_host_id")
|
||||||
|
if constraints.target_host_id is None:
|
||||||
|
return
|
||||||
|
if not any(host.host_id == constraints.target_host_id for host in pool.list_hosts()):
|
||||||
|
raise ValueError(f"target host {constraints.target_host_id!r} is not known")
|
||||||
|
if constraints.target_device_id is not None and not any(
|
||||||
|
device.host_id == constraints.target_host_id
|
||||||
|
and device.device_id == constraints.target_device_id
|
||||||
|
for device in pool.list_devices()
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
f"target device {constraints.target_device_id!r} is not owned by "
|
||||||
|
f"host {constraints.target_host_id!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _enforce_user_policy(principal, store, constraints) -> None:
|
||||||
|
if not principal.id.startswith("user:"):
|
||||||
|
return
|
||||||
|
policy = store.get_user_submission_policy(principal.id.removeprefix("user:"))
|
||||||
|
enforce_user_submission_policy(
|
||||||
|
policy,
|
||||||
|
target_host_id=constraints.target_host_id,
|
||||||
|
target_device_id=constraints.target_device_id,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -72,15 +72,24 @@ class CloudClient:
|
|||||||
workflow_definition_id: str | None = None,
|
workflow_definition_id: str | None = None,
|
||||||
driver_type: str | None = None,
|
driver_type: str | None = None,
|
||||||
capability_tags: list[str] | None = None,
|
capability_tags: list[str] | None = None,
|
||||||
|
target_host_id: str | None = None,
|
||||||
|
target_device_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
payload: dict[str, Any] = {
|
payload: dict[str, Any] = {
|
||||||
"goal": goal,
|
"goal": goal,
|
||||||
"workflow_definition_id": workflow_definition_id,
|
"workflow_definition_id": workflow_definition_id,
|
||||||
}
|
}
|
||||||
if driver_type is not None or capability_tags is not None:
|
if (
|
||||||
|
driver_type is not None
|
||||||
|
or capability_tags is not None
|
||||||
|
or target_host_id is not None
|
||||||
|
or target_device_id is not None
|
||||||
|
):
|
||||||
payload["constraints"] = {
|
payload["constraints"] = {
|
||||||
"driver_type": driver_type,
|
"driver_type": driver_type,
|
||||||
"capability_tags": list(capability_tags or []),
|
"capability_tags": list(capability_tags or []),
|
||||||
|
"target_host_id": target_host_id,
|
||||||
|
"target_device_id": target_device_id,
|
||||||
}
|
}
|
||||||
resp = self._request("POST", "/tasks", json=payload)
|
resp = self._request("POST", "/tasks", json=payload)
|
||||||
return resp.json()
|
return resp.json()
|
||||||
@@ -209,6 +218,28 @@ class CloudClient:
|
|||||||
def revoke_user_sessions(self, user_id: str) -> None:
|
def revoke_user_sessions(self, user_id: str) -> None:
|
||||||
self._request("DELETE", f"/users/{user_id}/sessions")
|
self._request("DELETE", f"/users/{user_id}/sessions")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- governance
|
||||||
|
|
||||||
|
def get_user_submission_policy(self, user_id: str) -> dict[str, Any]:
|
||||||
|
return self._request("GET", f"/users/{user_id}/submission-policy").json()
|
||||||
|
|
||||||
|
def update_user_submission_policy(
|
||||||
|
self, user_id: str, **policy: Any
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return self._request(
|
||||||
|
"PUT", f"/users/{user_id}/submission-policy", json=policy
|
||||||
|
).json()
|
||||||
|
|
||||||
|
def get_host_governance_policy(self, host_id: str) -> dict[str, Any]:
|
||||||
|
return self._request("GET", f"/hosts/{host_id}/governance-policy").json()
|
||||||
|
|
||||||
|
def update_host_governance_policy(
|
||||||
|
self, host_id: str, **policy: Any
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return self._request(
|
||||||
|
"PUT", f"/hosts/{host_id}/governance-policy", json=policy
|
||||||
|
).json()
|
||||||
|
|
||||||
# ------------------------------------------------------------------ helpers
|
# ------------------------------------------------------------------ helpers
|
||||||
|
|
||||||
def _url(self, path: str) -> str:
|
def _url(self, path: str) -> str:
|
||||||
|
|||||||
@@ -0,0 +1,169 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException, Request, status
|
||||||
|
|
||||||
|
from cloud.auth import AuthProvider, GOVERNANCE_ADMIN_SCOPE, GOVERNANCE_READ_SCOPE
|
||||||
|
from cloud.observability import current_correlation_id
|
||||||
|
from cloud.sdk.models import (
|
||||||
|
HostGovernancePolicyRequest,
|
||||||
|
HostGovernancePolicyResponse,
|
||||||
|
UserSubmissionPolicyRequest,
|
||||||
|
UserSubmissionPolicyResponse,
|
||||||
|
)
|
||||||
|
from cloud.user_auth import AuthAuditEvent
|
||||||
|
from core.models import utc_now
|
||||||
|
|
||||||
|
|
||||||
|
def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIRouter:
|
||||||
|
router = APIRouter(prefix="/v1", tags=["cloud-governance"])
|
||||||
|
|
||||||
|
def authorize(request: Request, required_scope: str):
|
||||||
|
principal = auth_provider.authenticate(request)
|
||||||
|
if principal is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
detail="unauthorized",
|
||||||
|
headers={"WWW-Authenticate": "Bearer"},
|
||||||
|
)
|
||||||
|
if principal.must_change_password or not principal.has_scope(required_scope):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail=f"missing required scope: {required_scope}",
|
||||||
|
)
|
||||||
|
return principal
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/users/{user_id}/submission-policy",
|
||||||
|
response_model=UserSubmissionPolicyResponse,
|
||||||
|
)
|
||||||
|
def get_user_policy(user_id: str, request: Request) -> UserSubmissionPolicyResponse:
|
||||||
|
authorize(request, GOVERNANCE_READ_SCOPE)
|
||||||
|
policy = repository.get_user_submission_policy(user_id)
|
||||||
|
if policy is None:
|
||||||
|
raise HTTPException(status_code=404, detail="submission policy not found")
|
||||||
|
return UserSubmissionPolicyResponse(
|
||||||
|
user_id=policy.user_id,
|
||||||
|
revision=policy.revision,
|
||||||
|
submission_enabled=policy.submission_enabled,
|
||||||
|
allowed_host_ids=list(policy.allowed_host_ids)
|
||||||
|
if policy.allowed_host_ids is not None
|
||||||
|
else None,
|
||||||
|
allowed_device_targets=[
|
||||||
|
{"host_id": host_id, "device_id": device_id}
|
||||||
|
for host_id, device_id in policy.allowed_device_targets or ()
|
||||||
|
]
|
||||||
|
if policy.allowed_device_targets is not None
|
||||||
|
else None,
|
||||||
|
updated_at=policy.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/users/{user_id}/submission-policy",
|
||||||
|
response_model=UserSubmissionPolicyResponse,
|
||||||
|
)
|
||||||
|
def put_user_policy(
|
||||||
|
user_id: str,
|
||||||
|
payload: UserSubmissionPolicyRequest,
|
||||||
|
request: Request,
|
||||||
|
) -> UserSubmissionPolicyResponse:
|
||||||
|
principal = authorize(request, GOVERNANCE_ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
policy = repository.upsert_user_submission_policy(
|
||||||
|
user_id=user_id,
|
||||||
|
submission_enabled=payload.submission_enabled,
|
||||||
|
allowed_host_ids=_unique_strings(payload.allowed_host_ids),
|
||||||
|
allowed_device_targets=(
|
||||||
|
tuple((item.host_id, item.device_id) for item in payload.allowed_device_targets)
|
||||||
|
if payload.allowed_device_targets is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
updated_at=utc_now(),
|
||||||
|
)
|
||||||
|
except KeyError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail="user not found") from exc
|
||||||
|
_audit(repository, principal.id, user_id, "user_submission_policy_update")
|
||||||
|
return UserSubmissionPolicyResponse(
|
||||||
|
user_id=policy.user_id,
|
||||||
|
revision=policy.revision,
|
||||||
|
submission_enabled=policy.submission_enabled,
|
||||||
|
allowed_host_ids=list(policy.allowed_host_ids)
|
||||||
|
if policy.allowed_host_ids is not None
|
||||||
|
else None,
|
||||||
|
allowed_device_targets=[
|
||||||
|
{"host_id": host_id, "device_id": device_id}
|
||||||
|
for host_id, device_id in policy.allowed_device_targets or ()
|
||||||
|
]
|
||||||
|
if policy.allowed_device_targets is not None
|
||||||
|
else None,
|
||||||
|
updated_at=policy.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/hosts/{host_id}/governance-policy",
|
||||||
|
response_model=HostGovernancePolicyResponse,
|
||||||
|
)
|
||||||
|
def get_host_policy(host_id: str, request: Request) -> HostGovernancePolicyResponse:
|
||||||
|
authorize(request, GOVERNANCE_READ_SCOPE)
|
||||||
|
policy = repository.get_host_governance_policy(host_id)
|
||||||
|
if policy is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Host policy not found")
|
||||||
|
return _host_response(policy)
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/hosts/{host_id}/governance-policy",
|
||||||
|
response_model=HostGovernancePolicyResponse,
|
||||||
|
)
|
||||||
|
def put_host_policy(
|
||||||
|
host_id: str,
|
||||||
|
payload: HostGovernancePolicyRequest,
|
||||||
|
request: Request,
|
||||||
|
) -> HostGovernancePolicyResponse:
|
||||||
|
principal = authorize(request, GOVERNANCE_ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
policy = repository.upsert_host_governance_policy(
|
||||||
|
host_id=host_id,
|
||||||
|
self_submission_enabled=payload.self_submission_enabled,
|
||||||
|
max_active_tasks=payload.max_active_tasks,
|
||||||
|
daily_token_budget=payload.daily_token_budget,
|
||||||
|
updated_at=utc_now(),
|
||||||
|
)
|
||||||
|
except KeyError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail="Host not found") from exc
|
||||||
|
_audit(repository, principal.id, host_id, "host_governance_policy_update")
|
||||||
|
return _host_response(policy)
|
||||||
|
|
||||||
|
return router
|
||||||
|
|
||||||
|
|
||||||
|
def _unique_strings(values: list[str] | None) -> tuple[str, ...] | None:
|
||||||
|
if values is None:
|
||||||
|
return None
|
||||||
|
return tuple(dict.fromkeys(value for value in values if value))
|
||||||
|
|
||||||
|
|
||||||
|
def _host_response(policy) -> HostGovernancePolicyResponse:
|
||||||
|
return HostGovernancePolicyResponse(
|
||||||
|
host_id=policy.host_id,
|
||||||
|
revision=policy.revision,
|
||||||
|
self_submission_enabled=policy.self_submission_enabled,
|
||||||
|
max_active_tasks=policy.max_active_tasks,
|
||||||
|
daily_token_budget=policy.daily_token_budget,
|
||||||
|
updated_at=policy.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _audit(repository, actor_id: str, target: str, action: str) -> None:
|
||||||
|
repository.record_auth_audit(
|
||||||
|
AuthAuditEvent(
|
||||||
|
id=uuid4().hex,
|
||||||
|
occurred_at=utc_now(),
|
||||||
|
actor_principal_id=actor_id,
|
||||||
|
target_user_id=target if action.startswith("user_") else None,
|
||||||
|
action=action,
|
||||||
|
outcome="success",
|
||||||
|
correlation_id=current_correlation_id(),
|
||||||
|
metadata={"target": target},
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -11,6 +11,8 @@ from pydantic import BaseModel, Field
|
|||||||
class TaskConstraintsModel(BaseModel):
|
class TaskConstraintsModel(BaseModel):
|
||||||
driver_type: str | None = None
|
driver_type: str | None = None
|
||||||
capability_tags: list[str] = Field(default_factory=list)
|
capability_tags: list[str] = Field(default_factory=list)
|
||||||
|
target_host_id: str | None = Field(default=None, min_length=1)
|
||||||
|
target_device_id: str | None = Field(default=None, min_length=1)
|
||||||
|
|
||||||
|
|
||||||
class TaskSubmissionRequest(BaseModel):
|
class TaskSubmissionRequest(BaseModel):
|
||||||
@@ -33,6 +35,8 @@ class TaskStatusResponse(BaseModel):
|
|||||||
attempt_count: int = 0
|
attempt_count: int = 0
|
||||||
lease_expires_at: datetime | None = None
|
lease_expires_at: datetime | None = None
|
||||||
failure_reason: str | None = None
|
failure_reason: str | None = None
|
||||||
|
target_host_id: str | None = None
|
||||||
|
target_device_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class TaskListItem(BaseModel):
|
class TaskListItem(BaseModel):
|
||||||
@@ -44,6 +48,8 @@ class TaskListItem(BaseModel):
|
|||||||
assigned_host_id: str | None = None
|
assigned_host_id: str | None = None
|
||||||
attempt_count: int = 0
|
attempt_count: int = 0
|
||||||
failure_reason: str | None = None
|
failure_reason: str | None = None
|
||||||
|
target_host_id: str | None = None
|
||||||
|
target_device_id: str | None = None
|
||||||
created_at: datetime
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
@@ -145,3 +151,32 @@ class PasswordResetRequest(BaseModel):
|
|||||||
|
|
||||||
class ErrorResponse(BaseModel):
|
class ErrorResponse(BaseModel):
|
||||||
detail: str
|
detail: str
|
||||||
|
|
||||||
|
|
||||||
|
class DeviceTargetModel(BaseModel):
|
||||||
|
host_id: str = Field(min_length=1)
|
||||||
|
device_id: str = Field(min_length=1)
|
||||||
|
|
||||||
|
|
||||||
|
class UserSubmissionPolicyRequest(BaseModel):
|
||||||
|
submission_enabled: bool = True
|
||||||
|
allowed_host_ids: list[str] | None = None
|
||||||
|
allowed_device_targets: list[DeviceTargetModel] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class UserSubmissionPolicyResponse(UserSubmissionPolicyRequest):
|
||||||
|
user_id: str
|
||||||
|
revision: int
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class HostGovernancePolicyRequest(BaseModel):
|
||||||
|
self_submission_enabled: bool = True
|
||||||
|
max_active_tasks: int | None = Field(default=None, ge=1)
|
||||||
|
daily_token_budget: int | None = Field(default=None, ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class HostGovernancePolicyResponse(HostGovernancePolicyRequest):
|
||||||
|
host_id: str
|
||||||
|
revision: int
|
||||||
|
updated_at: datetime
|
||||||
|
|||||||
@@ -19,9 +19,11 @@ from cloud.db_models import (
|
|||||||
ScheduledTaskRow,
|
ScheduledTaskRow,
|
||||||
TaskAttemptRow,
|
TaskAttemptRow,
|
||||||
AuthAuditRow,
|
AuthAuditRow,
|
||||||
|
HostGovernancePolicyRow,
|
||||||
LoginThrottleRow,
|
LoginThrottleRow,
|
||||||
UserRow,
|
UserRow,
|
||||||
UserSessionRow,
|
UserSessionRow,
|
||||||
|
UserSubmissionPolicyRow,
|
||||||
)
|
)
|
||||||
from cloud.observability import current_correlation_id
|
from cloud.observability import current_correlation_id
|
||||||
from core.models import utc_now
|
from core.models import utc_now
|
||||||
@@ -807,6 +809,86 @@ class SQLAlchemyCloudRepository:
|
|||||||
removed += len(stale_throttles)
|
removed += len(stale_throttles)
|
||||||
return removed
|
return removed
|
||||||
|
|
||||||
|
# -------------------------------------------------------------- governance
|
||||||
|
|
||||||
|
def get_user_submission_policy(self, user_id: str) -> Any | None:
|
||||||
|
with self._sessions() as session:
|
||||||
|
row = session.get(UserSubmissionPolicyRow, user_id)
|
||||||
|
return _user_submission_policy_from_row(row) if row is not None else None
|
||||||
|
|
||||||
|
def upsert_user_submission_policy(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
user_id: str,
|
||||||
|
submission_enabled: bool,
|
||||||
|
allowed_host_ids: tuple[str, ...] | None,
|
||||||
|
allowed_device_targets: tuple[tuple[str, str], ...] | None,
|
||||||
|
updated_at: datetime,
|
||||||
|
) -> Any:
|
||||||
|
with self._sessions.begin() as session:
|
||||||
|
if session.get(UserRow, user_id) is None:
|
||||||
|
raise KeyError(f"unknown user {user_id!r}")
|
||||||
|
row = session.get(UserSubmissionPolicyRow, user_id)
|
||||||
|
if row is None:
|
||||||
|
row = UserSubmissionPolicyRow(
|
||||||
|
user_id=user_id,
|
||||||
|
revision=1,
|
||||||
|
submission_enabled=1 if submission_enabled else 0,
|
||||||
|
allowed_host_ids_json=_dump_optional_list(allowed_host_ids),
|
||||||
|
allowed_device_targets_json=_dump_optional_list(
|
||||||
|
allowed_device_targets
|
||||||
|
),
|
||||||
|
updated_at=_iso(updated_at),
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
else:
|
||||||
|
row.revision += 1
|
||||||
|
row.submission_enabled = 1 if submission_enabled else 0
|
||||||
|
row.allowed_host_ids_json = _dump_optional_list(allowed_host_ids)
|
||||||
|
row.allowed_device_targets_json = _dump_optional_list(
|
||||||
|
allowed_device_targets
|
||||||
|
)
|
||||||
|
row.updated_at = _iso(updated_at)
|
||||||
|
session.flush()
|
||||||
|
return _user_submission_policy_from_row(row)
|
||||||
|
|
||||||
|
def get_host_governance_policy(self, host_id: str) -> Any | None:
|
||||||
|
with self._sessions() as session:
|
||||||
|
row = session.get(HostGovernancePolicyRow, host_id)
|
||||||
|
return _host_governance_policy_from_row(row) if row is not None else None
|
||||||
|
|
||||||
|
def upsert_host_governance_policy(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
host_id: str,
|
||||||
|
self_submission_enabled: bool,
|
||||||
|
max_active_tasks: int | None,
|
||||||
|
daily_token_budget: int | None,
|
||||||
|
updated_at: datetime,
|
||||||
|
) -> Any:
|
||||||
|
with self._sessions.begin() as session:
|
||||||
|
if session.get(HostRow, host_id) is None:
|
||||||
|
raise KeyError(f"unknown host {host_id!r}")
|
||||||
|
row = session.get(HostGovernancePolicyRow, host_id)
|
||||||
|
if row is None:
|
||||||
|
row = HostGovernancePolicyRow(
|
||||||
|
host_id=host_id,
|
||||||
|
revision=1,
|
||||||
|
self_submission_enabled=1 if self_submission_enabled else 0,
|
||||||
|
max_active_tasks=max_active_tasks,
|
||||||
|
daily_token_budget=daily_token_budget,
|
||||||
|
updated_at=_iso(updated_at),
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
else:
|
||||||
|
row.revision += 1
|
||||||
|
row.self_submission_enabled = 1 if self_submission_enabled else 0
|
||||||
|
row.max_active_tasks = max_active_tasks
|
||||||
|
row.daily_token_budget = daily_token_budget
|
||||||
|
row.updated_at = _iso(updated_at)
|
||||||
|
session.flush()
|
||||||
|
return _host_governance_policy_from_row(row)
|
||||||
|
|
||||||
def list_reserved_device_ids(self, *, now: datetime) -> set[str]:
|
def list_reserved_device_ids(self, *, now: datetime) -> set[str]:
|
||||||
with self._sessions() as session:
|
with self._sessions() as session:
|
||||||
device_ids = session.scalars(
|
device_ids = session.scalars(
|
||||||
@@ -1218,6 +1300,8 @@ def _task_from_row(row: ScheduledTaskRow) -> Any:
|
|||||||
constraints=TaskConstraints(
|
constraints=TaskConstraints(
|
||||||
driver_type=constraints_data.get("driver_type"),
|
driver_type=constraints_data.get("driver_type"),
|
||||||
capability_tags=list(constraints_data.get("capability_tags") or []),
|
capability_tags=list(constraints_data.get("capability_tags") or []),
|
||||||
|
target_host_id=constraints_data.get("target_host_id"),
|
||||||
|
target_device_id=constraints_data.get("target_device_id"),
|
||||||
),
|
),
|
||||||
status=row.status,
|
status=row.status,
|
||||||
assigned_device_id=row.assigned_device_id,
|
assigned_device_id=row.assigned_device_id,
|
||||||
@@ -1232,6 +1316,66 @@ def _task_from_row(row: ScheduledTaskRow) -> Any:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_optional_list(value: tuple[Any, ...] | None) -> str | None:
|
||||||
|
return json.dumps(value, ensure_ascii=False) if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_optional_string_tuple(value: str | None) -> tuple[str, ...] | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = json.loads(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
parsed = []
|
||||||
|
return tuple(item for item in parsed if isinstance(item, str))
|
||||||
|
|
||||||
|
|
||||||
|
def _load_optional_device_targets(
|
||||||
|
value: str | None,
|
||||||
|
) -> tuple[tuple[str, str], ...] | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = json.loads(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
parsed = []
|
||||||
|
return tuple(
|
||||||
|
(item[0], item[1])
|
||||||
|
for item in parsed
|
||||||
|
if isinstance(item, list | tuple)
|
||||||
|
and len(item) == 2
|
||||||
|
and all(isinstance(part, str) for part in item)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _user_submission_policy_from_row(row: UserSubmissionPolicyRow) -> Any:
|
||||||
|
from cloud.governance import UserSubmissionPolicy
|
||||||
|
|
||||||
|
return UserSubmissionPolicy(
|
||||||
|
user_id=row.user_id,
|
||||||
|
revision=row.revision,
|
||||||
|
submission_enabled=bool(row.submission_enabled),
|
||||||
|
allowed_host_ids=_load_optional_string_tuple(row.allowed_host_ids_json),
|
||||||
|
allowed_device_targets=_load_optional_device_targets(
|
||||||
|
row.allowed_device_targets_json
|
||||||
|
),
|
||||||
|
updated_at=_parse_dt(row.updated_at) or utc_now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _host_governance_policy_from_row(row: HostGovernancePolicyRow) -> Any:
|
||||||
|
from cloud.governance import HostGovernancePolicy
|
||||||
|
|
||||||
|
return HostGovernancePolicy(
|
||||||
|
host_id=row.host_id,
|
||||||
|
revision=row.revision,
|
||||||
|
self_submission_enabled=bool(row.self_submission_enabled),
|
||||||
|
max_active_tasks=row.max_active_tasks,
|
||||||
|
daily_token_budget=row.daily_token_budget,
|
||||||
|
updated_at=_parse_dt(row.updated_at) or utc_now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _task_attempt_from_row(row: TaskAttemptRow) -> Any:
|
def _task_attempt_from_row(row: TaskAttemptRow) -> Any:
|
||||||
from cloud.repository import TaskAttemptRecord
|
from cloud.repository import TaskAttemptRecord
|
||||||
|
|
||||||
|
|||||||
@@ -13,10 +13,18 @@ class ToolCallUnavailable(Exception):
|
|||||||
"""Internal signal for expected tool-calling transport/response failures."""
|
"""Internal signal for expected tool-calling transport/response failures."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ToolCallUsage:
|
||||||
|
input_tokens: int | None = None
|
||||||
|
output_tokens: int | None = None
|
||||||
|
total_tokens: int | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ToolCallDecision:
|
class ToolCallDecision:
|
||||||
tool_name: str
|
tool_name: str
|
||||||
arguments: dict[str, Any]
|
arguments: dict[str, Any]
|
||||||
|
usage: ToolCallUsage | None = None
|
||||||
|
|
||||||
|
|
||||||
class ToolCallingClient(Protocol):
|
class ToolCallingClient(Protocol):
|
||||||
@@ -234,7 +242,11 @@ def _decision_from_anthropic_response(response: Any) -> ToolCallDecision:
|
|||||||
name = _value(block, "name")
|
name = _value(block, "name")
|
||||||
arguments = _value(block, "input")
|
arguments = _value(block, "input")
|
||||||
if isinstance(name, str) and isinstance(arguments, dict):
|
if isinstance(name, str) and isinstance(arguments, dict):
|
||||||
return ToolCallDecision(tool_name=name, arguments=arguments)
|
return ToolCallDecision(
|
||||||
|
tool_name=name,
|
||||||
|
arguments=arguments,
|
||||||
|
usage=_anthropic_usage(response),
|
||||||
|
)
|
||||||
raise ValueError("anthropic response did not include a tool_use block")
|
raise ValueError("anthropic response did not include a tool_use block")
|
||||||
|
|
||||||
|
|
||||||
@@ -278,7 +290,40 @@ def _decision_from_openai_response(response: Any) -> ToolCallDecision:
|
|||||||
if not isinstance(name, str):
|
if not isinstance(name, str):
|
||||||
raise ValueError("openai tool call missing a function name")
|
raise ValueError("openai tool call missing a function name")
|
||||||
arguments = _decode_openai_arguments(_value(function, "arguments"))
|
arguments = _decode_openai_arguments(_value(function, "arguments"))
|
||||||
return ToolCallDecision(tool_name=name, arguments=arguments)
|
return ToolCallDecision(
|
||||||
|
tool_name=name,
|
||||||
|
arguments=arguments,
|
||||||
|
usage=_openai_usage(response),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _anthropic_usage(response: Any) -> ToolCallUsage | None:
|
||||||
|
usage = _value(response, "usage")
|
||||||
|
input_tokens = _integer_value(usage, "input_tokens")
|
||||||
|
output_tokens = _integer_value(usage, "output_tokens")
|
||||||
|
if input_tokens is None and output_tokens is None:
|
||||||
|
return None
|
||||||
|
return ToolCallUsage(
|
||||||
|
input_tokens=input_tokens,
|
||||||
|
output_tokens=output_tokens,
|
||||||
|
total_tokens=(input_tokens or 0) + (output_tokens or 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _openai_usage(response: Any) -> ToolCallUsage | None:
|
||||||
|
usage = _value(response, "usage")
|
||||||
|
input_tokens = _integer_value(usage, "prompt_tokens")
|
||||||
|
output_tokens = _integer_value(usage, "completion_tokens")
|
||||||
|
total_tokens = _integer_value(usage, "total_tokens")
|
||||||
|
if input_tokens is None and output_tokens is None and total_tokens is None:
|
||||||
|
return None
|
||||||
|
return ToolCallUsage(
|
||||||
|
input_tokens=input_tokens,
|
||||||
|
output_tokens=output_tokens,
|
||||||
|
total_tokens=total_tokens
|
||||||
|
if total_tokens is not None
|
||||||
|
else (input_tokens or 0) + (output_tokens or 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _decode_openai_arguments(raw_arguments: Any) -> dict[str, Any]:
|
def _decode_openai_arguments(raw_arguments: Any) -> dict[str, Any]:
|
||||||
@@ -296,3 +341,8 @@ def _value(source: Any, key: str) -> Any:
|
|||||||
return source.get(key)
|
return source.get(key)
|
||||||
value = getattr(source, key, None)
|
value = getattr(source, key, None)
|
||||||
return None if callable(value) else value
|
return None if callable(value) else value
|
||||||
|
|
||||||
|
|
||||||
|
def _integer_value(source: Any, key: str) -> int | None:
|
||||||
|
value = _value(source, key)
|
||||||
|
return value if isinstance(value, int) and value >= 0 else None
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from cloud.auth import Principal
|
||||||
|
from cloud.config import CloudConfig
|
||||||
|
from cloud.database import CloudDatabase
|
||||||
|
from cloud.governance import enforce_user_submission_policy
|
||||||
|
from cloud.plugins import PluginRegistry
|
||||||
|
from cloud.pool import DevicePool
|
||||||
|
from cloud.sdk.api import create_cloud_router
|
||||||
|
from cloud.sdk.governance_api import create_governance_router
|
||||||
|
from cloud.scheduler import TaskScheduler
|
||||||
|
from cloud.user_auth import UserAccount
|
||||||
|
from core.models import Device
|
||||||
|
|
||||||
|
|
||||||
|
pytest.importorskip("fastapi")
|
||||||
|
from fastapi import FastAPI # noqa: E402
|
||||||
|
from fastapi.testclient import TestClient # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class _PrincipalProvider:
|
||||||
|
def __init__(self, principal: Principal) -> None:
|
||||||
|
self.principal = principal
|
||||||
|
|
||||||
|
def authenticate(self, _request: object) -> Principal:
|
||||||
|
return self.principal
|
||||||
|
|
||||||
|
|
||||||
|
def _config() -> CloudConfig:
|
||||||
|
return CloudConfig(max_queue_depth=20)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_user(repository, user_id: str = "user-a") -> None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
repository.create_user(
|
||||||
|
UserAccount(
|
||||||
|
id=user_id,
|
||||||
|
username=user_id,
|
||||||
|
username_normalized=user_id,
|
||||||
|
display_name=user_id,
|
||||||
|
role="operator",
|
||||||
|
enabled=True,
|
||||||
|
must_change_password=False,
|
||||||
|
authentication_version=1,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
password_hash="not-used-by-this-test",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_submission_policy_requires_permitted_explicit_target(tmp_path) -> None:
|
||||||
|
database = CloudDatabase(f"sqlite:///{(tmp_path / 'governance.sqlite3').as_posix()}")
|
||||||
|
try:
|
||||||
|
_add_user(database.repository)
|
||||||
|
database.repository.upsert_user_submission_policy(
|
||||||
|
user_id="user-a",
|
||||||
|
submission_enabled=True,
|
||||||
|
allowed_host_ids=("host-a",),
|
||||||
|
allowed_device_targets=(("host-a", "device-a"),),
|
||||||
|
updated_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
pool = DevicePool(database.repository, _config())
|
||||||
|
pool.sync_host_devices(
|
||||||
|
"host-a",
|
||||||
|
[Device(id="device-a", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
scheduler = TaskScheduler(pool, database.repository, _config())
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
create_cloud_router(
|
||||||
|
pool=pool,
|
||||||
|
scheduler=scheduler,
|
||||||
|
plugin_registry=PluginRegistry(database.repository),
|
||||||
|
auth_provider=_PrincipalProvider(
|
||||||
|
Principal(id="user:user-a", scopes=frozenset({"tasks:submit"}))
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
denied = client.post("/v1/tasks", json={"goal": "unscoped"})
|
||||||
|
assert denied.status_code == 403
|
||||||
|
allowed = client.post(
|
||||||
|
"/v1/tasks",
|
||||||
|
json={
|
||||||
|
"goal": "scoped",
|
||||||
|
"constraints": {
|
||||||
|
"target_host_id": "host-a",
|
||||||
|
"target_device_id": "device-a",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert allowed.status_code == 201, allowed.text
|
||||||
|
finally:
|
||||||
|
database.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_governance_routes_persist_revisioned_policies(tmp_path) -> None:
|
||||||
|
database = CloudDatabase(f"sqlite:///{(tmp_path / 'governance-api.sqlite3').as_posix()}")
|
||||||
|
try:
|
||||||
|
_add_user(database.repository)
|
||||||
|
database.repository.upsert_host(
|
||||||
|
"host-a", address=None, last_seen_at=datetime.now(UTC)
|
||||||
|
)
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(
|
||||||
|
create_governance_router(
|
||||||
|
repository=database.repository,
|
||||||
|
auth_provider=_PrincipalProvider(
|
||||||
|
Principal(id="admin", scopes=frozenset({"governance:admin", "governance:read"}))
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
user_policy = client.put(
|
||||||
|
"/v1/users/user-a/submission-policy",
|
||||||
|
json={"submission_enabled": False, "allowed_host_ids": []},
|
||||||
|
)
|
||||||
|
assert user_policy.status_code == 200, user_policy.text
|
||||||
|
assert user_policy.json()["revision"] == 1
|
||||||
|
host_policy = client.put(
|
||||||
|
"/v1/hosts/host-a/governance-policy",
|
||||||
|
json={"max_active_tasks": 2, "daily_token_budget": 1000},
|
||||||
|
)
|
||||||
|
assert host_policy.status_code == 200, host_policy.text
|
||||||
|
assert host_policy.json()["revision"] == 1
|
||||||
|
reread = client.get("/v1/hosts/host-a/governance-policy")
|
||||||
|
assert reread.json()["daily_token_budget"] == 1000
|
||||||
|
finally:
|
||||||
|
database.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_absent_policy_preserves_existing_submission_behavior() -> None:
|
||||||
|
enforce_user_submission_policy(None, target_host_id=None, target_device_id=None)
|
||||||
@@ -267,6 +267,70 @@ def test_submit_with_constraints(tmp_path) -> None:
|
|||||||
assert status["status"] == "queued"
|
assert status["status"] == "queued"
|
||||||
|
|
||||||
|
|
||||||
|
def test_submit_with_explicit_target_is_listed_and_not_rerouted(tmp_path) -> None:
|
||||||
|
app, pool, scheduler, _ = _build_app(tmp_path)
|
||||||
|
pool.sync_host_devices(
|
||||||
|
"host-a",
|
||||||
|
[Device(id="device-a", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
pool.sync_host_devices(
|
||||||
|
"host-b",
|
||||||
|
[Device(id="device-b", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
client = _client_for(app)
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
"/v1/tasks",
|
||||||
|
json={
|
||||||
|
"goal": "target b",
|
||||||
|
"constraints": {
|
||||||
|
"target_host_id": "host-b",
|
||||||
|
"target_device_id": "device-b",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert response.status_code == 201, response.text
|
||||||
|
task_id = response.json()["task_id"]
|
||||||
|
|
||||||
|
scheduler.assign()
|
||||||
|
task = client.get(f"/v1/tasks/{task_id}").json()
|
||||||
|
assert task["target_host_id"] == "host-b"
|
||||||
|
assert task["target_device_id"] == "device-b"
|
||||||
|
assert task["assigned_host_id"] == "host-b"
|
||||||
|
assert task["assigned_device_id"] == "device-b"
|
||||||
|
listed = client.get("/v1/tasks").json()["items"]
|
||||||
|
assert next(item for item in listed if item["id"] == task_id)["target_host_id"] == "host-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_submit_rejects_incomplete_or_foreign_target(tmp_path) -> None:
|
||||||
|
app, pool, _, _ = _build_app(tmp_path)
|
||||||
|
pool.sync_host_devices(
|
||||||
|
"host-a",
|
||||||
|
[Device(id="device-a", driver_type="wda", status="idle")], # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
client = _client_for(app)
|
||||||
|
|
||||||
|
missing_host = client.post(
|
||||||
|
"/v1/tasks",
|
||||||
|
json={
|
||||||
|
"goal": "invalid",
|
||||||
|
"constraints": {"target_device_id": "device-a"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert missing_host.status_code == 400
|
||||||
|
foreign_device = client.post(
|
||||||
|
"/v1/tasks",
|
||||||
|
json={
|
||||||
|
"goal": "invalid",
|
||||||
|
"constraints": {
|
||||||
|
"target_host_id": "host-a",
|
||||||
|
"target_device_id": "unknown",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert foreign_device.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("method", "path", "payload", "required_scope"),
|
("method", "path", "payload", "required_scope"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from cloud.auth import BearerCredential, ConfiguredBearerAuthProvider
|
|||||||
from cloud.config import CloudConfig
|
from cloud.config import CloudConfig
|
||||||
from cloud.internal_api.api import create_internal_router
|
from cloud.internal_api.api import create_internal_router
|
||||||
from cloud.pool import DevicePool, PooledDevice
|
from cloud.pool import DevicePool, PooledDevice
|
||||||
from cloud.scheduler import ScheduledTask, TaskConstraints
|
from cloud.scheduler import ScheduledTask, TaskConstraints, TaskScheduler
|
||||||
from cloud.store import CloudStore
|
from cloud.store import CloudStore
|
||||||
|
|
||||||
|
|
||||||
@@ -34,7 +34,13 @@ def _build_client(tmp_path) -> tuple[TestClient, DevicePool]:
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
app.include_router(create_internal_router(pool=pool, auth_provider=auth_provider))
|
app.include_router(
|
||||||
|
create_internal_router(
|
||||||
|
pool=pool,
|
||||||
|
auth_provider=auth_provider,
|
||||||
|
scheduler=TaskScheduler(pool, pool.store, CloudConfig(stale_after_seconds=60)),
|
||||||
|
)
|
||||||
|
)
|
||||||
return TestClient(app), pool
|
return TestClient(app), pool
|
||||||
|
|
||||||
|
|
||||||
@@ -196,6 +202,36 @@ def test_host_token_cannot_submit_heartbeat_for_another_host(tmp_path) -> None:
|
|||||||
assert pool.store.get_host("host-b") is None
|
assert pool.store.get_host("host-b") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_heartbeat_and_self_submission_preserve_host_isolation(tmp_path) -> None:
|
||||||
|
client, pool = _build_client(tmp_path)
|
||||||
|
headers = {"Authorization": "Bearer token-a"}
|
||||||
|
heartbeat = client.put(
|
||||||
|
"/internal/v1/hosts/host-a/heartbeat",
|
||||||
|
headers=headers,
|
||||||
|
json=_heartbeat_payload("host-a", "device-a"),
|
||||||
|
)
|
||||||
|
assert heartbeat.status_code == 200
|
||||||
|
assert heartbeat.json()["policy_revision"] == 0
|
||||||
|
assert heartbeat.json()["policy"] is None
|
||||||
|
|
||||||
|
created = client.post(
|
||||||
|
"/internal/v1/hosts/host-a/tasks",
|
||||||
|
headers=headers,
|
||||||
|
json={"host_id": "host-a", "goal": "local work", "device_id": "device-a"},
|
||||||
|
)
|
||||||
|
assert created.status_code == 201, created.text
|
||||||
|
task = pool.store.get_task(created.json()["task_id"])
|
||||||
|
assert task is not None
|
||||||
|
assert task.constraints.target_host_id == "host-a"
|
||||||
|
assert task.constraints.target_device_id == "device-a"
|
||||||
|
foreign = client.post(
|
||||||
|
"/internal/v1/hosts/host-b/tasks",
|
||||||
|
headers=headers,
|
||||||
|
json={"host_id": "host-b", "goal": "forbidden"},
|
||||||
|
)
|
||||||
|
assert foreign.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
def test_long_poll_claim_returns_at_most_one_owned_assignment(tmp_path) -> None:
|
def test_long_poll_claim_returns_at_most_one_owned_assignment(tmp_path) -> None:
|
||||||
client, pool = _build_client(tmp_path)
|
client, pool = _build_client(tmp_path)
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
|
|||||||
@@ -225,3 +225,42 @@ def test_capability_tag_constraint_filters_candidates(tmp_path) -> None:
|
|||||||
assignments = scheduler.assign()
|
assignments = scheduler.assign()
|
||||||
assert [a.task_id for a in assignments] == [task_id]
|
assert [a.task_id for a in assignments] == [task_id]
|
||||||
assert assignments[0].device_id == "dev-2"
|
assert assignments[0].device_id == "dev-2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_host_and_device_target_is_a_hard_constraint(tmp_path) -> None:
|
||||||
|
pool = _pool_with_devices(tmp_path, _device("host-a-device"), host_id="host-a")
|
||||||
|
pool.sync_host_devices("host-b", [_device("host-b-device")])
|
||||||
|
scheduler = TaskScheduler(pool, pool.store, _config())
|
||||||
|
|
||||||
|
task_id = scheduler.submit(
|
||||||
|
goal="target host b",
|
||||||
|
constraints=TaskConstraints(
|
||||||
|
target_host_id="host-b",
|
||||||
|
target_device_id="host-b-device",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assignments = scheduler.assign()
|
||||||
|
|
||||||
|
assert [(item.host_id, item.device_id) for item in assignments] == [
|
||||||
|
("host-b", "host-b-device")
|
||||||
|
]
|
||||||
|
task = pool.store.get_task(task_id)
|
||||||
|
assert task is not None
|
||||||
|
assert task.constraints.target_host_id == "host-b"
|
||||||
|
assert task.constraints.target_device_id == "host-b-device"
|
||||||
|
|
||||||
|
|
||||||
|
def test_unavailable_explicit_target_is_never_rerouted(tmp_path) -> None:
|
||||||
|
pool = _pool_with_devices(tmp_path, _device("host-a-device"), host_id="host-a")
|
||||||
|
pool.sync_host_devices("host-b", [_device("host-b-device", status="busy")])
|
||||||
|
scheduler = TaskScheduler(pool, pool.store, _config())
|
||||||
|
task_id = scheduler.submit(
|
||||||
|
goal="wait for host b",
|
||||||
|
constraints=TaskConstraints(target_host_id="host-b"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert scheduler.assign() == []
|
||||||
|
task = pool.store.get_task(task_id)
|
||||||
|
assert task is not None
|
||||||
|
assert task.status == "queued"
|
||||||
|
|||||||
Reference in New Issue
Block a user