feat(cloud): enforce host governance budgets
Tests / Test passed: 662

This commit is contained in:
2026-07-13 22:56:31 +08:00
parent 2cd314b183
commit b4803f90e6
28 changed files with 1311 additions and 14 deletions
+10
View File
@@ -301,6 +301,12 @@ def create_app(
auth_provider=auth_provider,
lease_duration_seconds=control_config.lease_duration_seconds,
scheduler=scheduler,
planner_token_reservation_ceiling=(
control_config.planner_token_reservation_ceiling
),
planner_token_reservation_ttl_seconds=(
control_config.planner_token_reservation_ttl_seconds
),
)
)
@@ -397,6 +403,10 @@ async def _run_lease_reaper_loop(
now=utc_now(),
max_attempts=max_attempts,
)
services.repository.cleanup_expired_token_reservations(
now=utc_now(),
limit=100,
)
except Exception:
logger.exception(
"cloud lifecycle iteration failed",
+9
View File
@@ -19,6 +19,7 @@ from host_agent.history import ConsoleHistoryStore
from host_agent.identity import HostIdentityStore
from host_agent.lease import ActiveAssignmentRunner
from host_agent.local_account import LocalAccountStore
from host_agent.policy_cache import HostPolicyCacheStore
from host_agent.processor import AssignmentProcessingResult, AssignmentProcessor
from host_agent.status import AgentStatusTracker
from host_agent.web.app import create_console_app
@@ -195,6 +196,14 @@ def create_application(
if history_store is not None
else None
),
policy_cache=HostPolicyCacheStore(
resolved_config.identity_path.parent / "host_governance_policy.json"
),
on_policy_sync=(
(lambda revision: history_store.record_policy_sync(revision=revision))
if history_store is not None
else None
),
)
executor = AssignmentExecutor(
create_execution_factories(
+19 -2
View File
@@ -8,6 +8,7 @@ from core.errors import DeviceRuntimeError
from device.manager import DeviceManager
from host_agent.client import HostAgentClient
from host_agent.config import HostAgentConfig
from host_agent.policy_cache import HostPolicyCacheStore
from host_agent.status import AgentStatusTracker
if TYPE_CHECKING:
@@ -37,6 +38,8 @@ class HeartbeatSynchronizer:
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
status_tracker: AgentStatusTracker | None = None,
on_sync: Callable[[int], None] | None = None,
policy_cache: HostPolicyCacheStore | None = None,
on_policy_sync: Callable[[int], None] | None = None,
) -> None:
self.manager = manager
self.client = client
@@ -45,8 +48,12 @@ class HeartbeatSynchronizer:
self._sleep = sleep
self.status_tracker = status_tracker
self.on_sync = on_sync
self.policy_revision = 0
self.policy = None
self.policy_cache = policy_cache
self.on_policy_sync = on_policy_sync
self.policy = policy_cache.load() if policy_cache is not None else None
self.policy_revision = self.policy.revision if self.policy is not None else 0
if self.status_tracker is not None:
self.status_tracker.mark_host_policy(self.policy)
async def sync_once(self) -> HeartbeatResponse:
snapshot = build_device_snapshot(self.manager)
@@ -58,6 +65,16 @@ class HeartbeatSynchronizer:
self.policy_revision = response.policy_revision
if response.policy is not None:
self.policy = response.policy
if self.policy_cache is not None:
self.policy_cache.save(response.policy)
if self.on_policy_sync is not None:
self.on_policy_sync(response.policy.revision)
elif response.policy_revision == 0:
self.policy = None
if self.policy_cache is not None:
self.policy_cache.clear()
if self.status_tracker is not None:
self.status_tracker.mark_host_policy(self.policy)
if self.status_tracker is not None:
self.status_tracker.mark_heartbeat(ok=True, device_count=len(snapshot))
if self.on_sync is not None:
@@ -48,6 +48,13 @@ class ConsoleHistoryStore:
detail = {"device_count": device_count}
self._insert("heartbeat", summary, detail)
def record_policy_sync(self, *, revision: int) -> None:
self._insert(
"host_policy",
f"host policy synchronized: revision {revision}",
{"revision": revision},
)
def list_recent(self, limit: int | None = None) -> list[dict[str, Any]]:
effective_limit = limit if limit is not None else self.limit
with self._connect() as connection:
@@ -0,0 +1,47 @@
from __future__ import annotations
import json
import os
from pathlib import Path
from uuid import uuid4
from cloud.internal_api.models import HostGovernancePolicyModel
class HostPolicyCacheError(RuntimeError):
"""Raised when the locally cached non-secret Host policy is invalid."""
class HostPolicyCacheStore:
"""Atomically persists only the Cloud-supplied, non-secret policy cache."""
def __init__(self, path: str | Path) -> None:
self.path = Path(path)
def load(self) -> HostGovernancePolicyModel | None:
if not self.path.exists():
return None
try:
payload = json.loads(self.path.read_text(encoding="utf-8"))
return HostGovernancePolicyModel.model_validate(payload)
except (OSError, ValueError, json.JSONDecodeError) as exc:
raise HostPolicyCacheError("Host policy cache is invalid") from exc
def save(self, policy: HostGovernancePolicyModel) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
temporary = self.path.with_name(f".{self.path.name}.{uuid4().hex}.tmp")
try:
temporary.write_text(
policy.model_dump_json(indent=2) + "\n",
encoding="utf-8",
)
os.replace(temporary, self.path)
finally:
if temporary.exists():
temporary.unlink()
def clear(self) -> None:
try:
self.path.unlink()
except FileNotFoundError:
return
@@ -33,6 +33,7 @@ class AgentStatusTracker:
self._lock = threading.Lock()
self._current_assignment: _CurrentAssignment | None = None
self._last_heartbeat: _LastHeartbeat | None = None
self._host_policy: dict[str, Any] | None = None
def mark_assignment_started(self, assignment: AssignmentModel) -> None:
with self._lock:
@@ -56,6 +57,19 @@ class AgentStatusTracker:
at=self._now(),
)
def mark_host_policy(self, policy: Any | None) -> None:
with self._lock:
self._host_policy = (
{
"revision": policy.revision,
"self_submission_enabled": policy.self_submission_enabled,
"max_active_tasks": policy.max_active_tasks,
"daily_token_budget": policy.daily_token_budget,
}
if policy is not None
else None
)
def snapshot(self) -> dict[str, Any]:
with self._lock:
current_assignment = self._current_assignment
@@ -81,4 +95,5 @@ class AgentStatusTracker:
if last_heartbeat is not None
else None
),
"host_policy": self._host_policy.copy() if self._host_policy else None,
}
@@ -117,6 +117,7 @@ def _dashboard_body(
) -> str:
heartbeat = snapshot.get("last_heartbeat")
assignment = snapshot.get("current_assignment")
policy = snapshot.get("host_policy")
heartbeat_text = (
f"{'ok' if heartbeat['ok'] else 'failed'} at {heartbeat['at']} "
f"({heartbeat['device_count']} devices)"
@@ -129,6 +130,17 @@ def _dashboard_body(
if assignment
else "none"
)
policy_text = (
"revision {revision}; self-submission {self_submission}; "
"max active tasks {max_active}; daily token budget {daily_budget}".format(
revision=policy["revision"],
self_submission="enabled" if policy["self_submission_enabled"] else "disabled",
max_active=policy["max_active_tasks"] or "unlimited",
daily_budget=policy["daily_token_budget"] or "unmetered",
)
if policy
else "no Cloud policy cached"
)
device_rows = "".join(
f"<tr><td>{escape(device.id)}</td><td>{escape(device.name or '')}</td>"
f"<td>{escape(device.driver_type)}</td><td>{escape(device.status)}</td></tr>"
@@ -146,6 +158,10 @@ def _dashboard_body(
<h2>Heartbeat</h2>
<p id="last-heartbeat">{escape(heartbeat_text)}</p>
</section>
<section>
<h2>Cloud policy</h2>
<p id="host-policy">{escape(policy_text)}</p>
</section>
<section>
<h2>Current assignment</h2>
<p id="current-assignment">{escape(assignment_text)}</p>
@@ -168,6 +184,13 @@ def _dashboard_body(
document.getElementById("current-assignment").textContent = current
? current.task_id + " on " + current.device_id + " (started " + current.started_at + ")"
: "none";
var policy = data.status.host_policy;
document.getElementById("host-policy").textContent = policy
? "revision " + policy.revision + "; self-submission "
+ (policy.self_submission_enabled ? "enabled" : "disabled")
+ "; max active tasks " + (policy.max_active_tasks || "unlimited")
+ "; daily token budget " + (policy.daily_token_budget || "unmetered")
: "no Cloud policy cached";
var body = document.getElementById("device-status-body");
body.innerHTML = "";
data.devices.forEach(function (device) {{
@@ -4,9 +4,11 @@ import asyncio
from datetime import UTC, datetime
from cloud.internal_api.models import HeartbeatResponse
from cloud.internal_api.models import HostGovernancePolicyModel
from device.manager import DeviceManager
from host_agent.config import HostAgentConfig
from host_agent.heartbeat import HeartbeatSynchronizer, build_device_snapshot
from host_agent.policy_cache import HostPolicyCacheStore
from host_agent.status import AgentStatusTracker
@@ -123,3 +125,56 @@ def test_sync_once_notifies_status_tracker_and_on_sync_with_device_count() -> No
assert last_heartbeat["device_count"] == 2
asyncio.run(scenario())
def test_heartbeat_caches_safe_host_policy_and_reuses_its_revision(tmp_path) -> None:
manager = DeviceManager()
cache = HostPolicyCacheStore(tmp_path / "host_policy.json")
revisions: list[int] = []
class UpdatingClient:
async def heartbeat(self, devices, *, address=None, policy_revision=0):
revisions.append(policy_revision)
return HeartbeatResponse(
host_id="host-a",
accepted_devices=len(devices),
received_at=datetime.now(UTC),
policy_revision=4,
policy=HostGovernancePolicyModel(
revision=4,
self_submission_enabled=False,
max_active_tasks=2,
daily_token_budget=900,
),
)
async def scenario() -> None:
tracker = AgentStatusTracker()
synchronizer = HeartbeatSynchronizer(
manager,
UpdatingClient(), # type: ignore[arg-type]
_config(),
policy_cache=cache,
status_tracker=tracker,
)
await synchronizer.sync_once()
assert tracker.snapshot()["host_policy"] == {
"revision": 4,
"self_submission_enabled": False,
"max_active_tasks": 2,
"daily_token_budget": 900,
}
restarted = HeartbeatSynchronizer(
manager,
UpdatingClient(), # type: ignore[arg-type]
_config(),
policy_cache=cache,
)
assert restarted.policy_revision == 4
asyncio.run(scenario())
assert revisions == [0]
assert '"token":' not in (
tmp_path / "host_policy.json"
).read_text(encoding="utf-8")
+23 -1
View File
@@ -7,6 +7,7 @@ import {
LogOut,
MonitorSmartphone,
Puzzle,
UsersRound,
} from "@lucide/vue";
import {
AUTH_INVALID_EVENT,
@@ -19,8 +20,9 @@ import PasswordChangeScreen from "./views/PasswordChangeScreen.vue";
import TasksView from "./views/TasksView.vue";
import DevicesView from "./views/DevicesView.vue";
import PluginsView from "./views/PluginsView.vue";
import UsersView from "./views/UsersView.vue";
type ViewId = "tasks" | "devices" | "plugins";
type ViewId = "tasks" | "devices" | "plugins" | "users";
const activeView = ref<ViewId>("tasks");
const currentUser = ref<CloudUser | null>(null);
@@ -37,6 +39,16 @@ const canSubmitTasks = computed(
currentUser.value?.scopes.includes("*") ||
currentUser.value?.scopes.includes("tasks:submit"),
);
const canAdminUsers = computed(
() => Boolean(
currentUser.value?.scopes.includes("*") ||
currentUser.value?.scopes.includes("users:admin")),
);
const canAdminGovernance = computed(
() => Boolean(
currentUser.value?.scopes.includes("*") ||
currentUser.value?.scopes.includes("governance:admin")),
);
const isAuthenticated = computed(() => currentUser.value !== null);
const currentUserLabel = computed(() =>
currentUser.value ? `${currentUser.value.display_name} (${currentUser.value.role})` : "",
@@ -48,6 +60,9 @@ const navItems = computed<{ id: ViewId; label: string; icon: Component }[]>(() =
{ id: "devices", label: "Devices", icon: MonitorSmartphone },
{ id: "plugins", label: "Plugins", icon: Puzzle },
];
if (canAdminUsers.value || canAdminGovernance.value) {
items.push({ id: "users", label: "Users & limits", icon: UsersRound });
}
return items;
});
@@ -102,6 +117,8 @@ const activeComponent = computed(() => {
return DevicesView;
case "plugins":
return PluginsView;
case "users":
return UsersView;
default:
return TasksView;
}
@@ -133,6 +150,11 @@ const activeComponent = computed(() => {
</nav>
<main class="app-main">
<PluginsView v-if="activeView === 'plugins'" :can-admin="canAdminPlugins" />
<UsersView
v-else-if="activeView === 'users'"
:can-admin-users="canAdminUsers"
:can-admin-governance="canAdminGovernance"
/>
<component v-else :is="activeComponent" :can-submit="canSubmitTasks" />
</main>
</div>
+85
View File
@@ -1,6 +1,8 @@
import type {
CloudUser,
DeviceRecord,
HostGovernancePolicy,
HostTokenUsageSummary,
HostRecord,
PluginRecord,
PluginRegistrationPayload,
@@ -8,6 +10,8 @@ import type {
TaskListResponse,
TaskSubmissionPayload,
TaskStatus,
UserListResponse,
UserSubmissionPolicy,
} from "./types";
const configuredBaseUrl = import.meta.env.VITE_CLOUD_API_BASE_URL as
@@ -119,6 +123,87 @@ export function changePassword(
});
}
export function listUsers(): Promise<UserListResponse> {
return request<UserListResponse>("/v1/users?limit=100&offset=0");
}
export function createUser(payload: {
username: string;
display_name: string;
role: "viewer" | "operator" | "admin";
password: string;
}): Promise<CloudUser> {
return request<CloudUser>("/v1/users", {
method: "POST",
body: JSON.stringify(payload),
});
}
export function updateUser(
userId: string,
payload: { display_name?: string; role?: "viewer" | "operator" | "admin"; enabled?: boolean },
): Promise<CloudUser> {
return request<CloudUser>(`/v1/users/${encodeURIComponent(userId)}`, {
method: "PATCH",
body: JSON.stringify(payload),
});
}
export function resetUserPassword(userId: string, password: string): Promise<CloudUser> {
return request<CloudUser>(`/v1/users/${encodeURIComponent(userId)}/password`, {
method: "POST",
body: JSON.stringify({ password }),
});
}
export function revokeUserSessions(userId: string): Promise<void> {
return request<void>(`/v1/users/${encodeURIComponent(userId)}/sessions`, {
method: "DELETE",
});
}
export function getUserSubmissionPolicy(userId: string): Promise<UserSubmissionPolicy> {
return request<UserSubmissionPolicy>(
`/v1/users/${encodeURIComponent(userId)}/submission-policy`,
);
}
export function updateUserSubmissionPolicy(
userId: string,
payload: Omit<UserSubmissionPolicy, "user_id" | "revision" | "updated_at"> & {
expected_revision?: number;
},
): Promise<UserSubmissionPolicy> {
return request<UserSubmissionPolicy>(
`/v1/users/${encodeURIComponent(userId)}/submission-policy`,
{ method: "PUT", body: JSON.stringify(payload) },
);
}
export function getHostGovernancePolicy(hostId: string): Promise<HostGovernancePolicy> {
return request<HostGovernancePolicy>(
`/v1/hosts/${encodeURIComponent(hostId)}/governance-policy`,
);
}
export function updateHostGovernancePolicy(
hostId: string,
payload: Omit<HostGovernancePolicy, "host_id" | "revision" | "updated_at"> & {
expected_revision?: number;
},
): Promise<HostGovernancePolicy> {
return request<HostGovernancePolicy>(
`/v1/hosts/${encodeURIComponent(hostId)}/governance-policy`,
{ method: "PUT", body: JSON.stringify(payload) },
);
}
export function getHostTokenUsage(hostId: string): Promise<HostTokenUsageSummary> {
return request<HostTokenUsageSummary>(
`/v1/hosts/${encodeURIComponent(hostId)}/token-usage`,
);
}
export function listTasks(options?: {
status?: TaskStatus;
limit?: number;
+13
View File
@@ -319,6 +319,19 @@ tr.row-selected {
grid-column: 1 / -1;
}
.form-grid input:not([type="checkbox"]),
.form-grid select {
width: 100%;
}
.form-grid .actions {
align-self: end;
}
.policy-grid select {
min-height: 92px;
}
.muted {
color: var(--text-muted);
}
+38
View File
@@ -96,3 +96,41 @@ export interface CloudUser {
updated_at: string;
last_login_at: string | null;
}
export interface UserListResponse {
items: CloudUser[];
limit: number;
offset: number;
}
export interface DeviceTarget {
host_id: string;
device_id: string;
}
export interface UserSubmissionPolicy {
user_id: string;
revision: number;
submission_enabled: boolean;
allowed_host_ids: string[] | null;
allowed_device_targets: DeviceTarget[] | null;
updated_at: string;
}
export interface HostGovernancePolicy {
host_id: string;
revision: number;
self_submission_enabled: boolean;
max_active_tasks: number | null;
daily_token_budget: number | null;
updated_at: string;
}
export interface HostTokenUsageSummary {
host_id: string;
usage_day: string;
daily_token_budget: number | null;
used_tokens: number;
reserved_tokens: number;
remaining_tokens: number | null;
}
+380
View File
@@ -0,0 +1,380 @@
<script setup lang="ts">
import { computed, onMounted, ref, watch } from "vue";
import {
CloudApiError,
createUser,
getHostGovernancePolicy,
getHostTokenUsage,
getUserSubmissionPolicy,
listDevices,
listHosts,
listUsers,
resetUserPassword,
revokeUserSessions,
updateUser,
updateUserSubmissionPolicy,
updateHostGovernancePolicy,
} from "../api";
import type {
CloudUser,
DeviceRecord,
HostRecord,
HostTokenUsageSummary,
UserRole,
} from "../types";
const props = defineProps<{ canAdminUsers: boolean; canAdminGovernance: boolean }>();
const users = ref<CloudUser[]>([]);
const hosts = ref<HostRecord[]>([]);
const devices = ref<DeviceRecord[]>([]);
const selectedUserId = ref("");
const loading = ref(false);
const saving = ref(false);
const errorMessage = ref("");
const successMessage = ref("");
const createUsername = ref("");
const createDisplayName = ref("");
const createRole = ref<UserRole>("operator");
const createPassword = ref("");
const selectedRole = ref<UserRole>("viewer");
const selectedEnabled = ref(true);
const resetPassword = ref("");
const submissionEnabled = ref(true);
const restrictHosts = ref(false);
const allowedHostIds = ref<string[]>([]);
const restrictDevices = ref(false);
const allowedDeviceKeys = ref<string[]>([]);
const policyRevision = ref<number | null>(null);
const selectedHostId = ref("");
const hostPolicyRevision = ref<number | null>(null);
const hostSelfSubmissionEnabled = ref(true);
const hostMaxActiveTasks = ref("");
const hostDailyTokenBudget = ref("");
const hostUsage = ref<HostTokenUsageSummary | null>(null);
const selectedUser = computed(
() => users.value.find((user) => user.id === selectedUserId.value) ?? null,
);
const deviceKey = (device: Pick<DeviceRecord, "host_id" | "device_id">) =>
`${device.host_id}\u0000${device.device_id}`;
function showError(error: unknown, fallback: string) {
successMessage.value = "";
errorMessage.value = error instanceof Error ? error.message : fallback;
}
function selectUser(user: CloudUser) {
selectedUserId.value = user.id;
selectedRole.value = user.role;
selectedEnabled.value = user.enabled;
}
async function refresh() {
loading.value = true;
errorMessage.value = "";
try {
const requests: Promise<unknown>[] = [];
if (props.canAdminUsers) requests.push(listUsers());
if (props.canAdminGovernance) requests.push(listHosts(), listDevices());
const results = await Promise.all(requests);
let index = 0;
if (props.canAdminUsers) {
users.value = (results[index++] as { items: CloudUser[] }).items;
if (!selectedUserId.value && users.value[0]) selectUser(users.value[0]);
}
if (props.canAdminGovernance) {
hosts.value = results[index++] as HostRecord[];
devices.value = results[index++] as DeviceRecord[];
if (!selectedHostId.value && hosts.value[0]) {
selectedHostId.value = hosts.value[0].host_id;
}
}
} catch (error) {
showError(error, "failed to load user administration data");
} finally {
loading.value = false;
}
}
async function loadHostPolicy() {
if (!props.canAdminGovernance || !selectedHostId.value) return;
errorMessage.value = "";
try {
const [policy, usage] = await Promise.all([
getHostGovernancePolicy(selectedHostId.value),
getHostTokenUsage(selectedHostId.value),
]);
hostUsage.value = usage;
hostPolicyRevision.value = policy.revision;
hostSelfSubmissionEnabled.value = policy.self_submission_enabled;
hostMaxActiveTasks.value = policy.max_active_tasks?.toString() ?? "";
hostDailyTokenBudget.value = policy.daily_token_budget?.toString() ?? "";
} catch (error) {
if (error instanceof CloudApiError && error.status === 404) {
hostPolicyRevision.value = null;
hostSelfSubmissionEnabled.value = true;
hostMaxActiveTasks.value = "";
hostDailyTokenBudget.value = "";
hostUsage.value = await getHostTokenUsage(selectedHostId.value);
return;
}
showError(error, "failed to load Host policy");
}
}
async function loadPolicy() {
if (!props.canAdminGovernance || !selectedUserId.value) return;
errorMessage.value = "";
try {
const policy = await getUserSubmissionPolicy(selectedUserId.value);
submissionEnabled.value = policy.submission_enabled;
restrictHosts.value = policy.allowed_host_ids !== null;
allowedHostIds.value = policy.allowed_host_ids ?? [];
restrictDevices.value = policy.allowed_device_targets !== null;
allowedDeviceKeys.value = (policy.allowed_device_targets ?? []).map(deviceKey);
policyRevision.value = policy.revision;
} catch (error) {
if (error instanceof CloudApiError && error.status === 404) {
submissionEnabled.value = true;
restrictHosts.value = false;
allowedHostIds.value = [];
restrictDevices.value = false;
allowedDeviceKeys.value = [];
policyRevision.value = null;
return;
}
showError(error, "failed to load submission policy");
}
}
async function createAccount() {
saving.value = true;
errorMessage.value = "";
try {
const user = await createUser({
username: createUsername.value.trim(),
display_name: createDisplayName.value.trim(),
role: createRole.value,
password: createPassword.value,
});
createUsername.value = "";
createDisplayName.value = "";
successMessage.value = `created ${user.username}`;
await refresh();
selectUser(user);
} catch (error) {
showError(error, "failed to create user");
} finally {
createPassword.value = "";
saving.value = false;
}
}
async function saveUser() {
if (!selectedUser.value) return;
saving.value = true;
try {
await updateUser(selectedUser.value.id, {
role: selectedRole.value,
enabled: selectedEnabled.value,
});
successMessage.value = "user updated";
await refresh();
} catch (error) {
showError(error, "failed to update user");
} finally {
saving.value = false;
}
}
async function resetPasswordForUser() {
if (!selectedUser.value || !resetPassword.value) return;
saving.value = true;
try {
await resetUserPassword(selectedUser.value.id, resetPassword.value);
successMessage.value = "password reset; existing sessions were revoked";
} catch (error) {
showError(error, "failed to reset password");
} finally {
resetPassword.value = "";
saving.value = false;
}
}
async function revokeSessions() {
if (!selectedUser.value) return;
saving.value = true;
try {
await revokeUserSessions(selectedUser.value.id);
successMessage.value = "sessions revoked";
} catch (error) {
showError(error, "failed to revoke sessions");
} finally {
saving.value = false;
}
}
async function savePolicy() {
if (!selectedUserId.value) {
errorMessage.value = "select or enter a user id before saving a policy";
return;
}
saving.value = true;
try {
const selectedDevices = new Set(allowedDeviceKeys.value);
const policy = await updateUserSubmissionPolicy(selectedUserId.value, {
submission_enabled: submissionEnabled.value,
expected_revision: policyRevision.value ?? 0,
allowed_host_ids: restrictHosts.value ? allowedHostIds.value : null,
allowed_device_targets: restrictDevices.value
? devices.value
.filter((device) => selectedDevices.has(deviceKey(device)))
.map((device) => ({ host_id: device.host_id, device_id: device.device_id }))
: null,
});
policyRevision.value = policy.revision;
successMessage.value = `submission policy saved (revision ${policy.revision})`;
} catch (error) {
showError(error, "failed to save submission policy");
} finally {
saving.value = false;
}
}
function parseOptionalPositive(value: string, label: string): number | null {
if (!value.trim()) return null;
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer`);
}
return parsed;
}
async function saveHostPolicy() {
if (!selectedHostId.value) return;
saving.value = true;
try {
const policy = await updateHostGovernancePolicy(selectedHostId.value, {
self_submission_enabled: hostSelfSubmissionEnabled.value,
max_active_tasks: parseOptionalPositive(hostMaxActiveTasks.value, "max active tasks"),
daily_token_budget: parseOptionalPositive(hostDailyTokenBudget.value, "daily token budget"),
expected_revision: hostPolicyRevision.value ?? 0,
});
hostPolicyRevision.value = policy.revision;
hostUsage.value = await getHostTokenUsage(selectedHostId.value);
successMessage.value = `Host policy saved (revision ${policy.revision})`;
} catch (error) {
showError(error, "failed to save Host policy");
} finally {
saving.value = false;
}
}
watch(selectedUserId, () => {
if (selectedUser.value) {
selectedRole.value = selectedUser.value.role;
selectedEnabled.value = selectedUser.value.enabled;
}
void loadPolicy();
});
watch(selectedHostId, () => {
void loadHostPolicy();
});
onMounted(() => void refresh());
</script>
<template>
<div>
<div class="toolbar">
<h2>Users & task limits</h2>
<button :disabled="loading" @click="refresh">Refresh</button>
<span v-if="loading" class="muted">loading</span>
</div>
<div v-if="errorMessage" class="notice error">{{ errorMessage }}</div>
<div v-if="successMessage" class="notice success">{{ successMessage }}</div>
<div v-if="canAdminUsers" class="panel">
<h3>Create user</h3>
<form class="form-grid" @submit.prevent="createAccount">
<label>Username <input v-model="createUsername" required /></label>
<label>Display name <input v-model="createDisplayName" required /></label>
<label>Role <select v-model="createRole"><option value="viewer">Viewer</option><option value="operator">Operator</option><option value="admin">Admin</option></select></label>
<label>Password <input v-model="createPassword" type="password" required /></label>
<div class="field-full actions"><button class="primary" :disabled="saving">Create user</button></div>
</form>
</div>
<div v-if="canAdminUsers" class="panel">
<h3>Accounts</h3>
<table v-if="users.length">
<thead><tr><th>Username</th><th>Role</th><th>Status</th><th></th></tr></thead>
<tbody>
<tr v-for="user in users" :key="user.id" :class="{ 'row-selected': user.id === selectedUserId }">
<td>{{ user.display_name }} <span class="dim">({{ user.username }})</span></td>
<td>{{ user.role }}</td>
<td>{{ user.enabled ? "enabled" : "disabled" }}</td>
<td><button @click="selectUser(user)">Manage</button></td>
</tr>
</tbody>
</table>
<p v-else class="muted">No accounts found.</p>
</div>
<div v-if="canAdminGovernance && !canAdminUsers" class="panel">
<label>User ID <input v-model="selectedUserId" placeholder="Cloud user id" /></label>
<p class="muted">Use an existing user id to manage its task-submission policy.</p>
</div>
<div v-if="selectedUser && canAdminUsers" class="panel">
<h3>Account: {{ selectedUser.username }}</h3>
<div class="form-grid">
<label>Role <select v-model="selectedRole"><option value="viewer">Viewer</option><option value="operator">Operator</option><option value="admin">Admin</option></select></label>
<label><input v-model="selectedEnabled" type="checkbox" /> Enabled</label>
<div class="field-full actions"><button :disabled="saving" @click="saveUser">Save account</button></div>
<label>Password reset <input v-model="resetPassword" type="password" placeholder="New temporary password" /></label>
<div class="actions"><button :disabled="saving || !resetPassword" @click="resetPasswordForUser">Reset password</button><button :disabled="saving" @click="revokeSessions">Revoke sessions</button></div>
</div>
</div>
<div v-if="canAdminGovernance && selectedUserId" class="panel">
<h3>Task-submission policy <span class="dim">{{ policyRevision === null ? "not configured" : `revision ${policyRevision}` }}</span></h3>
<p class="muted">An unrestricted policy allows ordinary scope-authorized submissions. Enabling a restriction requires an explicit target.</p>
<label><input v-model="submissionEnabled" type="checkbox" /> Allow this user to submit tasks</label>
<div class="form-grid policy-grid">
<label><input v-model="restrictHosts" type="checkbox" /> Restrict Hosts</label>
<select v-model="allowedHostIds" multiple :disabled="!restrictHosts">
<option v-for="host in hosts" :key="host.host_id" :value="host.host_id">{{ host.host_id }}</option>
</select>
<label><input v-model="restrictDevices" type="checkbox" /> Restrict Devices</label>
<select v-model="allowedDeviceKeys" multiple :disabled="!restrictDevices">
<option v-for="device in devices" :key="deviceKey(device)" :value="deviceKey(device)">{{ device.host_id }} / {{ device.device_id }}</option>
</select>
</div>
<div class="actions"><button class="primary" :disabled="saving" @click="savePolicy">Save policy</button></div>
</div>
<div v-if="canAdminGovernance" class="panel">
<h3>Host operational limits</h3>
<p class="muted">Daily token budgets apply when this Host uses Cloud planner transport. Each decision reserves the Cloud-configured conservative ceiling before calling a provider, so a usable budget must cover that ceiling. Direct provider transport remains unmetered until it is moved behind the proxy.</p>
<div class="form-grid">
<label>Host
<select v-model="selectedHostId">
<option v-for="host in hosts" :key="host.host_id" :value="host.host_id">{{ host.host_id }}</option>
</select>
</label>
<label><input v-model="hostSelfSubmissionEnabled" type="checkbox" /> Allow edge self-submission</label>
<label>Maximum active tasks <input v-model="hostMaxActiveTasks" inputmode="numeric" placeholder="Unlimited" /></label>
<label>Daily Cloud-proxy token budget <input v-model="hostDailyTokenBudget" inputmode="numeric" placeholder="Unlimited" /></label>
</div>
<div class="actions"><button class="primary" :disabled="saving || !selectedHostId" @click="saveHostPolicy">Save Host policy {{ hostPolicyRevision === null ? "" : `(revision ${hostPolicyRevision})` }}</button></div>
<p v-if="hostUsage" class="muted">
{{ hostUsage.usage_day }}: used {{ hostUsage.used_tokens }}, reserved {{ hostUsage.reserved_tokens }},
remaining {{ hostUsage.remaining_tokens ?? "unmetered" }} tokens.
</p>
</div>
</div>
</template>
@@ -51,13 +51,13 @@
- [x] 4.1 Extend shared internal heartbeat request/response models with the
Host's last policy revision and a revision-aware effective-policy reply.
- [ ] 4.2 Update Host heartbeat synchronization to persist only safe cached
- [x] 4.2 Update Host heartbeat synchronization to persist only safe cached
policy state and expose it through local status/history without adding an
inbound Cloud connection.
- [x] 4.3 Add the Host-scoped goal-only task-submission route and
`HostAgentClient` method; derive Host targeting from authenticated
credentials and validate any named local Device ownership.
- [ ] 4.4 Enforce the Host policy's self-submission and active-task limits in
- [x] 4.4 Enforce the Host policy's self-submission and active-task limits in
the Cloud service/scheduler, not only in Host-local code.
- [ ] 4.5 Add Cloud API and Host Agent tests for revision convergence,
unchanged-policy replies, self-targeted task creation, foreign target
@@ -71,7 +71,7 @@
- [ ] 5.2 Extend planner-proxy request context and Host-Agent-local context
binding so Cloud-proxied calls carry known task/attempt metadata without
importing Host or Cloud concerns into `runtime`.
- [ ] 5.3 Add Cloud proxy preflight reservation, configured conservative
- [x] 5.3 Add Cloud proxy preflight reservation, configured conservative
per-call ceiling, provider invocation, actual-usage settlement, and
bounded unknown-usage reservation expiry.
- [ ] 5.4 Record non-secret usage events and expose accurate
@@ -87,7 +87,7 @@
- [x] 6.1 Add task composer API bindings and a scope-aware Console form for
goal/workflow submission, Host/Device target selection, and target/policy
validation errors.
- [ ] 6.2 Complete or reconcile the admin Users view, then add user-
- [x] 6.2 Complete or reconcile the admin Users view, then add user-
submission-policy editing with safe refresh and conflict/error handling.
- [ ] 6.3 Add Host policy administration and AI-usage/budget views, including
revision display, unmetered direct Hosts, and no rendering of prompts,
@@ -36,6 +36,8 @@ class CloudControlConfig:
login_block_seconds: int = 900
session_cookie_secure: bool = False
trust_proxy_headers: bool = False
planner_token_reservation_ceiling: int = 4096
planner_token_reservation_ttl_seconds: int = 300
def load_control_config(
@@ -115,6 +117,16 @@ def load_control_config(
values.get("CLOUD_TRUST_PROXY_HEADERS"),
default=False,
),
planner_token_reservation_ceiling=_positive_int(
values,
"CLOUD_PLANNER_TOKEN_RESERVATION_CEILING",
4096,
),
planner_token_reservation_ttl_seconds=_positive_int(
values,
"CLOUD_PLANNER_TOKEN_RESERVATION_TTL_SECONDS",
300,
),
)
validate_control_config(config)
return config
@@ -241,3 +241,44 @@ class HostGovernancePolicyRow(Base):
max_active_tasks: Mapped[int | None] = mapped_column(Integer, nullable=True)
daily_token_budget: Mapped[int | None] = mapped_column(Integer, nullable=True)
updated_at: Mapped[str] = mapped_column(String, nullable=False)
class TokenReservationRow(Base):
__tablename__ = "cloud_token_reservations"
__table_args__ = (
Index("ix_cloud_token_reservations_host_day", "host_id", "usage_day"),
Index("ix_cloud_token_reservations_expires_at", "expires_at"),
)
id: Mapped[str] = mapped_column(String, primary_key=True)
host_id: Mapped[str] = mapped_column(
ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False
)
usage_day: Mapped[str] = mapped_column(String, nullable=False)
reserved_tokens: Mapped[int] = mapped_column(Integer, nullable=False)
task_id: Mapped[str | None] = mapped_column(String, nullable=True)
attempt: Mapped[int | None] = mapped_column(Integer, nullable=True)
created_at: Mapped[str] = mapped_column(String, nullable=False)
expires_at: Mapped[str] = mapped_column(String, nullable=False)
class TokenUsageEventRow(Base):
__tablename__ = "cloud_token_usage_events"
__table_args__ = (
Index("ix_cloud_token_usage_events_host_day", "host_id", "usage_day"),
Index("ix_cloud_token_usage_events_occurred_at", "occurred_at"),
)
id: Mapped[str] = mapped_column(String, primary_key=True)
host_id: Mapped[str] = mapped_column(
ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False
)
usage_day: Mapped[str] = mapped_column(String, nullable=False)
task_id: Mapped[str | None] = mapped_column(String, nullable=True)
attempt: Mapped[int | None] = mapped_column(Integer, nullable=True)
provider: Mapped[str] = mapped_column(String, nullable=False)
model: Mapped[str] = mapped_column(String, nullable=False)
input_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
output_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
total_tokens: Mapped[int] = mapped_column(Integer, nullable=False)
occurred_at: Mapped[str] = mapped_column(String, nullable=False)
@@ -10,6 +10,14 @@ class TaskSubmissionPolicyError(PermissionError):
"""Raised when a human user's policy disallows a task submission."""
class GovernancePolicyConflictError(RuntimeError):
"""Raised when a policy write was based on an obsolete revision."""
class TokenBudgetExceededError(RuntimeError):
"""Raised before a Cloud-proxied call would exceed a Host's token budget."""
@dataclass(frozen=True)
class UserSubmissionPolicy:
user_id: str
@@ -30,6 +38,48 @@ class HostGovernancePolicy:
updated_at: datetime
@dataclass(frozen=True)
class TokenReservation:
id: str
host_id: str
usage_day: str
reserved_tokens: int
task_id: str | None
attempt: int | None
created_at: datetime
expires_at: datetime
@dataclass(frozen=True)
class TokenUsageEvent:
id: str
host_id: str
usage_day: str
task_id: str | None
attempt: int | None
provider: str
model: str
input_tokens: int | None
output_tokens: int | None
total_tokens: int
occurred_at: datetime
@dataclass(frozen=True)
class TokenUsageSummary:
host_id: str
usage_day: str
daily_token_budget: int | None
used_tokens: int
reserved_tokens: int
@property
def remaining_tokens(self) -> int | None:
if self.daily_token_budget is None:
return None
return max(0, self.daily_token_budget - self.used_tokens - self.reserved_tokens)
def enforce_user_submission_policy(
policy: UserSubmissionPolicy | None,
*,
@@ -44,6 +44,7 @@ from cloud.repository import (
DeviceEnrollmentConflictError,
HostEnrollmentConflictError,
)
from cloud.governance import TokenBudgetExceededError
from core.models import Device, utc_now
from runtime.tool_calling_client import ToolCallingClient, ToolCallUnavailable
from runtime.tool_specs import ToolSpec
@@ -65,11 +66,17 @@ def create_internal_router(
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
planner_client_factory: Callable[[], ToolCallingClient] | None = None,
scheduler: TaskScheduler | None = None,
planner_token_reservation_ceiling: int = 4096,
planner_token_reservation_ttl_seconds: float = 300.0,
) -> APIRouter:
if claim_poll_interval_seconds <= 0:
raise ValueError("claim_poll_interval_seconds must be greater than zero")
if lease_duration_seconds <= 0:
raise ValueError("lease_duration_seconds must be greater than zero")
if planner_token_reservation_ceiling <= 0:
raise ValueError("planner_token_reservation_ceiling must be greater than zero")
if planner_token_reservation_ttl_seconds <= 0:
raise ValueError("planner_token_reservation_ttl_seconds must be greater than zero")
router = APIRouter(prefix=version_prefix, tags=["host-agent"])
build_planner_client = planner_client_factory or _default_planner_client_factory
@@ -352,6 +359,7 @@ def create_internal_router(
response_model_exclude_none=True,
responses={
status.HTTP_502_BAD_GATEWAY: {"model": PlannerDecisionError},
status.HTTP_429_TOO_MANY_REQUESTS: {"model": PlannerDecisionError},
},
)
def decide_planner_call(
@@ -385,6 +393,25 @@ def create_internal_router(
for tool in payload.tools
]
now = utc_now()
reservation = None
try:
reservation = pool.store.reserve_host_token_budget(
reservation_id=uuid4().hex,
host_id=host_id,
usage_day=now.date().isoformat(),
reserved_tokens=planner_token_reservation_ceiling,
task_id=None,
attempt=None,
created_at=now,
expires_at=now + timedelta(seconds=planner_token_reservation_ttl_seconds),
)
except TokenBudgetExceededError as exc:
return JSONResponse(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
content=PlannerDecisionError(detail=str(exc)).model_dump(),
)
started_at = monotonic()
client = build_planner_client()
try:
@@ -408,6 +435,19 @@ def create_internal_router(
status_code=status.HTTP_502_BAD_GATEWAY,
content=PlannerDecisionError(detail=str(exc)).model_dump(),
)
usage = decision.usage
if reservation is not None and usage is not None and usage.total_tokens is not None:
planner_config = load_cloud_planner_config()
pool.store.settle_host_token_reservation(
reservation_id=reservation.id,
event_id=uuid4().hex,
provider=planner_config.provider,
model=planner_config.resolved_model(),
input_tokens=usage.input_tokens,
output_tokens=usage.output_tokens,
total_tokens=usage.total_tokens,
occurred_at=utc_now(),
)
logger.info(
"planner-decision request resolved",
extra={
@@ -0,0 +1,53 @@
"""Add Cloud-proxy token reservations and usage events."""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "0005_cloud_token_usage"
down_revision = "0004_cloud_governance"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"cloud_token_reservations",
sa.Column("id", sa.String(), primary_key=True),
sa.Column("host_id", sa.String(), sa.ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False),
sa.Column("usage_day", sa.String(), nullable=False),
sa.Column("reserved_tokens", sa.Integer(), nullable=False),
sa.Column("task_id", sa.String(), nullable=True),
sa.Column("attempt", sa.Integer(), nullable=True),
sa.Column("created_at", sa.String(), nullable=False),
sa.Column("expires_at", sa.String(), nullable=False),
)
op.create_index("ix_cloud_token_reservations_host_day", "cloud_token_reservations", ["host_id", "usage_day"])
op.create_index("ix_cloud_token_reservations_expires_at", "cloud_token_reservations", ["expires_at"])
op.create_table(
"cloud_token_usage_events",
sa.Column("id", sa.String(), primary_key=True),
sa.Column("host_id", sa.String(), sa.ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False),
sa.Column("usage_day", sa.String(), nullable=False),
sa.Column("task_id", sa.String(), nullable=True),
sa.Column("attempt", sa.Integer(), nullable=True),
sa.Column("provider", sa.String(), nullable=False),
sa.Column("model", sa.String(), nullable=False),
sa.Column("input_tokens", sa.Integer(), nullable=True),
sa.Column("output_tokens", sa.Integer(), nullable=True),
sa.Column("total_tokens", sa.Integer(), nullable=False),
sa.Column("occurred_at", sa.String(), nullable=False),
)
op.create_index("ix_cloud_token_usage_events_host_day", "cloud_token_usage_events", ["host_id", "usage_day"])
op.create_index("ix_cloud_token_usage_events_occurred_at", "cloud_token_usage_events", ["occurred_at"])
def downgrade() -> None:
op.drop_index("ix_cloud_token_usage_events_occurred_at", table_name="cloud_token_usage_events")
op.drop_index("ix_cloud_token_usage_events_host_day", table_name="cloud_token_usage_events")
op.drop_table("cloud_token_usage_events")
op.drop_index("ix_cloud_token_reservations_expires_at", table_name="cloud_token_reservations")
op.drop_index("ix_cloud_token_reservations_host_day", table_name="cloud_token_reservations")
op.drop_table("cloud_token_reservations")
+29 -1
View File
@@ -15,7 +15,13 @@ if TYPE_CHECKING:
UserAccount,
UserSession,
)
from cloud.governance import HostGovernancePolicy, UserSubmissionPolicy
from cloud.governance import (
HostGovernancePolicy,
TokenReservation,
TokenUsageSummary,
TokenUsageEvent,
UserSubmissionPolicy,
)
AttemptStatus = Literal["assigned", "dispatched", "done", "failed", "expired"]
@@ -293,6 +299,7 @@ class CloudRepository(Protocol):
submission_enabled: bool,
allowed_host_ids: tuple[str, ...] | None,
allowed_device_targets: tuple[tuple[str, str], ...] | None,
expected_revision: int | None,
updated_at: datetime,
) -> UserSubmissionPolicy: ...
@@ -305,9 +312,30 @@ class CloudRepository(Protocol):
self_submission_enabled: bool,
max_active_tasks: int | None,
daily_token_budget: int | None,
expected_revision: int | None,
updated_at: datetime,
) -> HostGovernancePolicy: ...
def count_active_tasks_for_host(self, host_id: str) -> int: ...
def reserve_host_token_budget(
self, *, reservation_id: str, host_id: str, usage_day: str,
reserved_tokens: int, task_id: str | None, attempt: int | None,
created_at: datetime, expires_at: datetime,
) -> TokenReservation | None: ...
def settle_host_token_reservation(
self, *, reservation_id: str, event_id: str, provider: str, model: str,
input_tokens: int | None, output_tokens: int | None, total_tokens: int,
occurred_at: datetime,
) -> TokenUsageEvent | None: ...
def cleanup_expired_token_reservations(self, *, now: datetime, limit: int) -> int: ...
def get_host_token_usage_summary(
self, *, host_id: str, usage_day: str, now: datetime,
) -> TokenUsageSummary: ...
def list_reserved_device_ids(self, *, now: datetime) -> set[str]: ...
def assign_task(
+1 -1
View File
@@ -9,7 +9,7 @@ from alembic.runtime.migration import MigrationContext
from cloud.database import create_database_engine, normalize_database_url
HEAD_REVISION = "0004_cloud_governance"
HEAD_REVISION = "0005_cloud_token_usage"
class SchemaVersionError(RuntimeError):
@@ -5,10 +5,12 @@ from uuid import uuid4
from fastapi import APIRouter, HTTPException, Request, status
from cloud.auth import AuthProvider, GOVERNANCE_ADMIN_SCOPE, GOVERNANCE_READ_SCOPE
from cloud.governance import GovernancePolicyConflictError
from cloud.observability import current_correlation_id
from cloud.sdk.models import (
HostGovernancePolicyRequest,
HostGovernancePolicyResponse,
HostTokenUsageSummaryResponse,
UserSubmissionPolicyRequest,
UserSubmissionPolicyResponse,
)
@@ -79,10 +81,13 @@ def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIR
if payload.allowed_device_targets is not None
else None
),
expected_revision=payload.expected_revision,
updated_at=utc_now(),
)
except KeyError as exc:
raise HTTPException(status_code=404, detail="user not found") from exc
except GovernancePolicyConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
_audit(repository, principal.id, user_id, "user_submission_policy_update")
return UserSubmissionPolicyResponse(
user_id=policy.user_id,
@@ -111,6 +116,27 @@ def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIR
raise HTTPException(status_code=404, detail="Host policy not found")
return _host_response(policy)
@router.get(
"/hosts/{host_id}/token-usage",
response_model=HostTokenUsageSummaryResponse,
)
def get_host_token_usage(
host_id: str, request: Request
) -> HostTokenUsageSummaryResponse:
authorize(request, GOVERNANCE_READ_SCOPE)
now = utc_now()
summary = repository.get_host_token_usage_summary(
host_id=host_id, usage_day=now.date().isoformat(), now=now
)
return HostTokenUsageSummaryResponse(
host_id=summary.host_id,
usage_day=summary.usage_day,
daily_token_budget=summary.daily_token_budget,
used_tokens=summary.used_tokens,
reserved_tokens=summary.reserved_tokens,
remaining_tokens=summary.remaining_tokens,
)
@router.put(
"/hosts/{host_id}/governance-policy",
response_model=HostGovernancePolicyResponse,
@@ -127,10 +153,13 @@ def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIR
self_submission_enabled=payload.self_submission_enabled,
max_active_tasks=payload.max_active_tasks,
daily_token_budget=payload.daily_token_budget,
expected_revision=payload.expected_revision,
updated_at=utc_now(),
)
except KeyError as exc:
raise HTTPException(status_code=404, detail="Host not found") from exc
except GovernancePolicyConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
_audit(repository, principal.id, host_id, "host_governance_policy_update")
return _host_response(policy)
+19 -2
View File
@@ -162,11 +162,15 @@ class UserSubmissionPolicyRequest(BaseModel):
submission_enabled: bool = True
allowed_host_ids: list[str] | None = None
allowed_device_targets: list[DeviceTargetModel] | None = None
expected_revision: int | None = Field(default=None, ge=0)
class UserSubmissionPolicyResponse(UserSubmissionPolicyRequest):
class UserSubmissionPolicyResponse(BaseModel):
user_id: str
revision: int
submission_enabled: bool
allowed_host_ids: list[str] | None = None
allowed_device_targets: list[DeviceTargetModel] | None = None
updated_at: datetime
@@ -174,9 +178,22 @@ class HostGovernancePolicyRequest(BaseModel):
self_submission_enabled: bool = True
max_active_tasks: int | None = Field(default=None, ge=1)
daily_token_budget: int | None = Field(default=None, ge=1)
expected_revision: int | None = Field(default=None, ge=0)
class HostGovernancePolicyResponse(HostGovernancePolicyRequest):
class HostGovernancePolicyResponse(BaseModel):
host_id: str
revision: int
self_submission_enabled: bool
max_active_tasks: int | None = None
daily_token_budget: int | None = None
updated_at: datetime
class HostTokenUsageSummaryResponse(BaseModel):
host_id: str
usage_day: str
daily_token_budget: int | None = None
used_tokens: int
reserved_tokens: int
remaining_tokens: int | None = None
+199 -2
View File
@@ -24,6 +24,8 @@ from cloud.db_models import (
UserRow,
UserSessionRow,
UserSubmissionPolicyRow,
TokenReservationRow,
TokenUsageEventRow,
)
from cloud.observability import current_correlation_id
from core.models import utc_now
@@ -824,12 +826,21 @@ class SQLAlchemyCloudRepository:
allowed_host_ids: tuple[str, ...] | None,
allowed_device_targets: tuple[tuple[str, str], ...] | None,
updated_at: datetime,
expected_revision: int | None = None,
) -> Any:
from cloud.governance import GovernancePolicyConflictError
with self._sessions.begin() as session:
if session.get(UserRow, user_id) is None:
raise KeyError(f"unknown user {user_id!r}")
row = session.get(UserSubmissionPolicyRow, user_id)
row = session.get(
UserSubmissionPolicyRow,
user_id,
with_for_update=self.engine.dialect.name == "postgresql",
)
if row is None:
if expected_revision not in {None, 0}:
raise GovernancePolicyConflictError("submission policy revision changed")
row = UserSubmissionPolicyRow(
user_id=user_id,
revision=1,
@@ -842,6 +853,11 @@ class SQLAlchemyCloudRepository:
)
session.add(row)
else:
if (
expected_revision is not None
and expected_revision != row.revision
):
raise GovernancePolicyConflictError("submission policy revision changed")
row.revision += 1
row.submission_enabled = 1 if submission_enabled else 0
row.allowed_host_ids_json = _dump_optional_list(allowed_host_ids)
@@ -865,12 +881,21 @@ class SQLAlchemyCloudRepository:
max_active_tasks: int | None,
daily_token_budget: int | None,
updated_at: datetime,
expected_revision: int | None = None,
) -> Any:
from cloud.governance import GovernancePolicyConflictError
with self._sessions.begin() as session:
if session.get(HostRow, host_id) is None:
raise KeyError(f"unknown host {host_id!r}")
row = session.get(HostGovernancePolicyRow, host_id)
row = session.get(
HostGovernancePolicyRow,
host_id,
with_for_update=self.engine.dialect.name == "postgresql",
)
if row is None:
if expected_revision not in {None, 0}:
raise GovernancePolicyConflictError("Host policy revision changed")
row = HostGovernancePolicyRow(
host_id=host_id,
revision=1,
@@ -881,6 +906,11 @@ class SQLAlchemyCloudRepository:
)
session.add(row)
else:
if (
expected_revision is not None
and expected_revision != row.revision
):
raise GovernancePolicyConflictError("Host policy revision changed")
row.revision += 1
row.self_submission_enabled = 1 if self_submission_enabled else 0
row.max_active_tasks = max_active_tasks
@@ -889,6 +919,133 @@ class SQLAlchemyCloudRepository:
session.flush()
return _host_governance_policy_from_row(row)
def count_active_tasks_for_host(self, host_id: str) -> int:
with self._sessions() as session:
count = session.scalar(
select(func.count())
.select_from(ScheduledTaskRow)
.where(
ScheduledTaskRow.assigned_host_id == host_id,
ScheduledTaskRow.status.in_(("assigned", "dispatched")),
)
)
return int(count or 0)
def reserve_host_token_budget(
self,
*,
reservation_id: str,
host_id: str,
usage_day: str,
reserved_tokens: int,
task_id: str | None,
attempt: int | None,
created_at: datetime,
expires_at: datetime,
) -> Any | None:
from cloud.governance import TokenBudgetExceededError
with self._sessions.begin() as session:
statement = select(HostGovernancePolicyRow).where(
HostGovernancePolicyRow.host_id == host_id
)
if self.engine.dialect.name == "postgresql":
statement = statement.with_for_update()
policy = session.scalars(statement).first()
if policy is None or policy.daily_token_budget is None:
return None
used = session.scalar(
select(func.coalesce(func.sum(TokenUsageEventRow.total_tokens), 0)).where(
TokenUsageEventRow.host_id == host_id,
TokenUsageEventRow.usage_day == usage_day,
)
)
reserved = session.scalar(
select(func.coalesce(func.sum(TokenReservationRow.reserved_tokens), 0)).where(
TokenReservationRow.host_id == host_id,
TokenReservationRow.usage_day == usage_day,
TokenReservationRow.expires_at > _iso(created_at),
)
)
if int(used or 0) + int(reserved or 0) + reserved_tokens > policy.daily_token_budget:
raise TokenBudgetExceededError("Host daily token budget is exhausted")
row = TokenReservationRow(
id=reservation_id, host_id=host_id, usage_day=usage_day,
reserved_tokens=reserved_tokens, task_id=task_id, attempt=attempt,
created_at=_iso(created_at), expires_at=_iso(expires_at),
)
session.add(row)
session.flush()
return _token_reservation_from_row(row)
def settle_host_token_reservation(
self,
*,
reservation_id: str,
event_id: str,
provider: str,
model: str,
input_tokens: int | None,
output_tokens: int | None,
total_tokens: int,
occurred_at: datetime,
) -> Any | None:
with self._sessions.begin() as session:
row = session.get(
TokenReservationRow, reservation_id,
with_for_update=self.engine.dialect.name == "postgresql",
)
if row is None:
return None
event = TokenUsageEventRow(
id=event_id, host_id=row.host_id, usage_day=row.usage_day,
task_id=row.task_id, attempt=row.attempt, provider=provider, model=model,
input_tokens=input_tokens, output_tokens=output_tokens,
total_tokens=total_tokens, occurred_at=_iso(occurred_at),
)
session.add(event)
session.delete(row)
session.flush()
return _token_usage_event_from_row(event)
def cleanup_expired_token_reservations(self, *, now: datetime, limit: int) -> int:
with self._sessions.begin() as session:
rows = session.scalars(
select(TokenReservationRow)
.where(TokenReservationRow.expires_at <= _iso(now))
.order_by(TokenReservationRow.expires_at)
.limit(limit)
).all()
for row in rows:
session.delete(row)
return len(rows)
def get_host_token_usage_summary(
self, *, host_id: str, usage_day: str, now: datetime,
) -> Any:
from cloud.governance import TokenUsageSummary
with self._sessions() as session:
policy = session.get(HostGovernancePolicyRow, host_id)
used = session.scalar(
select(func.coalesce(func.sum(TokenUsageEventRow.total_tokens), 0)).where(
TokenUsageEventRow.host_id == host_id,
TokenUsageEventRow.usage_day == usage_day,
)
)
reserved = session.scalar(
select(func.coalesce(func.sum(TokenReservationRow.reserved_tokens), 0)).where(
TokenReservationRow.host_id == host_id,
TokenReservationRow.usage_day == usage_day,
TokenReservationRow.expires_at > _iso(now),
)
)
return TokenUsageSummary(
host_id=host_id, usage_day=usage_day,
daily_token_budget=(policy.daily_token_budget if policy else None),
used_tokens=int(used or 0), reserved_tokens=int(reserved or 0),
)
def list_reserved_device_ids(self, *, now: datetime) -> set[str]:
with self._sessions() as session:
device_ids = session.scalars(
@@ -930,6 +1087,24 @@ class SQLAlchemyCloudRepository:
if device is None or device.status != "idle":
return None
policy_statement = select(HostGovernancePolicyRow).where(
HostGovernancePolicyRow.host_id == host_id
)
if self.engine.dialect.name == "postgresql":
policy_statement = policy_statement.with_for_update()
policy = session.scalars(policy_statement).first()
if policy is not None and policy.max_active_tasks is not None:
active_count = session.scalar(
select(func.count())
.select_from(ScheduledTaskRow)
.where(
ScheduledTaskRow.assigned_host_id == host_id,
ScheduledTaskRow.status.in_(("assigned", "dispatched")),
)
)
if int(active_count or 0) >= policy.max_active_tasks:
return None
active_reservation = session.scalar(
select(ScheduledTaskRow.id)
.where(
@@ -1376,6 +1551,28 @@ def _host_governance_policy_from_row(row: HostGovernancePolicyRow) -> Any:
)
def _token_reservation_from_row(row: TokenReservationRow) -> Any:
from cloud.governance import TokenReservation
return TokenReservation(
id=row.id, host_id=row.host_id, usage_day=row.usage_day,
reserved_tokens=row.reserved_tokens, task_id=row.task_id, attempt=row.attempt,
created_at=_parse_dt(row.created_at) or utc_now(),
expires_at=_parse_dt(row.expires_at) or utc_now(),
)
def _token_usage_event_from_row(row: TokenUsageEventRow) -> Any:
from cloud.governance import TokenUsageEvent
return TokenUsageEvent(
id=row.id, host_id=row.host_id, usage_day=row.usage_day,
task_id=row.task_id, attempt=row.attempt, provider=row.provider, model=row.model,
input_tokens=row.input_tokens, output_tokens=row.output_tokens,
total_tokens=row.total_tokens, occurred_at=_parse_dt(row.occurred_at) or utc_now(),
)
def _task_attempt_from_row(row: TaskAttemptRow) -> Any:
from cloud.repository import TaskAttemptRecord
+18
View File
@@ -132,6 +132,24 @@ def test_governance_routes_persist_revisioned_policies(tmp_path) -> None:
assert host_policy.json()["revision"] == 1
reread = client.get("/v1/hosts/host-a/governance-policy")
assert reread.json()["daily_token_budget"] == 1000
conflict = client.put(
"/v1/hosts/host-a/governance-policy",
json={
"max_active_tasks": 3,
"daily_token_budget": 1000,
"expected_revision": 1,
},
)
assert conflict.status_code == 200
stale = client.put(
"/v1/hosts/host-a/governance-policy",
json={"max_active_tasks": 4, "expected_revision": 1},
)
assert stale.status_code == 409
usage = client.get("/v1/hosts/host-a/token-usage")
assert usage.status_code == 200
assert usage.json()["used_tokens"] == 0
assert usage.json()["daily_token_budget"] == 1000
finally:
database.close()
+2
View File
@@ -38,6 +38,8 @@ def test_forward_and_downgrade_migrations_on_empty_database(tmp_path) -> None:
"cloud_user_sessions",
"cloud_login_throttles",
"cloud_auth_audit_events",
"cloud_token_reservations",
"cloud_token_usage_events",
} <= table_names
assert current_revision(database_url) == HEAD_REVISION
host_columns = {
+61 -1
View File
@@ -12,9 +12,15 @@ from cloud.internal_api.api import create_internal_router
from cloud.pool import DevicePool, PooledDevice
from cloud.scheduler import ScheduledTask, TaskConstraints, TaskScheduler
from cloud.store import CloudStore
from runtime.tool_calling_client import ToolCallDecision, ToolCallUsage
def _build_client(tmp_path) -> tuple[TestClient, DevicePool]:
def _build_client(
tmp_path,
*,
planner_client_factory=None,
planner_token_reservation_ceiling: int = 4096,
) -> tuple[TestClient, DevicePool]:
pool = DevicePool(
CloudStore(tmp_path / "internal.sqlite3"),
CloudConfig(stale_after_seconds=60),
@@ -39,6 +45,8 @@ def _build_client(tmp_path) -> tuple[TestClient, DevicePool]:
pool=pool,
auth_provider=auth_provider,
scheduler=TaskScheduler(pool, pool.store, CloudConfig(stale_after_seconds=60)),
planner_client_factory=planner_client_factory,
planner_token_reservation_ceiling=planner_token_reservation_ceiling,
)
)
return TestClient(app), pool
@@ -232,6 +240,58 @@ def test_heartbeat_and_self_submission_preserve_host_isolation(tmp_path) -> None
assert foreign.status_code == 403
def test_planner_proxy_reserves_and_enforces_host_daily_token_budget(tmp_path) -> None:
class FakePlannerClient:
calls = 0
def decide(self, **_kwargs):
self.calls += 1
return ToolCallDecision(
tool_name="tap",
arguments={"x": 1, "y": 2},
usage=ToolCallUsage(input_tokens=1, output_tokens=1, total_tokens=2),
)
planner = FakePlannerClient()
client, pool = _build_client(
tmp_path,
planner_client_factory=lambda: planner,
planner_token_reservation_ceiling=5,
)
headers = {"Authorization": "Bearer token-a"}
client.put(
"/internal/v1/hosts/host-a/heartbeat",
headers=headers,
json=_heartbeat_payload("host-a", "device-a"),
)
pool.store.upsert_host_governance_policy(
host_id="host-a",
self_submission_enabled=True,
max_active_tasks=None,
daily_token_budget=5,
updated_at=datetime.now(UTC),
)
payload = {
"host_id": "host-a",
"system_prompt": "system",
"user_prompt": "user",
"tools": [{"name": "tap", "description": "tap", "parameters": {}}],
"timeout_seconds": 1,
}
first = client.post(
"/internal/v1/hosts/host-a/planner/decide", headers=headers, json=payload
)
second = client.post(
"/internal/v1/hosts/host-a/planner/decide", headers=headers, json=payload
)
assert first.status_code == 200, first.text
assert first.json()["total_tokens"] == 2
assert second.status_code == 429
assert planner.calls == 1
def test_long_poll_claim_returns_at_most_one_owned_assignment(tmp_path) -> None:
client, pool = _build_client(tmp_path)
now = datetime.now(UTC)
+29
View File
@@ -264,3 +264,32 @@ def test_unavailable_explicit_target_is_never_rerouted(tmp_path) -> None:
task = pool.store.get_task(task_id)
assert task is not None
assert task.status == "queued"
def test_host_active_task_policy_keeps_excess_tasks_queued(tmp_path) -> None:
from datetime import UTC, datetime
pool = _pool_with_devices(
tmp_path,
_device("device-a"),
_device("device-b"),
host_id="host-a",
)
pool.store.upsert_host_governance_policy(
host_id="host-a",
self_submission_enabled=True,
max_active_tasks=1,
daily_token_budget=None,
updated_at=datetime.now(UTC),
)
scheduler = TaskScheduler(pool, pool.store, _config())
first_id = scheduler.submit(goal="first")
second_id = scheduler.submit(goal="second")
assignments = scheduler.assign()
assert [assignment.task_id for assignment in assignments] == [first_id]
assert pool.store.count_active_tasks_for_host("host-a") == 1
second = pool.store.get_task(second_id)
assert second is not None
assert second.status == "queued"