@@ -36,6 +36,8 @@ class CloudControlConfig:
|
||||
login_block_seconds: int = 900
|
||||
session_cookie_secure: bool = False
|
||||
trust_proxy_headers: bool = False
|
||||
planner_token_reservation_ceiling: int = 4096
|
||||
planner_token_reservation_ttl_seconds: int = 300
|
||||
|
||||
|
||||
def load_control_config(
|
||||
@@ -115,6 +117,16 @@ def load_control_config(
|
||||
values.get("CLOUD_TRUST_PROXY_HEADERS"),
|
||||
default=False,
|
||||
),
|
||||
planner_token_reservation_ceiling=_positive_int(
|
||||
values,
|
||||
"CLOUD_PLANNER_TOKEN_RESERVATION_CEILING",
|
||||
4096,
|
||||
),
|
||||
planner_token_reservation_ttl_seconds=_positive_int(
|
||||
values,
|
||||
"CLOUD_PLANNER_TOKEN_RESERVATION_TTL_SECONDS",
|
||||
300,
|
||||
),
|
||||
)
|
||||
validate_control_config(config)
|
||||
return config
|
||||
|
||||
@@ -241,3 +241,44 @@ class HostGovernancePolicyRow(Base):
|
||||
max_active_tasks: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
daily_token_budget: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
updated_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
|
||||
class TokenReservationRow(Base):
|
||||
__tablename__ = "cloud_token_reservations"
|
||||
__table_args__ = (
|
||||
Index("ix_cloud_token_reservations_host_day", "host_id", "usage_day"),
|
||||
Index("ix_cloud_token_reservations_expires_at", "expires_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
usage_day: Mapped[str] = mapped_column(String, nullable=False)
|
||||
reserved_tokens: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
task_id: Mapped[str | None] = mapped_column(String, nullable=True)
|
||||
attempt: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
created_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||
expires_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
|
||||
class TokenUsageEventRow(Base):
|
||||
__tablename__ = "cloud_token_usage_events"
|
||||
__table_args__ = (
|
||||
Index("ix_cloud_token_usage_events_host_day", "host_id", "usage_day"),
|
||||
Index("ix_cloud_token_usage_events_occurred_at", "occurred_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
usage_day: Mapped[str] = mapped_column(String, nullable=False)
|
||||
task_id: Mapped[str | None] = mapped_column(String, nullable=True)
|
||||
attempt: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
provider: Mapped[str] = mapped_column(String, nullable=False)
|
||||
model: Mapped[str] = mapped_column(String, nullable=False)
|
||||
input_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
output_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
total_tokens: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
occurred_at: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
@@ -10,6 +10,14 @@ class TaskSubmissionPolicyError(PermissionError):
|
||||
"""Raised when a human user's policy disallows a task submission."""
|
||||
|
||||
|
||||
class GovernancePolicyConflictError(RuntimeError):
|
||||
"""Raised when a policy write was based on an obsolete revision."""
|
||||
|
||||
|
||||
class TokenBudgetExceededError(RuntimeError):
|
||||
"""Raised before a Cloud-proxied call would exceed a Host's token budget."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UserSubmissionPolicy:
|
||||
user_id: str
|
||||
@@ -30,6 +38,48 @@ class HostGovernancePolicy:
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TokenReservation:
|
||||
id: str
|
||||
host_id: str
|
||||
usage_day: str
|
||||
reserved_tokens: int
|
||||
task_id: str | None
|
||||
attempt: int | None
|
||||
created_at: datetime
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TokenUsageEvent:
|
||||
id: str
|
||||
host_id: str
|
||||
usage_day: str
|
||||
task_id: str | None
|
||||
attempt: int | None
|
||||
provider: str
|
||||
model: str
|
||||
input_tokens: int | None
|
||||
output_tokens: int | None
|
||||
total_tokens: int
|
||||
occurred_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TokenUsageSummary:
|
||||
host_id: str
|
||||
usage_day: str
|
||||
daily_token_budget: int | None
|
||||
used_tokens: int
|
||||
reserved_tokens: int
|
||||
|
||||
@property
|
||||
def remaining_tokens(self) -> int | None:
|
||||
if self.daily_token_budget is None:
|
||||
return None
|
||||
return max(0, self.daily_token_budget - self.used_tokens - self.reserved_tokens)
|
||||
|
||||
|
||||
def enforce_user_submission_policy(
|
||||
policy: UserSubmissionPolicy | None,
|
||||
*,
|
||||
|
||||
@@ -44,6 +44,7 @@ from cloud.repository import (
|
||||
DeviceEnrollmentConflictError,
|
||||
HostEnrollmentConflictError,
|
||||
)
|
||||
from cloud.governance import TokenBudgetExceededError
|
||||
from core.models import Device, utc_now
|
||||
from runtime.tool_calling_client import ToolCallingClient, ToolCallUnavailable
|
||||
from runtime.tool_specs import ToolSpec
|
||||
@@ -65,11 +66,17 @@ def create_internal_router(
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
planner_client_factory: Callable[[], ToolCallingClient] | None = None,
|
||||
scheduler: TaskScheduler | None = None,
|
||||
planner_token_reservation_ceiling: int = 4096,
|
||||
planner_token_reservation_ttl_seconds: float = 300.0,
|
||||
) -> APIRouter:
|
||||
if claim_poll_interval_seconds <= 0:
|
||||
raise ValueError("claim_poll_interval_seconds must be greater than zero")
|
||||
if lease_duration_seconds <= 0:
|
||||
raise ValueError("lease_duration_seconds must be greater than zero")
|
||||
if planner_token_reservation_ceiling <= 0:
|
||||
raise ValueError("planner_token_reservation_ceiling must be greater than zero")
|
||||
if planner_token_reservation_ttl_seconds <= 0:
|
||||
raise ValueError("planner_token_reservation_ttl_seconds must be greater than zero")
|
||||
router = APIRouter(prefix=version_prefix, tags=["host-agent"])
|
||||
build_planner_client = planner_client_factory or _default_planner_client_factory
|
||||
|
||||
@@ -352,6 +359,7 @@ def create_internal_router(
|
||||
response_model_exclude_none=True,
|
||||
responses={
|
||||
status.HTTP_502_BAD_GATEWAY: {"model": PlannerDecisionError},
|
||||
status.HTTP_429_TOO_MANY_REQUESTS: {"model": PlannerDecisionError},
|
||||
},
|
||||
)
|
||||
def decide_planner_call(
|
||||
@@ -385,6 +393,25 @@ def create_internal_router(
|
||||
for tool in payload.tools
|
||||
]
|
||||
|
||||
now = utc_now()
|
||||
reservation = None
|
||||
try:
|
||||
reservation = pool.store.reserve_host_token_budget(
|
||||
reservation_id=uuid4().hex,
|
||||
host_id=host_id,
|
||||
usage_day=now.date().isoformat(),
|
||||
reserved_tokens=planner_token_reservation_ceiling,
|
||||
task_id=None,
|
||||
attempt=None,
|
||||
created_at=now,
|
||||
expires_at=now + timedelta(seconds=planner_token_reservation_ttl_seconds),
|
||||
)
|
||||
except TokenBudgetExceededError as exc:
|
||||
return JSONResponse(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
content=PlannerDecisionError(detail=str(exc)).model_dump(),
|
||||
)
|
||||
|
||||
started_at = monotonic()
|
||||
client = build_planner_client()
|
||||
try:
|
||||
@@ -408,6 +435,19 @@ def create_internal_router(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
content=PlannerDecisionError(detail=str(exc)).model_dump(),
|
||||
)
|
||||
usage = decision.usage
|
||||
if reservation is not None and usage is not None and usage.total_tokens is not None:
|
||||
planner_config = load_cloud_planner_config()
|
||||
pool.store.settle_host_token_reservation(
|
||||
reservation_id=reservation.id,
|
||||
event_id=uuid4().hex,
|
||||
provider=planner_config.provider,
|
||||
model=planner_config.resolved_model(),
|
||||
input_tokens=usage.input_tokens,
|
||||
output_tokens=usage.output_tokens,
|
||||
total_tokens=usage.total_tokens,
|
||||
occurred_at=utc_now(),
|
||||
)
|
||||
logger.info(
|
||||
"planner-decision request resolved",
|
||||
extra={
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Add Cloud-proxy token reservations and usage events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0005_cloud_token_usage"
|
||||
down_revision = "0004_cloud_governance"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"cloud_token_reservations",
|
||||
sa.Column("id", sa.String(), primary_key=True),
|
||||
sa.Column("host_id", sa.String(), sa.ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("usage_day", sa.String(), nullable=False),
|
||||
sa.Column("reserved_tokens", sa.Integer(), nullable=False),
|
||||
sa.Column("task_id", sa.String(), nullable=True),
|
||||
sa.Column("attempt", sa.Integer(), nullable=True),
|
||||
sa.Column("created_at", sa.String(), nullable=False),
|
||||
sa.Column("expires_at", sa.String(), nullable=False),
|
||||
)
|
||||
op.create_index("ix_cloud_token_reservations_host_day", "cloud_token_reservations", ["host_id", "usage_day"])
|
||||
op.create_index("ix_cloud_token_reservations_expires_at", "cloud_token_reservations", ["expires_at"])
|
||||
op.create_table(
|
||||
"cloud_token_usage_events",
|
||||
sa.Column("id", sa.String(), primary_key=True),
|
||||
sa.Column("host_id", sa.String(), sa.ForeignKey("host_registrations.host_id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("usage_day", sa.String(), nullable=False),
|
||||
sa.Column("task_id", sa.String(), nullable=True),
|
||||
sa.Column("attempt", sa.Integer(), nullable=True),
|
||||
sa.Column("provider", sa.String(), nullable=False),
|
||||
sa.Column("model", sa.String(), nullable=False),
|
||||
sa.Column("input_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("output_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("total_tokens", sa.Integer(), nullable=False),
|
||||
sa.Column("occurred_at", sa.String(), nullable=False),
|
||||
)
|
||||
op.create_index("ix_cloud_token_usage_events_host_day", "cloud_token_usage_events", ["host_id", "usage_day"])
|
||||
op.create_index("ix_cloud_token_usage_events_occurred_at", "cloud_token_usage_events", ["occurred_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_cloud_token_usage_events_occurred_at", table_name="cloud_token_usage_events")
|
||||
op.drop_index("ix_cloud_token_usage_events_host_day", table_name="cloud_token_usage_events")
|
||||
op.drop_table("cloud_token_usage_events")
|
||||
op.drop_index("ix_cloud_token_reservations_expires_at", table_name="cloud_token_reservations")
|
||||
op.drop_index("ix_cloud_token_reservations_host_day", table_name="cloud_token_reservations")
|
||||
op.drop_table("cloud_token_reservations")
|
||||
@@ -15,7 +15,13 @@ if TYPE_CHECKING:
|
||||
UserAccount,
|
||||
UserSession,
|
||||
)
|
||||
from cloud.governance import HostGovernancePolicy, UserSubmissionPolicy
|
||||
from cloud.governance import (
|
||||
HostGovernancePolicy,
|
||||
TokenReservation,
|
||||
TokenUsageSummary,
|
||||
TokenUsageEvent,
|
||||
UserSubmissionPolicy,
|
||||
)
|
||||
|
||||
|
||||
AttemptStatus = Literal["assigned", "dispatched", "done", "failed", "expired"]
|
||||
@@ -293,6 +299,7 @@ class CloudRepository(Protocol):
|
||||
submission_enabled: bool,
|
||||
allowed_host_ids: tuple[str, ...] | None,
|
||||
allowed_device_targets: tuple[tuple[str, str], ...] | None,
|
||||
expected_revision: int | None,
|
||||
updated_at: datetime,
|
||||
) -> UserSubmissionPolicy: ...
|
||||
|
||||
@@ -305,9 +312,30 @@ class CloudRepository(Protocol):
|
||||
self_submission_enabled: bool,
|
||||
max_active_tasks: int | None,
|
||||
daily_token_budget: int | None,
|
||||
expected_revision: int | None,
|
||||
updated_at: datetime,
|
||||
) -> HostGovernancePolicy: ...
|
||||
|
||||
def count_active_tasks_for_host(self, host_id: str) -> int: ...
|
||||
|
||||
def reserve_host_token_budget(
|
||||
self, *, reservation_id: str, host_id: str, usage_day: str,
|
||||
reserved_tokens: int, task_id: str | None, attempt: int | None,
|
||||
created_at: datetime, expires_at: datetime,
|
||||
) -> TokenReservation | None: ...
|
||||
|
||||
def settle_host_token_reservation(
|
||||
self, *, reservation_id: str, event_id: str, provider: str, model: str,
|
||||
input_tokens: int | None, output_tokens: int | None, total_tokens: int,
|
||||
occurred_at: datetime,
|
||||
) -> TokenUsageEvent | None: ...
|
||||
|
||||
def cleanup_expired_token_reservations(self, *, now: datetime, limit: int) -> int: ...
|
||||
|
||||
def get_host_token_usage_summary(
|
||||
self, *, host_id: str, usage_day: str, now: datetime,
|
||||
) -> TokenUsageSummary: ...
|
||||
|
||||
def list_reserved_device_ids(self, *, now: datetime) -> set[str]: ...
|
||||
|
||||
def assign_task(
|
||||
|
||||
@@ -9,7 +9,7 @@ from alembic.runtime.migration import MigrationContext
|
||||
from cloud.database import create_database_engine, normalize_database_url
|
||||
|
||||
|
||||
HEAD_REVISION = "0004_cloud_governance"
|
||||
HEAD_REVISION = "0005_cloud_token_usage"
|
||||
|
||||
|
||||
class SchemaVersionError(RuntimeError):
|
||||
|
||||
@@ -5,10 +5,12 @@ from uuid import uuid4
|
||||
from fastapi import APIRouter, HTTPException, Request, status
|
||||
|
||||
from cloud.auth import AuthProvider, GOVERNANCE_ADMIN_SCOPE, GOVERNANCE_READ_SCOPE
|
||||
from cloud.governance import GovernancePolicyConflictError
|
||||
from cloud.observability import current_correlation_id
|
||||
from cloud.sdk.models import (
|
||||
HostGovernancePolicyRequest,
|
||||
HostGovernancePolicyResponse,
|
||||
HostTokenUsageSummaryResponse,
|
||||
UserSubmissionPolicyRequest,
|
||||
UserSubmissionPolicyResponse,
|
||||
)
|
||||
@@ -79,10 +81,13 @@ def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIR
|
||||
if payload.allowed_device_targets is not None
|
||||
else None
|
||||
),
|
||||
expected_revision=payload.expected_revision,
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="user not found") from exc
|
||||
except GovernancePolicyConflictError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
_audit(repository, principal.id, user_id, "user_submission_policy_update")
|
||||
return UserSubmissionPolicyResponse(
|
||||
user_id=policy.user_id,
|
||||
@@ -111,6 +116,27 @@ def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIR
|
||||
raise HTTPException(status_code=404, detail="Host policy not found")
|
||||
return _host_response(policy)
|
||||
|
||||
@router.get(
|
||||
"/hosts/{host_id}/token-usage",
|
||||
response_model=HostTokenUsageSummaryResponse,
|
||||
)
|
||||
def get_host_token_usage(
|
||||
host_id: str, request: Request
|
||||
) -> HostTokenUsageSummaryResponse:
|
||||
authorize(request, GOVERNANCE_READ_SCOPE)
|
||||
now = utc_now()
|
||||
summary = repository.get_host_token_usage_summary(
|
||||
host_id=host_id, usage_day=now.date().isoformat(), now=now
|
||||
)
|
||||
return HostTokenUsageSummaryResponse(
|
||||
host_id=summary.host_id,
|
||||
usage_day=summary.usage_day,
|
||||
daily_token_budget=summary.daily_token_budget,
|
||||
used_tokens=summary.used_tokens,
|
||||
reserved_tokens=summary.reserved_tokens,
|
||||
remaining_tokens=summary.remaining_tokens,
|
||||
)
|
||||
|
||||
@router.put(
|
||||
"/hosts/{host_id}/governance-policy",
|
||||
response_model=HostGovernancePolicyResponse,
|
||||
@@ -127,10 +153,13 @@ def create_governance_router(*, repository, auth_provider: AuthProvider) -> APIR
|
||||
self_submission_enabled=payload.self_submission_enabled,
|
||||
max_active_tasks=payload.max_active_tasks,
|
||||
daily_token_budget=payload.daily_token_budget,
|
||||
expected_revision=payload.expected_revision,
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="Host not found") from exc
|
||||
except GovernancePolicyConflictError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
_audit(repository, principal.id, host_id, "host_governance_policy_update")
|
||||
return _host_response(policy)
|
||||
|
||||
|
||||
@@ -162,11 +162,15 @@ class UserSubmissionPolicyRequest(BaseModel):
|
||||
submission_enabled: bool = True
|
||||
allowed_host_ids: list[str] | None = None
|
||||
allowed_device_targets: list[DeviceTargetModel] | None = None
|
||||
expected_revision: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class UserSubmissionPolicyResponse(UserSubmissionPolicyRequest):
|
||||
class UserSubmissionPolicyResponse(BaseModel):
|
||||
user_id: str
|
||||
revision: int
|
||||
submission_enabled: bool
|
||||
allowed_host_ids: list[str] | None = None
|
||||
allowed_device_targets: list[DeviceTargetModel] | None = None
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@@ -174,9 +178,22 @@ class HostGovernancePolicyRequest(BaseModel):
|
||||
self_submission_enabled: bool = True
|
||||
max_active_tasks: int | None = Field(default=None, ge=1)
|
||||
daily_token_budget: int | None = Field(default=None, ge=1)
|
||||
expected_revision: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class HostGovernancePolicyResponse(HostGovernancePolicyRequest):
|
||||
class HostGovernancePolicyResponse(BaseModel):
|
||||
host_id: str
|
||||
revision: int
|
||||
self_submission_enabled: bool
|
||||
max_active_tasks: int | None = None
|
||||
daily_token_budget: int | None = None
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class HostTokenUsageSummaryResponse(BaseModel):
|
||||
host_id: str
|
||||
usage_day: str
|
||||
daily_token_budget: int | None = None
|
||||
used_tokens: int
|
||||
reserved_tokens: int
|
||||
remaining_tokens: int | None = None
|
||||
|
||||
@@ -24,6 +24,8 @@ from cloud.db_models import (
|
||||
UserRow,
|
||||
UserSessionRow,
|
||||
UserSubmissionPolicyRow,
|
||||
TokenReservationRow,
|
||||
TokenUsageEventRow,
|
||||
)
|
||||
from cloud.observability import current_correlation_id
|
||||
from core.models import utc_now
|
||||
@@ -824,12 +826,21 @@ class SQLAlchemyCloudRepository:
|
||||
allowed_host_ids: tuple[str, ...] | None,
|
||||
allowed_device_targets: tuple[tuple[str, str], ...] | None,
|
||||
updated_at: datetime,
|
||||
expected_revision: int | None = None,
|
||||
) -> Any:
|
||||
from cloud.governance import GovernancePolicyConflictError
|
||||
|
||||
with self._sessions.begin() as session:
|
||||
if session.get(UserRow, user_id) is None:
|
||||
raise KeyError(f"unknown user {user_id!r}")
|
||||
row = session.get(UserSubmissionPolicyRow, user_id)
|
||||
row = session.get(
|
||||
UserSubmissionPolicyRow,
|
||||
user_id,
|
||||
with_for_update=self.engine.dialect.name == "postgresql",
|
||||
)
|
||||
if row is None:
|
||||
if expected_revision not in {None, 0}:
|
||||
raise GovernancePolicyConflictError("submission policy revision changed")
|
||||
row = UserSubmissionPolicyRow(
|
||||
user_id=user_id,
|
||||
revision=1,
|
||||
@@ -842,6 +853,11 @@ class SQLAlchemyCloudRepository:
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
if (
|
||||
expected_revision is not None
|
||||
and expected_revision != row.revision
|
||||
):
|
||||
raise GovernancePolicyConflictError("submission policy revision changed")
|
||||
row.revision += 1
|
||||
row.submission_enabled = 1 if submission_enabled else 0
|
||||
row.allowed_host_ids_json = _dump_optional_list(allowed_host_ids)
|
||||
@@ -865,12 +881,21 @@ class SQLAlchemyCloudRepository:
|
||||
max_active_tasks: int | None,
|
||||
daily_token_budget: int | None,
|
||||
updated_at: datetime,
|
||||
expected_revision: int | None = None,
|
||||
) -> Any:
|
||||
from cloud.governance import GovernancePolicyConflictError
|
||||
|
||||
with self._sessions.begin() as session:
|
||||
if session.get(HostRow, host_id) is None:
|
||||
raise KeyError(f"unknown host {host_id!r}")
|
||||
row = session.get(HostGovernancePolicyRow, host_id)
|
||||
row = session.get(
|
||||
HostGovernancePolicyRow,
|
||||
host_id,
|
||||
with_for_update=self.engine.dialect.name == "postgresql",
|
||||
)
|
||||
if row is None:
|
||||
if expected_revision not in {None, 0}:
|
||||
raise GovernancePolicyConflictError("Host policy revision changed")
|
||||
row = HostGovernancePolicyRow(
|
||||
host_id=host_id,
|
||||
revision=1,
|
||||
@@ -881,6 +906,11 @@ class SQLAlchemyCloudRepository:
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
if (
|
||||
expected_revision is not None
|
||||
and expected_revision != row.revision
|
||||
):
|
||||
raise GovernancePolicyConflictError("Host policy revision changed")
|
||||
row.revision += 1
|
||||
row.self_submission_enabled = 1 if self_submission_enabled else 0
|
||||
row.max_active_tasks = max_active_tasks
|
||||
@@ -889,6 +919,133 @@ class SQLAlchemyCloudRepository:
|
||||
session.flush()
|
||||
return _host_governance_policy_from_row(row)
|
||||
|
||||
def count_active_tasks_for_host(self, host_id: str) -> int:
|
||||
with self._sessions() as session:
|
||||
count = session.scalar(
|
||||
select(func.count())
|
||||
.select_from(ScheduledTaskRow)
|
||||
.where(
|
||||
ScheduledTaskRow.assigned_host_id == host_id,
|
||||
ScheduledTaskRow.status.in_(("assigned", "dispatched")),
|
||||
)
|
||||
)
|
||||
return int(count or 0)
|
||||
|
||||
def reserve_host_token_budget(
|
||||
self,
|
||||
*,
|
||||
reservation_id: str,
|
||||
host_id: str,
|
||||
usage_day: str,
|
||||
reserved_tokens: int,
|
||||
task_id: str | None,
|
||||
attempt: int | None,
|
||||
created_at: datetime,
|
||||
expires_at: datetime,
|
||||
) -> Any | None:
|
||||
from cloud.governance import TokenBudgetExceededError
|
||||
|
||||
with self._sessions.begin() as session:
|
||||
statement = select(HostGovernancePolicyRow).where(
|
||||
HostGovernancePolicyRow.host_id == host_id
|
||||
)
|
||||
if self.engine.dialect.name == "postgresql":
|
||||
statement = statement.with_for_update()
|
||||
policy = session.scalars(statement).first()
|
||||
if policy is None or policy.daily_token_budget is None:
|
||||
return None
|
||||
used = session.scalar(
|
||||
select(func.coalesce(func.sum(TokenUsageEventRow.total_tokens), 0)).where(
|
||||
TokenUsageEventRow.host_id == host_id,
|
||||
TokenUsageEventRow.usage_day == usage_day,
|
||||
)
|
||||
)
|
||||
reserved = session.scalar(
|
||||
select(func.coalesce(func.sum(TokenReservationRow.reserved_tokens), 0)).where(
|
||||
TokenReservationRow.host_id == host_id,
|
||||
TokenReservationRow.usage_day == usage_day,
|
||||
TokenReservationRow.expires_at > _iso(created_at),
|
||||
)
|
||||
)
|
||||
if int(used or 0) + int(reserved or 0) + reserved_tokens > policy.daily_token_budget:
|
||||
raise TokenBudgetExceededError("Host daily token budget is exhausted")
|
||||
row = TokenReservationRow(
|
||||
id=reservation_id, host_id=host_id, usage_day=usage_day,
|
||||
reserved_tokens=reserved_tokens, task_id=task_id, attempt=attempt,
|
||||
created_at=_iso(created_at), expires_at=_iso(expires_at),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return _token_reservation_from_row(row)
|
||||
|
||||
def settle_host_token_reservation(
|
||||
self,
|
||||
*,
|
||||
reservation_id: str,
|
||||
event_id: str,
|
||||
provider: str,
|
||||
model: str,
|
||||
input_tokens: int | None,
|
||||
output_tokens: int | None,
|
||||
total_tokens: int,
|
||||
occurred_at: datetime,
|
||||
) -> Any | None:
|
||||
with self._sessions.begin() as session:
|
||||
row = session.get(
|
||||
TokenReservationRow, reservation_id,
|
||||
with_for_update=self.engine.dialect.name == "postgresql",
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
event = TokenUsageEventRow(
|
||||
id=event_id, host_id=row.host_id, usage_day=row.usage_day,
|
||||
task_id=row.task_id, attempt=row.attempt, provider=provider, model=model,
|
||||
input_tokens=input_tokens, output_tokens=output_tokens,
|
||||
total_tokens=total_tokens, occurred_at=_iso(occurred_at),
|
||||
)
|
||||
session.add(event)
|
||||
session.delete(row)
|
||||
session.flush()
|
||||
return _token_usage_event_from_row(event)
|
||||
|
||||
def cleanup_expired_token_reservations(self, *, now: datetime, limit: int) -> int:
|
||||
with self._sessions.begin() as session:
|
||||
rows = session.scalars(
|
||||
select(TokenReservationRow)
|
||||
.where(TokenReservationRow.expires_at <= _iso(now))
|
||||
.order_by(TokenReservationRow.expires_at)
|
||||
.limit(limit)
|
||||
).all()
|
||||
for row in rows:
|
||||
session.delete(row)
|
||||
return len(rows)
|
||||
|
||||
def get_host_token_usage_summary(
|
||||
self, *, host_id: str, usage_day: str, now: datetime,
|
||||
) -> Any:
|
||||
from cloud.governance import TokenUsageSummary
|
||||
|
||||
with self._sessions() as session:
|
||||
policy = session.get(HostGovernancePolicyRow, host_id)
|
||||
used = session.scalar(
|
||||
select(func.coalesce(func.sum(TokenUsageEventRow.total_tokens), 0)).where(
|
||||
TokenUsageEventRow.host_id == host_id,
|
||||
TokenUsageEventRow.usage_day == usage_day,
|
||||
)
|
||||
)
|
||||
reserved = session.scalar(
|
||||
select(func.coalesce(func.sum(TokenReservationRow.reserved_tokens), 0)).where(
|
||||
TokenReservationRow.host_id == host_id,
|
||||
TokenReservationRow.usage_day == usage_day,
|
||||
TokenReservationRow.expires_at > _iso(now),
|
||||
)
|
||||
)
|
||||
return TokenUsageSummary(
|
||||
host_id=host_id, usage_day=usage_day,
|
||||
daily_token_budget=(policy.daily_token_budget if policy else None),
|
||||
used_tokens=int(used or 0), reserved_tokens=int(reserved or 0),
|
||||
)
|
||||
|
||||
def list_reserved_device_ids(self, *, now: datetime) -> set[str]:
|
||||
with self._sessions() as session:
|
||||
device_ids = session.scalars(
|
||||
@@ -930,6 +1087,24 @@ class SQLAlchemyCloudRepository:
|
||||
if device is None or device.status != "idle":
|
||||
return None
|
||||
|
||||
policy_statement = select(HostGovernancePolicyRow).where(
|
||||
HostGovernancePolicyRow.host_id == host_id
|
||||
)
|
||||
if self.engine.dialect.name == "postgresql":
|
||||
policy_statement = policy_statement.with_for_update()
|
||||
policy = session.scalars(policy_statement).first()
|
||||
if policy is not None and policy.max_active_tasks is not None:
|
||||
active_count = session.scalar(
|
||||
select(func.count())
|
||||
.select_from(ScheduledTaskRow)
|
||||
.where(
|
||||
ScheduledTaskRow.assigned_host_id == host_id,
|
||||
ScheduledTaskRow.status.in_(("assigned", "dispatched")),
|
||||
)
|
||||
)
|
||||
if int(active_count or 0) >= policy.max_active_tasks:
|
||||
return None
|
||||
|
||||
active_reservation = session.scalar(
|
||||
select(ScheduledTaskRow.id)
|
||||
.where(
|
||||
@@ -1376,6 +1551,28 @@ def _host_governance_policy_from_row(row: HostGovernancePolicyRow) -> Any:
|
||||
)
|
||||
|
||||
|
||||
def _token_reservation_from_row(row: TokenReservationRow) -> Any:
|
||||
from cloud.governance import TokenReservation
|
||||
|
||||
return TokenReservation(
|
||||
id=row.id, host_id=row.host_id, usage_day=row.usage_day,
|
||||
reserved_tokens=row.reserved_tokens, task_id=row.task_id, attempt=row.attempt,
|
||||
created_at=_parse_dt(row.created_at) or utc_now(),
|
||||
expires_at=_parse_dt(row.expires_at) or utc_now(),
|
||||
)
|
||||
|
||||
|
||||
def _token_usage_event_from_row(row: TokenUsageEventRow) -> Any:
|
||||
from cloud.governance import TokenUsageEvent
|
||||
|
||||
return TokenUsageEvent(
|
||||
id=row.id, host_id=row.host_id, usage_day=row.usage_day,
|
||||
task_id=row.task_id, attempt=row.attempt, provider=row.provider, model=row.model,
|
||||
input_tokens=row.input_tokens, output_tokens=row.output_tokens,
|
||||
total_tokens=row.total_tokens, occurred_at=_parse_dt(row.occurred_at) or utc_now(),
|
||||
)
|
||||
|
||||
|
||||
def _task_attempt_from_row(row: TaskAttemptRow) -> Any:
|
||||
from cloud.repository import TaskAttemptRecord
|
||||
|
||||
|
||||
Reference in New Issue
Block a user