feat: checkpoint device agent runtime milestones
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
"""Semantic scene enrichment primitives."""
|
||||
|
||||
from semantic.config import SemanticConfig, load_config
|
||||
from semantic.models import SemanticScene, SemanticWidget
|
||||
|
||||
__all__ = [
|
||||
"SemanticConfig",
|
||||
"SemanticScene",
|
||||
"SemanticWidget",
|
||||
"load_config",
|
||||
]
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
DEFAULT_MODEL = "claude-haiku-4-5"
|
||||
DEFAULT_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
ENABLED_ENV = "SEMANTIC_ENRICHMENT_ENABLED"
|
||||
MODEL_ENV = "SEMANTIC_MODEL"
|
||||
TIMEOUT_ENV = "SEMANTIC_TIMEOUT_SECONDS"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticConfig:
|
||||
enabled: bool = False
|
||||
model: str = DEFAULT_MODEL
|
||||
timeout: float = DEFAULT_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def load_config(env: Mapping[str, str] | None = None) -> SemanticConfig:
|
||||
values = env or os.environ
|
||||
return SemanticConfig(
|
||||
enabled=_parse_bool(values.get(ENABLED_ENV), default=False),
|
||||
model=values.get(MODEL_ENV) or DEFAULT_MODEL,
|
||||
timeout=_parse_timeout(values.get(TIMEOUT_ENV)),
|
||||
)
|
||||
|
||||
|
||||
def _parse_bool(value: str | None, *, default: bool) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on", "enabled"}
|
||||
|
||||
|
||||
def _parse_timeout(value: str | None) -> float:
|
||||
if value is None:
|
||||
return DEFAULT_TIMEOUT_SECONDS
|
||||
try:
|
||||
timeout = float(value)
|
||||
except ValueError:
|
||||
return DEFAULT_TIMEOUT_SECONDS
|
||||
return timeout if timeout > 0 else DEFAULT_TIMEOUT_SECONDS
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Protocol
|
||||
|
||||
from core.models import Scene
|
||||
from semantic.config import SemanticConfig, load_config
|
||||
from semantic.llm_client import AnthropicSemanticClient, EnrichmentUnavailable
|
||||
from semantic.models import SemanticScene
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SemanticLLMClient(Protocol):
|
||||
def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
|
||||
...
|
||||
|
||||
|
||||
def enrich_scene(
|
||||
scene: Scene,
|
||||
*,
|
||||
client: SemanticLLMClient | None = None,
|
||||
config: SemanticConfig | None = None,
|
||||
) -> SemanticScene | None:
|
||||
settings = config or load_config()
|
||||
if not settings.enabled:
|
||||
return None
|
||||
|
||||
llm_client = client or AnthropicSemanticClient(model=settings.model)
|
||||
try:
|
||||
payload = llm_client.enrich(scene.to_dict(), timeout=settings.timeout)
|
||||
semantic_scene = SemanticScene.from_dict(payload)
|
||||
return _with_known_widgets_only(scene, semantic_scene)
|
||||
except EnrichmentUnavailable as exc:
|
||||
logger.info("semantic enrichment unavailable: %s", exc)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.info("semantic enrichment failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _with_known_widgets_only(
|
||||
scene: Scene,
|
||||
semantic_scene: SemanticScene,
|
||||
) -> SemanticScene | None:
|
||||
known_element_ids = {element.id for element in scene.elements}
|
||||
if not known_element_ids:
|
||||
return semantic_scene
|
||||
|
||||
widgets = [
|
||||
widget
|
||||
for widget in semantic_scene.widgets
|
||||
if widget.element_id in known_element_ids
|
||||
]
|
||||
if not widgets:
|
||||
return None
|
||||
|
||||
return SemanticScene(
|
||||
page=semantic_scene.page,
|
||||
intents=semantic_scene.intents,
|
||||
widgets=widgets,
|
||||
)
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from semantic.config import DEFAULT_MODEL
|
||||
from semantic.models import SEMANTIC_SCENE_SCHEMA, SemanticScene
|
||||
from semantic.prompts import ENRICHMENT_SYSTEM_PROMPT, scene_user_prompt
|
||||
|
||||
_SDK_UNAVAILABLE_EXCEPTION_NAMES = {
|
||||
"APITimeoutError",
|
||||
"APIConnectionError",
|
||||
"RateLimitError",
|
||||
"AuthenticationError",
|
||||
"APIStatusError",
|
||||
}
|
||||
|
||||
|
||||
class EnrichmentUnavailable(Exception):
|
||||
"""Internal signal for expected enrichment transport/response failures."""
|
||||
|
||||
|
||||
class AnthropicSemanticClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model: str = DEFAULT_MODEL,
|
||||
transport: Any | None = None,
|
||||
max_tokens: int = 1024,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self._transport = transport
|
||||
self.max_tokens = max_tokens
|
||||
|
||||
def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
|
||||
try:
|
||||
response = self._create_message(scene_json, timeout=timeout)
|
||||
payload = _extract_response_body(response)
|
||||
return _validate_payload(payload)
|
||||
except EnrichmentUnavailable:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise EnrichmentUnavailable(str(exc)) from exc
|
||||
|
||||
def _create_message(self, scene_json: dict[str, Any], *, timeout: float) -> Any:
|
||||
client = self._client()
|
||||
kwargs = {
|
||||
"model": self.model,
|
||||
"max_tokens": self.max_tokens,
|
||||
"timeout": timeout,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": ENRICHMENT_SYSTEM_PROMPT,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": scene_user_prompt(scene_json),
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"output_config": {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"schema": SEMANTIC_SCENE_SCHEMA,
|
||||
}
|
||||
},
|
||||
}
|
||||
messages = getattr(client, "messages", None)
|
||||
if messages is not None:
|
||||
return messages.create(**kwargs)
|
||||
return client.create(**kwargs)
|
||||
|
||||
def _client(self) -> Any:
|
||||
if self._transport is not None:
|
||||
return self._transport
|
||||
|
||||
try:
|
||||
import anthropic
|
||||
except Exception as exc:
|
||||
raise EnrichmentUnavailable("anthropic SDK is unavailable") from exc
|
||||
|
||||
self._transport = anthropic.Anthropic()
|
||||
return self._transport
|
||||
|
||||
|
||||
def _extract_response_body(response: Any) -> dict[str, Any]:
|
||||
if _looks_like_semantic_scene(response):
|
||||
return response
|
||||
|
||||
for key in ("output", "parsed", "json"):
|
||||
value = _value(response, key)
|
||||
if _looks_like_semantic_scene(value):
|
||||
return value
|
||||
|
||||
content = _value(response, "content")
|
||||
if _looks_like_semantic_scene(content):
|
||||
return content
|
||||
if isinstance(content, str):
|
||||
return _decode_json(content)
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
for key in ("parsed", "json", "input", "content"):
|
||||
value = _value(block, key)
|
||||
if _looks_like_semantic_scene(value):
|
||||
return value
|
||||
text = _value(block, "text")
|
||||
if isinstance(text, str):
|
||||
return _decode_json(text)
|
||||
|
||||
raise ValueError("structured semantic response body not found")
|
||||
|
||||
|
||||
def _decode_json(text: str) -> dict[str, Any]:
|
||||
decoded = json.loads(text)
|
||||
if not isinstance(decoded, dict):
|
||||
raise ValueError("structured semantic response must be a JSON object")
|
||||
return decoded
|
||||
|
||||
|
||||
def _validate_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return SemanticScene.from_dict(payload).to_dict()
|
||||
|
||||
|
||||
def _looks_like_semantic_scene(value: Any) -> bool:
|
||||
return isinstance(value, dict) and {
|
||||
"page",
|
||||
"intents",
|
||||
"widgets",
|
||||
}.issubset(value)
|
||||
|
||||
|
||||
def _value(source: Any, key: str) -> Any:
|
||||
if isinstance(source, dict):
|
||||
return source.get(key)
|
||||
value = getattr(source, key, None)
|
||||
return None if callable(value) else value
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
SEMANTIC_SCENE_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["page", "intents", "widgets"],
|
||||
"properties": {
|
||||
"page": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
},
|
||||
"intents": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
},
|
||||
},
|
||||
"widgets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["element_id", "purpose"],
|
||||
"properties": {
|
||||
"element_id": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
},
|
||||
"purpose": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticWidget:
|
||||
element_id: str
|
||||
purpose: str
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"element_id": self.element_id,
|
||||
"purpose": self.purpose,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SemanticWidget":
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError("semantic widget must be a mapping")
|
||||
element_id = data.get("element_id")
|
||||
purpose = data.get("purpose")
|
||||
if not isinstance(element_id, str) or not element_id.strip():
|
||||
raise ValueError("semantic widget element_id must be a non-empty string")
|
||||
if not isinstance(purpose, str) or not purpose.strip():
|
||||
raise ValueError("semantic widget purpose must be a non-empty string")
|
||||
return cls(element_id=element_id, purpose=purpose)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticScene:
|
||||
page: str
|
||||
intents: list[str] = field(default_factory=list)
|
||||
widgets: list[SemanticWidget] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"page": self.page,
|
||||
"intents": list(self.intents),
|
||||
"widgets": [widget.to_dict() for widget in self.widgets],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SemanticScene":
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError("semantic scene must be a mapping")
|
||||
page = data.get("page")
|
||||
intents = data.get("intents")
|
||||
widgets = data.get("widgets")
|
||||
if not isinstance(page, str) or not page.strip():
|
||||
raise ValueError("semantic scene page must be a non-empty string")
|
||||
if not isinstance(intents, list) or not all(
|
||||
isinstance(intent, str) and intent.strip() for intent in intents
|
||||
):
|
||||
raise ValueError("semantic scene intents must be non-empty strings")
|
||||
if not isinstance(widgets, list):
|
||||
raise ValueError("semantic scene widgets must be a list")
|
||||
return cls(
|
||||
page=page,
|
||||
intents=list(intents),
|
||||
widgets=[SemanticWidget.from_dict(widget) for widget in widgets],
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from semantic.models import SEMANTIC_SCENE_SCHEMA
|
||||
|
||||
ENRICHMENT_SYSTEM_PROMPT = f"""You enrich a mobile device Scene into a SemanticScene.
|
||||
|
||||
Identify the current page in a short page string, list plain-language intents the
|
||||
screen appears to support, and label the purpose of relevant widgets. Use only
|
||||
element_id values that appear in the input Scene's elements[].id list; never invent
|
||||
or rewrite element IDs. Prefer concise, stable labels over visual descriptions.
|
||||
|
||||
Return structured output matching this JSON schema exactly:
|
||||
{json.dumps(SEMANTIC_SCENE_SCHEMA, sort_keys=True)}
|
||||
"""
|
||||
|
||||
|
||||
def scene_user_prompt(scene_json: dict[str, Any]) -> str:
|
||||
return (
|
||||
"Enrich this Scene. Widget labels must reference only element IDs present "
|
||||
"in this JSON:\n"
|
||||
f"{json.dumps(scene_json, ensure_ascii=False, sort_keys=True)}"
|
||||
)
|
||||
Reference in New Issue
Block a user