feat: add skill learning runtime

This commit is contained in:
2026-07-06 18:07:53 +08:00
parent 5658735bca
commit c7ae2d86ec
17 changed files with 1470 additions and 31 deletions
+16
View File
@@ -0,0 +1,16 @@
"""Local skill learning from completed task timelines."""
from skills_learning.config import SkillAuthoringConfig, load_config
from skills_learning.models import FlowStep, FlowTemplateSkill, Skill, SkillMetadata
from skills_learning.store import SkillStore, get_default_store
__all__ = [
"FlowStep",
"FlowTemplateSkill",
"Skill",
"SkillAuthoringConfig",
"SkillMetadata",
"SkillStore",
"get_default_store",
"load_config",
]
+61
View File
@@ -0,0 +1,61 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from dataclasses import dataclass
DEFAULT_DIVERGENCE_TOLERANCE = 0.0
DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
DEFAULT_TOP_K = 5
ENABLED_ENV = "SKILL_AUTHORING_ENABLED"
DIVERGENCE_TOLERANCE_ENV = "SKILL_DIVERGENCE_TOLERANCE"
EMBEDDING_MODEL_ENV = "SKILL_EMBEDDING_MODEL"
TOP_K_ENV = "SKILL_RETRIEVAL_TOP_K"
@dataclass(frozen=True)
class SkillAuthoringConfig:
enabled: bool = False
divergence_tolerance: float = DEFAULT_DIVERGENCE_TOLERANCE
embedding_model: str = DEFAULT_EMBEDDING_MODEL
top_k: int = DEFAULT_TOP_K
def load_config(env: Mapping[str, str] | None = None) -> SkillAuthoringConfig:
values = env or os.environ
return SkillAuthoringConfig(
enabled=_parse_bool(values.get(ENABLED_ENV), default=False),
divergence_tolerance=_parse_float(
values.get(DIVERGENCE_TOLERANCE_ENV),
default=DEFAULT_DIVERGENCE_TOLERANCE,
),
embedding_model=values.get(EMBEDDING_MODEL_ENV) or DEFAULT_EMBEDDING_MODEL,
top_k=_parse_int(values.get(TOP_K_ENV), default=DEFAULT_TOP_K),
)
def _parse_bool(value: str | None, *, default: bool) -> bool:
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on", "enabled"}
def _parse_float(value: str | None, *, default: float) -> float:
if value is None:
return default
try:
parsed = float(value)
except ValueError:
return default
return parsed if parsed >= 0 else default
def _parse_int(value: str | None, *, default: int) -> int:
if value is None:
return default
try:
parsed = int(value)
except ValueError:
return default
return parsed if parsed > 0 else default
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from typing import Any, Protocol
from skills_learning.config import SkillAuthoringConfig, load_config
class EmbeddingClient(Protocol):
def embed(self, text: str, *, model: str) -> list[float]:
...
class OpenAIEmbeddingClient:
def __init__(self, *, transport: Any | None = None) -> None:
self._transport = transport
def embed(self, text: str, *, model: str) -> list[float]:
client = self._client()
response = client.embeddings.create(model=model, input=text)
return [float(value) for value in response.data[0].embedding]
def _client(self) -> Any:
if self._transport is not None:
return self._transport
from openai import OpenAI
self._transport = OpenAI()
return self._transport
def embed_skill_text(
text: str,
*,
client: EmbeddingClient | None = None,
config: SkillAuthoringConfig | None = None,
) -> list[float] | None:
settings = config or load_config()
if not settings.enabled:
return None
try:
embedding_client = client or OpenAIEmbeddingClient()
return embedding_client.embed(text, model=settings.embedding_model)
except Exception:
return None
+186
View File
@@ -0,0 +1,186 @@
from __future__ import annotations
from dataclasses import dataclass, field, replace
from datetime import datetime
from typing import Any, Literal
from uuid import uuid4
from core.models import utc_now
LOCAL_SYNTHESIS_SOURCE = "local-synthesis"
SkillKind = Literal["knowledge", "flow_template"]
@dataclass(frozen=True)
class SkillMetadata:
id: str = field(default_factory=lambda: uuid4().hex)
name: str = ""
description: str = ""
kind: SkillKind = "flow_template"
tags: list[str] = field(default_factory=list)
source: str = LOCAL_SYNTHESIS_SOURCE
version: int = 1
parent_version_id: str | None = None
originating_goal: str | None = None
created_at: datetime = field(default_factory=utc_now)
updated_at: datetime = field(default_factory=utc_now)
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"name": self.name,
"description": self.description,
"kind": self.kind,
"tags": list(self.tags),
"source": self.source,
"version": self.version,
"parent_version_id": self.parent_version_id,
"originating_goal": self.originating_goal,
"created_at": self.created_at.isoformat(),
"updated_at": self.updated_at.isoformat(),
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "SkillMetadata":
return cls(
id=str(data.get("id") or uuid4().hex),
name=str(data.get("name") or ""),
description=str(data.get("description") or ""),
kind=data.get("kind") or "flow_template",
tags=[str(tag) for tag in data.get("tags", [])],
source=str(data.get("source") or LOCAL_SYNTHESIS_SOURCE),
version=int(data.get("version") or 1),
parent_version_id=data.get("parent_version_id"),
originating_goal=data.get("originating_goal"),
created_at=_parse_datetime(data.get("created_at")),
updated_at=_parse_datetime(data.get("updated_at")),
)
@dataclass(frozen=True)
class FlowStep:
tool_name: str
args: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"tool_name": self.tool_name,
"args": dict(self.args),
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "FlowStep":
return cls(
tool_name=str(data.get("tool_name") or data.get("action") or ""),
args=dict(data.get("args") or {}),
)
@dataclass(frozen=True)
class Skill:
metadata: SkillMetadata
@property
def id(self) -> str:
return self.metadata.id
@property
def name(self) -> str:
return self.metadata.name
@property
def description(self) -> str:
return self.metadata.description
@property
def source(self) -> str:
return self.metadata.source
@property
def version(self) -> int:
return self.metadata.version
@property
def parent_version_id(self) -> str | None:
return self.metadata.parent_version_id
@property
def originating_goal(self) -> str | None:
return self.metadata.originating_goal
@dataclass(frozen=True)
class FlowTemplateSkill(Skill):
steps: list[FlowStep] = field(default_factory=list)
parameters: dict[str, dict[str, Any]] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
**self.metadata.to_dict(),
"steps": [step.to_dict() for step in self.steps],
"parameters": {
name: dict(schema)
for name, schema in self.parameters.items()
},
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "FlowTemplateSkill":
return cls(
metadata=SkillMetadata.from_dict(data),
steps=[
FlowStep.from_dict(step)
for step in data.get("steps", [])
],
parameters={
str(name): dict(schema)
for name, schema in (data.get("parameters") or {}).items()
},
)
def with_metadata(self, **changes: Any) -> "FlowTemplateSkill":
return replace(self, metadata=replace(self.metadata, **changes))
def with_updates(
self,
*,
steps: list[FlowStep] | None = None,
parameters: dict[str, dict[str, Any]] | None = None,
**metadata_changes: Any,
) -> "FlowTemplateSkill":
metadata = replace(
self.metadata,
updated_at=utc_now(),
**metadata_changes,
)
return replace(
self,
metadata=metadata,
steps=list(steps) if steps is not None else list(self.steps),
parameters={
name: dict(schema)
for name, schema in (
parameters if parameters is not None else self.parameters
).items()
},
)
def skill_embedding_text(skill: FlowTemplateSkill) -> str:
goal = skill.originating_goal or ""
return f"{skill.name}: {skill.description}\nOriginal goal: {goal}"
def clone_skill(skill: FlowTemplateSkill) -> FlowTemplateSkill:
return FlowTemplateSkill.from_dict(skill.to_dict())
def _parse_datetime(value: Any) -> datetime:
if isinstance(value, datetime):
return value
if isinstance(value, str):
try:
return datetime.fromisoformat(value)
except ValueError:
pass
return utc_now()
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
import math
from dataclasses import dataclass
from skills_learning.config import SkillAuthoringConfig, load_config
from skills_learning.embeddings import EmbeddingClient, embed_skill_text
from skills_learning.models import FlowTemplateSkill
from skills_learning.store import SkillStore, get_default_store
@dataclass(frozen=True)
class ScoredSkill:
skill: FlowTemplateSkill
score: float
def retrieve_candidate_skills(
goal: str,
*,
store: SkillStore | None = None,
top_k: int | None = None,
embedding_client: EmbeddingClient | None = None,
config: SkillAuthoringConfig | None = None,
) -> list[ScoredSkill]:
settings = config or load_config()
skill_store = store or get_default_store()
query_vector = embed_skill_text(
goal,
client=embedding_client,
config=settings,
)
if query_vector is None:
return []
scored: list[ScoredSkill] = []
for record in skill_store.list_embeddings():
skill = skill_store.get_by_id(record.skill_id)
if skill is None:
continue
score = _cosine_similarity(query_vector, record.vector)
scored.append(ScoredSkill(skill=skill, score=score))
scored.sort(key=lambda item: item.score, reverse=True)
return scored[: top_k or settings.top_k]
def _cosine_similarity(left: list[float], right: list[float]) -> float:
if not left or not right or len(left) != len(right):
return 0.0
dot = sum(a * b for a, b in zip(left, right, strict=True))
left_norm = math.sqrt(sum(value * value for value in left))
right_norm = math.sqrt(sum(value * value for value in right))
if left_norm == 0 or right_norm == 0:
return 0.0
return dot / (left_norm * right_norm)
+141
View File
@@ -0,0 +1,141 @@
from __future__ import annotations
from dataclasses import dataclass, field, replace
from datetime import datetime
from typing import Any
from uuid import uuid4
from core.models import utc_now
from skills_learning.models import (
LOCAL_SYNTHESIS_SOURCE,
FlowTemplateSkill,
clone_skill,
)
@dataclass(frozen=True)
class SkillEmbeddingRecord:
skill_id: str
version: int
vector: list[float]
model_name: str
updated_at: datetime = field(default_factory=utc_now)
def to_dict(self) -> dict[str, Any]:
return {
"skill_id": self.skill_id,
"version": self.version,
"vector": list(self.vector),
"model_name": self.model_name,
"updated_at": self.updated_at.isoformat(),
}
class SkillStore:
def __init__(self) -> None:
self._skills: dict[str, FlowTemplateSkill] = {}
self._embeddings: dict[tuple[str, int], SkillEmbeddingRecord] = {}
def create_version(
self,
skill: FlowTemplateSkill,
*,
parent: FlowTemplateSkill | None = None,
) -> FlowTemplateSkill:
version = parent.version + 1 if parent else self._next_version(skill.name)
stored = skill.with_metadata(
id=uuid4().hex,
source=LOCAL_SYNTHESIS_SOURCE,
version=version,
parent_version_id=parent.id if parent else skill.parent_version_id,
created_at=utc_now(),
updated_at=utc_now(),
)
self._skills[stored.id] = clone_skill(stored)
return clone_skill(stored)
def update_skill(self, skill: FlowTemplateSkill) -> FlowTemplateSkill:
if skill.id not in self._skills:
raise KeyError(f"unknown skill {skill.id}")
stored = skill.with_metadata(
source=LOCAL_SYNTHESIS_SOURCE,
updated_at=utc_now(),
)
self._skills[stored.id] = clone_skill(stored)
return clone_skill(stored)
def get_by_id(self, skill_id: str) -> FlowTemplateSkill | None:
skill = self._skills.get(skill_id)
return clone_skill(skill) if skill else None
def get_latest_by_name(self, name: str) -> FlowTemplateSkill | None:
versions = self.list_versions(name)
return versions[-1] if versions else None
def list_versions(self, name: str) -> list[FlowTemplateSkill]:
return sorted(
[
clone_skill(skill)
for skill in self._skills.values()
if skill.name == name
],
key=lambda skill: skill.version,
)
def list_latest(self) -> list[FlowTemplateSkill]:
latest: dict[str, FlowTemplateSkill] = {}
for skill in self._skills.values():
current = latest.get(skill.name)
if current is None or skill.version > current.version:
latest[skill.name] = skill
return [clone_skill(skill) for skill in latest.values()]
def list_all(self) -> list[FlowTemplateSkill]:
return [clone_skill(skill) for skill in self._skills.values()]
def store_embedding(
self,
skill: FlowTemplateSkill,
vector: list[float],
*,
model_name: str,
) -> SkillEmbeddingRecord:
record = SkillEmbeddingRecord(
skill_id=skill.id,
version=skill.version,
vector=[float(value) for value in vector],
model_name=model_name,
)
self._embeddings[(record.skill_id, record.version)] = record
return replace(record, vector=list(record.vector))
def get_embedding(
self,
skill_id: str,
version: int,
) -> SkillEmbeddingRecord | None:
record = self._embeddings.get((skill_id, version))
return replace(record, vector=list(record.vector)) if record else None
def list_embeddings(self) -> list[SkillEmbeddingRecord]:
return [
replace(record, vector=list(record.vector))
for record in self._embeddings.values()
]
def _next_version(self, name: str) -> int:
latest = self.get_latest_by_name(name)
return latest.version + 1 if latest else 1
_DEFAULT_STORE = SkillStore()
def get_default_store() -> SkillStore:
return _DEFAULT_STORE
def reset_default_store() -> SkillStore:
global _DEFAULT_STORE
_DEFAULT_STORE = SkillStore()
return _DEFAULT_STORE
+248
View File
@@ -0,0 +1,248 @@
from __future__ import annotations
import re
from collections.abc import Iterable
from typing import Any
from uuid import uuid4
from skills_learning.models import (
LOCAL_SYNTHESIS_SOURCE,
FlowStep,
FlowTemplateSkill,
SkillMetadata,
)
from skills_learning.store import SkillStore
READ_ONLY_TOOL_NAMES = {
"describe_screen",
"describe_screen_semantic",
"screenshot",
"take_screenshot",
"ui_tree",
"get_ui_tree",
"find_text",
"find_text_on_screen",
"find_icon",
"find_icon_on_screen",
}
def extract_tool_calls(
task_id: str,
timeline: Any,
) -> list[FlowStep]:
records = _timeline_records(timeline, task_id)
steps: list[FlowStep] = []
for record in records:
tool_call = _record_value(record, "tool_call")
if not isinstance(tool_call, dict):
continue
tool_name = str(tool_call.get("action") or tool_call.get("tool_name") or "")
if not tool_name or tool_name in READ_ONLY_TOOL_NAMES:
continue
steps.append(FlowStep(tool_name=tool_name, args=dict(tool_call.get("args") or {})))
return steps
def find_matching_skeleton(
steps: list[FlowStep],
store: SkillStore | None,
) -> FlowTemplateSkill | None:
if store is None:
return None
sequence = _tool_sequence(steps)
for skill in store.list_latest():
if _tool_sequence(skill.steps) == sequence:
return skill
return None
def synthesize_flow_skill(
goal: str,
timeline: Any,
*,
task_id: str = "",
store: SkillStore | None = None,
) -> FlowTemplateSkill:
records = _timeline_records(timeline, task_id)
executed_steps = extract_tool_calls(task_id, records)
matched = find_matching_skeleton(executed_steps, store)
if matched is None:
steps = executed_steps
parameters: dict[str, dict[str, Any]] = {}
name = _skill_name_from_goal(goal)
else:
steps, parameters = promote_parameters(
matched.steps,
executed_steps,
records=records,
existing_parameters=matched.parameters,
)
name = matched.name
return FlowTemplateSkill(
metadata=SkillMetadata(
id=uuid4().hex,
name=name,
description=f"Learned flow for: {goal}",
kind="flow_template",
source=LOCAL_SYNTHESIS_SOURCE,
version=matched.version if matched else 1,
parent_version_id=matched.parent_version_id if matched else None,
originating_goal=goal,
),
steps=steps,
parameters=parameters,
)
def promote_parameters(
stored_steps: list[FlowStep],
executed_steps: list[FlowStep],
*,
records: Iterable[Any] = (),
existing_parameters: dict[str, dict[str, Any]] | None = None,
) -> tuple[list[FlowStep], dict[str, dict[str, Any]]]:
parameters = {
name: dict(schema)
for name, schema in (existing_parameters or {}).items()
}
parameterized_steps = [
FlowStep(step.tool_name, dict(step.args))
for step in executed_steps
]
used_names = set(parameters)
parameter_index = len(used_names) + 1
for step_index, (stored_step, executed_step) in enumerate(
zip(stored_steps, executed_steps, strict=False)
):
for arg_name, executed_value in executed_step.args.items():
stored_value = stored_step.args.get(arg_name)
if stored_value == executed_value:
continue
if _is_placeholder(stored_value):
parameterized_steps[step_index].args[arg_name] = stored_value
continue
if _is_placeholder(executed_value):
continue
preferred_name = _parameter_name_from_semantic_record(
list(records),
step_index,
)
parameter_name = _unique_parameter_name(
preferred_name or f"param_{parameter_index}",
used_names,
)
used_names.add(parameter_name)
parameter_index += 1
parameterized_steps[step_index].args[arg_name] = f"{{{parameter_name}}}"
parameters[parameter_name] = _parameter_schema(
arg_name,
executed_value,
preferred_name is not None,
)
return parameterized_steps, parameters
def _timeline_records(timeline: Any, task_id: str) -> list[Any]:
if isinstance(timeline, list):
return list(timeline)
if hasattr(timeline, "read"):
return list(timeline.read(task_id))
return list(timeline)
def _record_value(record: Any, key: str) -> Any:
if isinstance(record, dict):
return record.get(key)
return getattr(record, key, None)
def _tool_sequence(steps: list[FlowStep]) -> list[str]:
return [step.tool_name for step in steps]
def _is_placeholder(value: Any) -> bool:
return isinstance(value, str) and value.startswith("{") and value.endswith("}")
def _parameter_schema(
arg_name: str,
value: Any,
from_semantic_label: bool,
) -> dict[str, Any]:
return {
"type": _json_type(value),
"description": (
f"Value for {arg_name} inferred from a semantic widget label"
if from_semantic_label
else f"Value for {arg_name}"
),
}
def _json_type(value: Any) -> str:
if isinstance(value, bool):
return "boolean"
if isinstance(value, int | float):
return "number"
if isinstance(value, list):
return "array"
if isinstance(value, dict):
return "object"
return "string"
def _parameter_name_from_semantic_record(
records: list[Any],
step_index: int,
) -> str | None:
if step_index >= len(records):
return None
result = _record_value(records[step_index], "result")
semantic_scene = _semantic_scene_payload(result)
if not isinstance(semantic_scene, dict):
return None
widgets = semantic_scene.get("widgets")
if not isinstance(widgets, list):
return None
for widget in widgets:
if isinstance(widget, dict) and widget.get("purpose"):
return _sanitize_name(str(widget["purpose"]))
return None
def _semantic_scene_payload(result: Any) -> Any:
if not isinstance(result, dict):
return None
if "semantic_scene" in result:
return result["semantic_scene"]
nested = result.get("result")
if isinstance(nested, dict):
return nested.get("semantic_scene")
return None
def _unique_parameter_name(name: str, used_names: set[str]) -> str:
candidate = _sanitize_name(name) or "param"
if candidate not in used_names:
return candidate
index = 2
while f"{candidate}_{index}" in used_names:
index += 1
return f"{candidate}_{index}"
def _sanitize_name(value: str) -> str:
sanitized = re.sub(r"[^0-9a-zA-Z]+", "_", value.strip().lower()).strip("_")
if sanitized and sanitized[0].isdigit():
return f"param_{sanitized}"
return sanitized
def _skill_name_from_goal(goal: str) -> str:
return goal.strip() or "learned flow"
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from skills_learning.models import FlowStep, FlowTemplateSkill
from skills_learning.store import SkillStore
@dataclass(frozen=True)
class VersionDiff:
structural_divergence: bool
stored_sequence: list[str]
executed_sequence: list[str]
argument_differences: list[tuple[int, str, Any, Any]]
@dataclass(frozen=True)
class VersioningResult:
skill: FlowTemplateSkill
created_new_version: bool
def diff_flow_versions(
stored_steps: list[FlowStep],
executed_steps: list[FlowStep],
) -> VersionDiff:
stored_sequence = [step.tool_name for step in stored_steps]
executed_sequence = [step.tool_name for step in executed_steps]
structural = stored_sequence != executed_sequence
differences: list[tuple[int, str, Any, Any]] = []
if not structural:
for index, (stored_step, executed_step) in enumerate(
zip(stored_steps, executed_steps, strict=True)
):
keys = set(stored_step.args) | set(executed_step.args)
for key in sorted(keys):
stored_value = stored_step.args.get(key)
executed_value = executed_step.args.get(key)
if stored_value != executed_value:
differences.append((index, key, stored_value, executed_value))
return VersionDiff(
structural_divergence=structural,
stored_sequence=stored_sequence,
executed_sequence=executed_sequence,
argument_differences=differences,
)
def store_synthesized_skill(
store: SkillStore,
candidate: FlowTemplateSkill,
) -> VersioningResult:
latest = store.get_latest_by_name(candidate.name)
if latest is None:
return VersioningResult(store.create_version(candidate), True)
diff = diff_flow_versions(latest.steps, candidate.steps)
if diff.structural_divergence:
return VersioningResult(store.create_version(candidate, parent=latest), True)
merged_parameters = {
**latest.parameters,
**candidate.parameters,
}
updated = latest.with_updates(
steps=candidate.steps,
parameters=merged_parameters,
description=candidate.description,
originating_goal=candidate.originating_goal,
)
return VersioningResult(store.update_skill(updated), False)