feat: add skill learning runtime

2026-07-06 18:07:53 +08:00
parent 5658735bca
commit c7ae2d86ec
17 changed files with 1470 additions and 31 deletions
@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from skills_learning.config import SkillAuthoringConfig
+from skills_learning.embeddings import embed_skill_text
+from skills_learning.models import FlowStep, FlowTemplateSkill, SkillMetadata
+from skills_learning.retrieval import retrieve_candidate_skills
+from skills_learning.store import SkillStore
+
+
+class FakeEmbeddingClient:
+    def __init__(self, *, fail: bool = False) -> None:
+        self.fail = fail
+
+    def embed(self, text: str, *, model: str) -> list[float]:
+        if self.fail:
+            raise TimeoutError("embedding timeout")
+        lowered = text.lower()
+        if "coffee" in lowered:
+            return [1.0, 0.0]
+        if "tea" in lowered:
+            return [0.0, 1.0]
+        return [0.5, 0.5]
+
+
+def _skill(name: str, goal: str) -> FlowTemplateSkill:
+    return FlowTemplateSkill(
+        metadata=SkillMetadata(
+            name=name,
+            description=f"Learned flow for {goal}",
+            originating_goal=goal,
+        ),
+        steps=[FlowStep("input_text", {"text": goal})],
+        parameters={},
+    )
+
+
+def test_embed_skill_text_degrades_to_none_on_provider_failure() -> None:
+    result = embed_skill_text(
+        "coffee",
+        client=FakeEmbeddingClient(fail=True),
+        config=SkillAuthoringConfig(enabled=True),
+    )
+
+    assert result is None
+
+
+def test_retrieve_candidate_skills_ranks_by_similarity_and_truncates_top_k() -> None:
+    store = SkillStore()
+    coffee = store.create_version(_skill("search coffee", "search coffee"))
+    tea = store.create_version(_skill("search tea", "search tea"))
+    store.store_embedding(coffee, [1.0, 0.0], model_name="fake")
+    store.store_embedding(tea, [0.0, 1.0], model_name="fake")
+
+    results = retrieve_candidate_skills(
+        "find coffee",
+        store=store,
+        top_k=1,
+        embedding_client=FakeEmbeddingClient(),
+        config=SkillAuthoringConfig(enabled=True),
+    )
+
+    assert [result.skill.name for result in results] == ["search coffee"]
+
+
+def test_retrieve_candidate_skills_returns_empty_without_embeddings() -> None:
+    store = SkillStore()
+    store.create_version(_skill("search coffee", "search coffee"))
+
+    results = retrieve_candidate_skills(
+        "find coffee",
+        store=store,
+        embedding_client=FakeEmbeddingClient(),
+        config=SkillAuthoringConfig(enabled=True),
+    )
+
+    assert results == []
+
+
+def test_skill_without_embedding_is_stored_but_excluded_from_retrieval() -> None:
+    store = SkillStore()
+    skill = store.create_version(_skill("search coffee", "search coffee"))
+
+    results = retrieve_candidate_skills(
+        "find coffee",
+        store=store,
+        embedding_client=FakeEmbeddingClient(),
+        config=SkillAuthoringConfig(enabled=True),
+    )
+
+    assert store.get_by_id(skill.id) == skill
+    assert results == []
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import sys
+
+from skills_learning.models import (
+    LOCAL_SYNTHESIS_SOURCE,
+    FlowStep,
+    FlowTemplateSkill,
+    SkillMetadata,
+)
+from skills_learning.store import SkillStore
+
+
+def _skill(
+    *,
+    name: str = "search web",
+    steps: list[FlowStep] | None = None,
+    parameters: dict[str, dict[str, object]] | None = None,
+) -> FlowTemplateSkill:
+    return FlowTemplateSkill(
+        metadata=SkillMetadata(
+            name=name,
+            description=f"Learned flow for {name}",
+            source="external-source",
+            originating_goal=name,
+        ),
+        steps=steps or [FlowStep("input_text", {"text": "coffee"})],
+        parameters=parameters or {},
+    )
+
+
+def test_skill_store_enforces_local_synthesis_source_without_catalog_write_path() -> None:
+    store = SkillStore()
+
+    stored = store.create_version(_skill())
+
+    assert stored.source == LOCAL_SYNTHESIS_SOURCE
+    assert "skills.catalog" not in sys.modules
+
+
+def test_skill_store_keeps_version_chain_and_returns_latest_by_name() -> None:
+    store = SkillStore()
+    first = store.create_version(_skill())
+    second = store.create_version(
+        _skill(steps=[FlowStep("input_text", {"text": "tea"})]),
+        parent=first,
+    )
+
+    assert first.version == 1
+    assert second.version == 2
+    assert second.parent_version_id == first.id
+    assert store.get_latest_by_name("search web") == second
+    assert store.get_by_id(first.id) == first
+    assert [skill.version for skill in store.list_versions("search web")] == [1, 2]
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from skills_learning.models import FlowStep, FlowTemplateSkill, SkillMetadata
+from skills_learning.store import SkillStore
+from skills_learning.synthesis import extract_tool_calls, synthesize_flow_skill
+
+
+def _record(
+    action: str,
+    args: dict[str, object] | None = None,
+    *,
+    result: dict[str, object] | None = None,
+) -> dict[str, object]:
+    return {
+        "tool_call": {
+            "action": action,
+            "args": args or {},
+        },
+        "result": result or {},
+    }
+
+
+def test_extract_tool_calls_filters_read_only_tools_in_order() -> None:
+    records = [
+        _record("describe_screen"),
+        _record("launch_app", {"app_id": "com.example"}),
+        _record("screenshot"),
+        _record("tap", {"x": 1, "y": 2}),
+        _record("find_text", {"query": "Send"}),
+        _record("input_text", {"text": "coffee"}),
+    ]
+
+    steps = extract_tool_calls("", records)
+
+    assert steps == [
+        FlowStep("launch_app", {"app_id": "com.example"}),
+        FlowStep("tap", {"x": 1, "y": 2}),
+        FlowStep("input_text", {"text": "coffee"}),
+    ]
+
+
+def test_first_time_synthesis_has_literal_steps_and_no_parameters() -> None:
+    skill = synthesize_flow_skill(
+        "search coffee",
+        [_record("input_text", {"text": "coffee"})],
+    )
+
+    assert skill.name == "search coffee"
+    assert skill.steps == [FlowStep("input_text", {"text": "coffee"})]
+    assert skill.parameters == {}
+
+
+def test_second_execution_promotes_differing_argument_to_parameter() -> None:
+    store = SkillStore()
+    store.create_version(
+        FlowTemplateSkill(
+            metadata=SkillMetadata(
+                name="search",
+                description="Learned search",
+                originating_goal="search coffee",
+            ),
+            steps=[FlowStep("input_text", {"text": "coffee"})],
+            parameters={},
+        )
+    )
+    records = [
+        _record(
+            "input_text",
+            {"text": "tea"},
+            result={
+                "semantic_scene": {
+                    "page": "Search",
+                    "intents": ["search"],
+                    "widgets": [
+                        {
+                            "element_id": "search-input",
+                            "purpose": "search field",
+                        }
+                    ],
+                }
+            },
+        )
+    ]
+
+    skill = synthesize_flow_skill("search tea", records, store=store)
+
+    assert skill.name == "search"
+    assert skill.steps == [FlowStep("input_text", {"text": "{search_field}"})]
+    assert "search_field" in skill.parameters
+
+
+def test_identical_repeat_does_not_add_parameters() -> None:
+    store = SkillStore()
+    store.create_version(
+        FlowTemplateSkill(
+            metadata=SkillMetadata(
+                name="search",
+                description="Learned search",
+                originating_goal="search coffee",
+            ),
+            steps=[FlowStep("input_text", {"text": "coffee"})],
+            parameters={},
+        )
+    )
+
+    skill = synthesize_flow_skill(
+        "search coffee again",
+        [_record("input_text", {"text": "coffee"})],
+        store=store,
+    )
+
+    assert skill.steps == [FlowStep("input_text", {"text": "coffee"})]
+    assert skill.parameters == {}
@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+from core.models import Bounds, Scene, SceneElement, Task
+from runtime.executor import Executor, ExecutorConfig
+from runtime.planner import PlannedStep, Planner
+from runtime.task import TaskRunner, TaskRunnerConfig
+from skills_learning.config import SkillAuthoringConfig
+from skills_learning.retrieval import retrieve_candidate_skills
+from skills_learning.store import SkillStore
+from storage.artifact_store import ArtifactStore
+from storage.timeline import Timeline
+from tests.fakes import PNG_10X20
+
+
+class ScriptedPlanner(Planner):
+    def __init__(self, steps: list[PlannedStep]) -> None:
+        self.steps = steps
+
+    def plan(self, *, goal, scene, context):
+        if len(context.step_results) >= len(self.steps):
+            return []
+        return [self.steps[len(context.step_results)]]
+
+    def goal_reached(self, *, goal, scene, context):
+        return len(context.step_results) >= len(self.steps) and all(
+            result.success for result in context.step_results
+        )
+
+
+class FakeEmbeddingClient:
+    def __init__(self, *, fail: bool = False) -> None:
+        self.fail = fail
+
+    def embed(self, text: str, *, model: str) -> list[float]:
+        if self.fail:
+            raise TimeoutError("embedding timeout")
+        lowered = text.lower()
+        if "coffee" in lowered:
+            return [1.0, 0.0]
+        if "tea" in lowered:
+            return [0.0, 1.0]
+        return [0.5, 0.5]
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="input",
+                type="input",
+                text="Search",
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def _runner(
+    *,
+    tmp_path,
+    planner: Planner,
+    timeline: Timeline | None = None,
+    store: SkillStore | None = None,
+    skill_config: SkillAuthoringConfig | None = None,
+    embedding_client: FakeEmbeddingClient | None = None,
+    on_task_succeeded=None,
+) -> TaskRunner:
+    return TaskRunner(
+        planner=planner,
+        executor=Executor(
+            tools={"input_text": lambda **kwargs: {"ok": True, **kwargs}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        timeline=timeline or Timeline(ArtifactStore(tmp_path / "history")),
+        config=TaskRunnerConfig(max_steps=5),
+        observer=lambda device_id: _scene(),
+        screenshot_provider=lambda device_id: PNG_10X20,
+        skill_store=store,
+        skill_authoring_config=skill_config,
+        skill_embedding_client=embedding_client,
+        on_task_succeeded=on_task_succeeded,
+    )
+
+
+def test_task_runner_calls_explicit_success_hook_once(tmp_path) -> None:
+    calls: list[tuple[str, str, Timeline]] = []
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    runner = _runner(
+        tmp_path=tmp_path,
+        timeline=timeline,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type", {"text": "coffee"})]
+        ),
+        on_task_succeeded=lambda task_id, goal, timeline: calls.append(
+            (task_id, goal, timeline)
+        ),
+    )
+    task = Task(goal="search coffee", device_id="phone")
+
+    result = runner.run(task)
+
+    assert result.status == "completed"
+    assert calls == [(task.id, "search coffee", timeline)]
+
+
+def test_task_runner_skill_authoring_disabled_by_default_writes_no_skill(tmp_path) -> None:
+    store = SkillStore()
+    runner = _runner(
+        tmp_path=tmp_path,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type", {"text": "coffee"})]
+        ),
+        store=store,
+    )
+
+    result = runner.run(Task(goal="search coffee", device_id="phone"))
+
+    assert result.status == "completed"
+    assert store.list_all() == []
+
+
+def test_task_runner_skill_authoring_enabled_stores_skill_and_embedding(tmp_path) -> None:
+    store = SkillStore()
+    runner = _runner(
+        tmp_path=tmp_path,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type", {"text": "coffee"})]
+        ),
+        store=store,
+        skill_config=SkillAuthoringConfig(enabled=True, embedding_model="fake"),
+        embedding_client=FakeEmbeddingClient(),
+    )
+
+    result = runner.run(Task(goal="search coffee", device_id="phone"))
+
+    assert result.status == "completed"
+    skill = store.get_latest_by_name("search coffee")
+    assert skill is not None
+    assert skill.steps[0].args == {"text": "coffee"}
+    assert store.get_embedding(skill.id, skill.version) is not None
+
+
+def test_task_runner_embedding_failure_still_stores_skill_without_embedding(tmp_path) -> None:
+    store = SkillStore()
+    runner = _runner(
+        tmp_path=tmp_path,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type", {"text": "coffee"})]
+        ),
+        store=store,
+        skill_config=SkillAuthoringConfig(enabled=True, embedding_model="fake"),
+        embedding_client=FakeEmbeddingClient(fail=True),
+    )
+
+    result = runner.run(Task(goal="search coffee", device_id="phone"))
+
+    assert result.status == "completed"
+    skill = store.get_latest_by_name("search coffee")
+    assert skill is not None
+    assert store.get_embedding(skill.id, skill.version) is None
+
+
+def test_two_successful_tasks_promote_parameter_and_store_embedding(tmp_path) -> None:
+    store = SkillStore()
+    config = SkillAuthoringConfig(enabled=True, embedding_model="fake")
+
+    _runner(
+        tmp_path=tmp_path,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type coffee", {"text": "coffee"})]
+        ),
+        store=store,
+        skill_config=config,
+        embedding_client=FakeEmbeddingClient(),
+    ).run(Task(goal="search coffee", device_id="phone"))
+
+    _runner(
+        tmp_path=tmp_path,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type tea", {"text": "tea"})]
+        ),
+        store=store,
+        skill_config=config,
+        embedding_client=FakeEmbeddingClient(),
+    ).run(Task(goal="search tea", device_id="phone"))
+
+    skill = store.get_latest_by_name("search coffee")
+    assert skill is not None
+    assert skill.version == 1
+    assert skill.steps[0].args == {"text": "{param_1}"}
+    assert "param_1" in skill.parameters
+    assert store.get_embedding(skill.id, skill.version) is not None
+
+
+def test_retrieve_candidate_skills_after_successful_task(tmp_path) -> None:
+    store = SkillStore()
+    config = SkillAuthoringConfig(enabled=True, embedding_model="fake")
+    _runner(
+        tmp_path=tmp_path,
+        planner=ScriptedPlanner(
+            [PlannedStep("input_text", "type coffee", {"text": "coffee"})]
+        ),
+        store=store,
+        skill_config=config,
+        embedding_client=FakeEmbeddingClient(),
+    ).run(Task(goal="search coffee", device_id="phone"))
+
+    results = retrieve_candidate_skills(
+        "find coffee",
+        store=store,
+        embedding_client=FakeEmbeddingClient(),
+        config=config,
+    )
+
+    assert [result.skill.name for result in results] == ["search coffee"]
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from skills_learning.models import FlowStep, FlowTemplateSkill, SkillMetadata
+from skills_learning.store import SkillStore
+from skills_learning.versioning import diff_flow_versions, store_synthesized_skill
+
+
+def _skill(
+    *,
+    name: str = "search",
+    steps: list[FlowStep] | None = None,
+    parameters: dict[str, dict[str, object]] | None = None,
+) -> FlowTemplateSkill:
+    return FlowTemplateSkill(
+        metadata=SkillMetadata(
+            name=name,
+            description="Learned search",
+            originating_goal="search coffee",
+        ),
+        steps=steps or [FlowStep("input_text", {"text": "coffee"})],
+        parameters=parameters or {},
+    )
+
+
+def test_diff_flow_versions_detects_extra_missing_and_reordered_steps() -> None:
+    stored = [FlowStep("tap"), FlowStep("input_text")]
+
+    assert diff_flow_versions(stored, [FlowStep("tap")]).structural_divergence
+    assert diff_flow_versions(
+        stored,
+        [FlowStep("tap"), FlowStep("input_text"), FlowStep("tap")],
+    ).structural_divergence
+    assert diff_flow_versions(
+        stored,
+        [FlowStep("input_text"), FlowStep("tap")],
+    ).structural_divergence
+
+
+def test_argument_only_difference_updates_existing_version_without_bump() -> None:
+    store = SkillStore()
+    first = store.create_version(_skill())
+    candidate = _skill(
+        steps=[FlowStep("input_text", {"text": "{search_query}"})],
+        parameters={"search_query": {"type": "string"}},
+    )
+
+    result = store_synthesized_skill(store, candidate)
+
+    assert result.created_new_version is False
+    assert result.skill.version == first.version
+    assert result.skill.id == first.id
+    assert result.skill.parameters == {"search_query": {"type": "string"}}
+    assert store.get_latest_by_name("search") == result.skill
+
+
+def test_structural_divergence_creates_new_version_and_preserves_parent() -> None:
+    store = SkillStore()
+    first = store.create_version(_skill())
+    candidate = _skill(
+        steps=[
+            FlowStep("tap", {"x": 1}),
+            FlowStep("input_text", {"text": "coffee"}),
+        ]
+    )
+
+    result = store_synthesized_skill(store, candidate)
+
+    assert result.created_new_version is True
+    assert result.skill.version == 2
+    assert result.skill.parent_version_id == first.id
+    assert store.get_by_id(first.id) == first
+    assert store.get_latest_by_name("search") == result.skill
@@ -12,6 +12,7 @@ def test_imports_new_packages() -> None:
        "perception",
        "runtime",
        "semantic",
+        "skills_learning",
        "storage",
        "tools",
        "world",