feat(skill-catalog-subscription): synced catalog + HTTP sync + MCP tools
Consumes the external Subscription Platform as source of truth for skill content; reuses skills_learning domain models (extended with KnowledgeSkill) and workflow.skill_exec resolver. HTTP/MCP deps land in api/ per CONSTITUTION.md; synced skills use a physically separate SQLite file (tasks/skills.sqlite3) to preserve the skill-authoring capability boundary. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -0,0 +1,432 @@
|
||||
"""Tests for storage/skill_catalog.py.
|
||||
|
||||
Covers task 2.7: visibility filtering, not-found indistinguishability,
|
||||
dangling-tool-reference invalidation, stable sort, round-trip serialization
|
||||
for both knowledge and flow_template kinds.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from core.models import utc_now
|
||||
from skills_learning.models import (
|
||||
FlowStep,
|
||||
FlowTemplateSkill,
|
||||
KnowledgeSkill,
|
||||
SkillMetadata,
|
||||
)
|
||||
from storage.skill_catalog import (
|
||||
SkillCatalogStore,
|
||||
validate_flow_template_tools,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return SkillCatalogStore(db_path=tmp_path / "skills.sqlite3")
|
||||
|
||||
|
||||
def _knowledge(skill_id: str, name: str, content: str, *, tags=None) -> KnowledgeSkill:
|
||||
return KnowledgeSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="knowledge",
|
||||
tags=tags or [],
|
||||
source="subscription",
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def _flow(
|
||||
skill_id: str,
|
||||
name: str,
|
||||
*,
|
||||
steps: list[tuple[str, dict]],
|
||||
parameters: dict | None = None,
|
||||
tags=None,
|
||||
) -> FlowTemplateSkill:
|
||||
return FlowTemplateSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="flow_template",
|
||||
tags=tags or [],
|
||||
source="subscription",
|
||||
),
|
||||
steps=[FlowStep(tool_name=t, args=a) for t, a in steps],
|
||||
parameters=parameters or {},
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 2.1: schema + sync-only write contract
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_store_creates_db_file(tmp_path):
|
||||
db = tmp_path / "nested" / "skills.sqlite3"
|
||||
SkillCatalogStore(db_path=db)
|
||||
assert db.exists()
|
||||
|
||||
|
||||
def test_register_subscription_is_idempotent(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._register_subscription("sub-a")
|
||||
# No error, single row, active by default
|
||||
assert store._effective_active({"sub-a"}) == {"sub-a"}
|
||||
|
||||
|
||||
def test_apply_sync_replace_all_clears_prior_skills(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_replace_all(
|
||||
"sub-a",
|
||||
[_knowledge("k1", "keeps", "c"), _flow("f1", "flowy", steps=[("tap", {})])],
|
||||
latest_version=5,
|
||||
)
|
||||
assert len(store.list_skills({"sub-a"})) == 2
|
||||
# Replace with a single skill → previous two must be gone
|
||||
store._apply_sync_replace_all("sub-a", [_knowledge("k2", "new", "c")])
|
||||
ids = {m.id for m in store.list_skills({"sub-a"})}
|
||||
assert ids == {"k2"}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 2.2 / 2.3 / 2.4: visibility filtering
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _seed_two_subscriptions(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._register_subscription("sub-b")
|
||||
store._apply_sync_replace_all(
|
||||
"sub-a",
|
||||
[
|
||||
_knowledge("k-a1", "Alpha Knowledge", "c-a1", tags=["alpha"]),
|
||||
_flow(
|
||||
"f-a2",
|
||||
"Alpha Flow",
|
||||
steps=[("tap", {"x": 1})],
|
||||
tags=["alpha"],
|
||||
),
|
||||
],
|
||||
)
|
||||
store._apply_sync_replace_all(
|
||||
"sub-b",
|
||||
[
|
||||
_knowledge("k-b1", "Beta Knowledge", "c-b1", tags=["beta"]),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_list_skills_filters_by_active_subscription(store):
|
||||
_seed_two_subscriptions(store)
|
||||
a = {m.id for m in store.list_skills({"sub-a"})}
|
||||
b = {m.id for m in store.list_skills({"sub-b"})}
|
||||
both = {m.id for m in store.list_skills({"sub-a", "sub-b"})}
|
||||
assert a == {"k-a1", "f-a2"}
|
||||
assert b == {"k-b1"}
|
||||
assert both == {"k-a1", "f-a2", "k-b1"}
|
||||
|
||||
|
||||
def test_list_skills_empty_active_set_returns_empty(store):
|
||||
_seed_two_subscriptions(store)
|
||||
assert store.list_skills(set()) == []
|
||||
|
||||
|
||||
def test_list_skills_with_inactive_subscription(store):
|
||||
_seed_two_subscriptions(store)
|
||||
# Mark sub-a inactive in storage
|
||||
store._set_subscription_state("sub-a", active=False)
|
||||
assert store.list_skills({"sub-a"}) == []
|
||||
# sub-b still visible
|
||||
assert {m.id for m in store.list_skills({"sub-a", "sub-b"})} == {"k-b1"}
|
||||
|
||||
|
||||
def test_list_skills_unknown_subscription_returns_empty(store):
|
||||
_seed_two_subscriptions(store)
|
||||
assert store.list_skills({"unknown-sub"}) == []
|
||||
|
||||
|
||||
def test_search_skills_matches_name_tag_description(store):
|
||||
_seed_two_subscriptions(store)
|
||||
# Name match
|
||||
assert {m.id for m in store.search_skills("Alpha", {"sub-a", "sub-b"})} == {
|
||||
"k-a1",
|
||||
"f-a2",
|
||||
}
|
||||
# Tag match
|
||||
assert {m.id for m in store.search_skills("beta", {"sub-a", "sub-b"})} == {"k-b1"}
|
||||
# Description match would work the same way; verify no-match returns empty
|
||||
assert store.search_skills("zzz", {"sub-a", "sub-b"}) == []
|
||||
|
||||
|
||||
def test_search_skills_respects_visibility(store):
|
||||
_seed_two_subscriptions(store)
|
||||
# sub-a inactive → its skills don't surface even if query matches
|
||||
store._set_subscription_state("sub-a", active=False)
|
||||
assert {m.id for m in store.search_skills("Alpha", {"sub-a", "sub-b"})} == set()
|
||||
|
||||
|
||||
def test_get_skill_returns_full_content_for_knowledge(store):
|
||||
_seed_two_subscriptions(store)
|
||||
skill = store.get_skill("k-a1", {"sub-a"})
|
||||
assert isinstance(skill, KnowledgeSkill)
|
||||
assert skill.content == "c-a1"
|
||||
assert skill.metadata.name == "Alpha Knowledge"
|
||||
|
||||
|
||||
def test_get_skill_returns_full_steps_for_flow_template(store):
|
||||
_seed_two_subscriptions(store)
|
||||
skill = store.get_skill("f-a2", {"sub-a"})
|
||||
assert isinstance(skill, FlowTemplateSkill)
|
||||
assert len(skill.steps) == 1
|
||||
assert skill.steps[0].tool_name == "tap"
|
||||
assert skill.steps[0].args == {"x": 1}
|
||||
|
||||
|
||||
def test_get_skill_unknown_id_returns_none(store):
|
||||
_seed_two_subscriptions(store)
|
||||
assert store.get_skill("does-not-exist", {"sub-a"}) is None
|
||||
|
||||
|
||||
def test_get_skill_not_visible_indistinguishable_from_not_found(store):
|
||||
"""Existence must not leak: invisible skill returns None, same as unknown."""
|
||||
_seed_two_subscriptions(store)
|
||||
# Skill exists under sub-a but caller asks with only sub-b
|
||||
invisible = store.get_skill("k-a1", {"sub-b"})
|
||||
unknown = store.get_skill("never-existed", {"sub-b"})
|
||||
assert invisible is None
|
||||
assert unknown is None
|
||||
|
||||
|
||||
def test_get_skill_with_inactive_subscription_returns_none(store):
|
||||
_seed_two_subscriptions(store)
|
||||
store._set_subscription_state("sub-a", active=False)
|
||||
assert store.get_skill("k-a1", {"sub-a"}) is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 2.5: sync-only write contract (no public writes)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_no_public_write_methods_on_store():
|
||||
"""SkillCatalogStore must not expose create/update/remove publicly."""
|
||||
public_methods = {
|
||||
name
|
||||
for name in dir(SkillCatalogStore)
|
||||
if not name.startswith("_")
|
||||
and callable(getattr(SkillCatalogStore, name))
|
||||
}
|
||||
forbidden = {"create", "update", "remove", "delete", "save", "put", "post"}
|
||||
assert not (public_methods & forbidden), (
|
||||
f"Public write methods found: {public_methods & forbidden}"
|
||||
)
|
||||
|
||||
|
||||
def test_apply_sync_upsert_round_trips_both_kinds(store):
|
||||
store._register_subscription("sub-a")
|
||||
original_k = _knowledge("k1", "Knowledge One", "content body", tags=["t1"])
|
||||
original_f = _flow(
|
||||
"f1",
|
||||
"Flow One",
|
||||
steps=[("tap", {"x": 1}), ("input_text", {"text": "hi"})],
|
||||
parameters={"text": {"type": "string", "required": True}},
|
||||
tags=["t2"],
|
||||
)
|
||||
store._apply_sync_upsert(original_k, "sub-a")
|
||||
store._apply_sync_upsert(original_f, "sub-a")
|
||||
|
||||
fetched_k = store.get_skill("k1", {"sub-a"})
|
||||
fetched_f = store.get_skill("f1", {"sub-a"})
|
||||
assert isinstance(fetched_k, KnowledgeSkill)
|
||||
assert isinstance(fetched_f, FlowTemplateSkill)
|
||||
assert fetched_k.content == "content body"
|
||||
assert fetched_k.metadata.tags == ["t1"]
|
||||
assert [s.tool_name for s in fetched_f.steps] == ["tap", "input_text"]
|
||||
assert fetched_f.steps[0].args == {"x": 1}
|
||||
assert fetched_f.parameters == {"text": {"type": "string", "required": True}}
|
||||
|
||||
|
||||
def test_apply_sync_upsert_overwrites_on_conflict(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_upsert(_knowledge("k1", "v1", "old"), "sub-a")
|
||||
store._apply_sync_upsert(_knowledge("k1", "v1-updated", "new"), "sub-a")
|
||||
skill = store.get_skill("k1", {"sub-a"})
|
||||
assert isinstance(skill, KnowledgeSkill)
|
||||
assert skill.content == "new"
|
||||
assert skill.metadata.name == "v1-updated"
|
||||
|
||||
|
||||
def test_apply_sync_remove_deletes_skill(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_upsert(_knowledge("k1", "v1", "c"), "sub-a")
|
||||
store._apply_sync_remove("k1")
|
||||
assert store.get_skill("k1", {"sub-a"}) is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 2.6: flow-template tool-reference validation
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_validate_flow_template_tools_all_known_returns_true():
|
||||
skill = _flow("f", "f", steps=[("tap", {}), ("input_text", {"text": "x"})])
|
||||
assert validate_flow_template_tools(skill, {"tap", "input_text"}) is True
|
||||
|
||||
|
||||
def test_validate_flow_template_tools_dangling_returns_false():
|
||||
skill = _flow("f", "f", steps=[("tap", {}), ("ghost_tool", {})])
|
||||
assert validate_flow_template_tools(skill, {"tap"}) is False
|
||||
|
||||
|
||||
def test_validate_flow_template_tools_empty_steps_returns_true():
|
||||
skill = _flow("f", "f", steps=[])
|
||||
assert validate_flow_template_tools(skill, set()) is True
|
||||
|
||||
|
||||
def test_get_skill_returns_none_when_flow_template_has_dangling_reference(store):
|
||||
store._register_subscription("sub-a")
|
||||
skill = _flow("f1", "Flow", steps=[("tap", {}), ("ghost_tool", {})])
|
||||
store._apply_sync_upsert(skill, "sub-a")
|
||||
# Without registered_tools: skill is returned as-is
|
||||
assert store.get_skill("f1", {"sub-a"}) is not None
|
||||
# With registered_tools missing ghost_tool: returns None (unavailable)
|
||||
result = store.get_skill("f1", {"sub-a"}, registered_tools={"tap"})
|
||||
assert result is None
|
||||
# With all tools registered: returns the skill
|
||||
result = store.get_skill("f1", {"sub-a"}, registered_tools={"tap", "ghost_tool"})
|
||||
assert isinstance(result, FlowTemplateSkill)
|
||||
|
||||
|
||||
def test_get_skill_validation_skipped_for_knowledge(store):
|
||||
"""registered_tools must not affect knowledge skills."""
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_upsert(_knowledge("k1", "K", "c"), "sub-a")
|
||||
result = store.get_skill("k1", {"sub-a"}, registered_tools=set())
|
||||
assert isinstance(result, KnowledgeSkill)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Stable sort order
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_list_skills_is_sorted_by_name(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_replace_all(
|
||||
"sub-a",
|
||||
[
|
||||
_knowledge("z", "Zebra", "c"),
|
||||
_knowledge("a", "Apple", "c"),
|
||||
_knowledge("m", "Mango", "c"),
|
||||
],
|
||||
)
|
||||
names = [m.name for m in store.list_skills({"sub-a"})]
|
||||
assert names == ["Apple", "Mango", "Zebra"]
|
||||
|
||||
|
||||
def test_search_skills_ranks_name_prefix_above_contains(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_replace_all(
|
||||
"sub-a",
|
||||
[
|
||||
_knowledge("k1", "Find Search", "c"), # contains only
|
||||
_knowledge("k2", "Search Tips", "c"), # prefix match
|
||||
],
|
||||
)
|
||||
results = store.search_skills("Search", {"sub-a"})
|
||||
names = [m.name for m in results]
|
||||
# Prefix match "Search Tips" must rank above contains-only "Find Search"
|
||||
assert names[0] == "Search Tips"
|
||||
assert names[1] == "Find Search"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Round-trip serialization for both kinds
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_round_trip_knowledge_preserves_all_metadata(store):
|
||||
store._register_subscription("sub-a")
|
||||
original = KnowledgeSkill(
|
||||
metadata=SkillMetadata(
|
||||
id="rt-k",
|
||||
name="Round",
|
||||
description="desc",
|
||||
kind="knowledge",
|
||||
tags=["a", "b"],
|
||||
source="subscription",
|
||||
version=3,
|
||||
created_at=utc_now(),
|
||||
updated_at=utc_now(),
|
||||
),
|
||||
content="body",
|
||||
)
|
||||
store._apply_sync_upsert(original, "sub-a")
|
||||
fetched = store.get_skill("rt-k", {"sub-a"})
|
||||
assert isinstance(fetched, KnowledgeSkill)
|
||||
assert fetched.metadata.id == "rt-k"
|
||||
assert fetched.metadata.name == "Round"
|
||||
assert fetched.metadata.description == "desc"
|
||||
assert fetched.metadata.kind == "knowledge"
|
||||
assert fetched.metadata.tags == ["a", "b"]
|
||||
assert fetched.metadata.source == "subscription"
|
||||
assert fetched.metadata.version == 3
|
||||
assert fetched.content == "body"
|
||||
|
||||
|
||||
def test_round_trip_flow_template_preserves_steps_and_parameters(store):
|
||||
store._register_subscription("sub-a")
|
||||
original = FlowTemplateSkill(
|
||||
metadata=SkillMetadata(
|
||||
id="rt-f",
|
||||
name="RT Flow",
|
||||
description="d",
|
||||
kind="flow_template",
|
||||
tags=["x"],
|
||||
source="subscription",
|
||||
version=2,
|
||||
),
|
||||
steps=[
|
||||
FlowStep(tool_name="tap", args={"x": 10, "y": 20}),
|
||||
FlowStep(tool_name="input_text", args={"text": "hello"}),
|
||||
],
|
||||
parameters={"text": {"type": "string", "required": True}},
|
||||
)
|
||||
store._apply_sync_upsert(original, "sub-a")
|
||||
fetched = store.get_skill("rt-f", {"sub-a"})
|
||||
assert isinstance(fetched, FlowTemplateSkill)
|
||||
assert [s.tool_name for s in fetched.steps] == ["tap", "input_text"]
|
||||
assert fetched.steps[0].args == {"x": 10, "y": 20}
|
||||
assert fetched.parameters == {"text": {"type": "string", "required": True}}
|
||||
assert fetched.metadata.version == 2
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Subscription state observability (used by sync failure path)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_set_subscription_state_records_error_and_version(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._set_subscription_state(
|
||||
"sub-a",
|
||||
active=False,
|
||||
last_synced_version=7,
|
||||
last_error="401 Unauthorized",
|
||||
)
|
||||
with store._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT active, last_synced_version, last_error FROM subscriptions WHERE id = ?",
|
||||
("sub-a",),
|
||||
).fetchone()
|
||||
assert dict(row) == {
|
||||
"active": 0,
|
||||
"last_synced_version": 7,
|
||||
"last_error": "401 Unauthorized",
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
"""End-to-end validation for skill-catalog-subscription (tasks 5.1-5.3).
|
||||
|
||||
Combines: FakeSubscriptionClient → SkillSyncRunner → SkillCatalogStore →
|
||||
api.skill_catalog_mcp handlers. Verifies the full sync+query+resolve loop
|
||||
and that resolved flow templates drive through the existing device-capability
|
||||
tools one step at a time (no batched execution; Observe-Think-Act preserved).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from api.mcp import tool_handlers
|
||||
from api.skill_catalog_mcp import skill_tool_handlers
|
||||
from api.skill_sync import SkillSyncRunner, SyncDelta
|
||||
from device.manager import DeviceManager
|
||||
from runtime.executor import Executor, ExecutorConfig
|
||||
from runtime.planner import PlannedStep
|
||||
from skills_learning.models import (
|
||||
FlowStep,
|
||||
FlowTemplateSkill,
|
||||
KnowledgeSkill,
|
||||
SkillMetadata,
|
||||
)
|
||||
from storage.skill_catalog import SkillCatalogStore
|
||||
from tests.fakes import FakeDriver
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Shared fake client
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class E2EFakeClient:
|
||||
"""In-process SubscriptionClient with revocable entitlements."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.skills_by_sub: dict[str, list[Any]] = {}
|
||||
|
||||
def set_skills(self, sub_id: str, skills: list[Any]) -> None:
|
||||
self.skills_by_sub[sub_id] = list(skills)
|
||||
|
||||
def revoke(self, sub_id: str) -> None:
|
||||
self.skills_by_sub.pop(sub_id, None)
|
||||
|
||||
def fetch_entitled_skills(self, subscription_id, since_version=None):
|
||||
skills = self.skills_by_sub.get(subscription_id, [])
|
||||
return SyncDelta(
|
||||
skills=list(skills),
|
||||
removed_ids=[],
|
||||
latest_version=1,
|
||||
is_full_replace=True,
|
||||
)
|
||||
|
||||
|
||||
def _knowledge(skill_id: str, name: str, content: str, tags=None) -> KnowledgeSkill:
|
||||
return KnowledgeSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="knowledge",
|
||||
tags=tags or [],
|
||||
source="subscription",
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def _flow(
|
||||
skill_id: str,
|
||||
name: str,
|
||||
*,
|
||||
steps: list[tuple[str, dict]],
|
||||
parameters: dict | None = None,
|
||||
tags=None,
|
||||
) -> FlowTemplateSkill:
|
||||
return FlowTemplateSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="flow_template",
|
||||
tags=tags or [],
|
||||
source="subscription",
|
||||
),
|
||||
steps=[FlowStep(tool_name=t, args=a) for t, a in steps],
|
||||
parameters=parameters or {},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return SkillCatalogStore(db_path=tmp_path / "e2e_skills.sqlite3")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_client():
|
||||
return E2EFakeClient()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runner(store, fake_client):
|
||||
return SkillSyncRunner(
|
||||
store=store,
|
||||
client=fake_client,
|
||||
subscriptions=["sub-a"],
|
||||
poll_interval=999.0, # no auto-poll
|
||||
)
|
||||
|
||||
|
||||
def _mcp_handlers(store, *, registered_tools, active=None):
|
||||
return skill_tool_handlers(
|
||||
store=store,
|
||||
get_active_subscriptions=lambda: set(active if active is not None else {"sub-a"}),
|
||||
get_registered_tools=lambda: set(registered_tools),
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 5.1: seed + verify listable / searchable / fetchable via MCP
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_e2e_synced_skills_are_listable_searchable_fetchable_via_mcp(
|
||||
store, fake_client, runner
|
||||
):
|
||||
# Seed the fake Subscription Platform with one of each kind
|
||||
fake_client.set_skills(
|
||||
"sub-a",
|
||||
[
|
||||
_knowledge(
|
||||
"xhs-tips",
|
||||
"Xiaohongshu Search Tips",
|
||||
"Use specific keywords and filter by recent posts.",
|
||||
tags=["social", "search"],
|
||||
),
|
||||
_flow(
|
||||
"xhs-search-flow",
|
||||
"Xiaohongshu Search Flow",
|
||||
steps=[
|
||||
("tap", {"x": "{search_x}", "y": "{search_y}"}),
|
||||
("input_text", {"text": "{query}"}),
|
||||
("tap", {"x": "{submit_x}", "y": "{submit_y}"}),
|
||||
],
|
||||
parameters={
|
||||
"search_x": {"type": "number", "required": True},
|
||||
"search_y": {"type": "number", "required": True},
|
||||
"query": {"type": "string", "required": True},
|
||||
"submit_x": {"type": "number", "required": True},
|
||||
"submit_y": {"type": "number", "required": True},
|
||||
},
|
||||
tags=["social", "automation"],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Run one sync tick
|
||||
outcomes = runner.tick()
|
||||
assert outcomes["sub-a"].success is True
|
||||
assert outcomes["sub-a"].fetched == 2
|
||||
|
||||
# All device-capability tools are registered (used as the validator set)
|
||||
registered = {"tap", "input_text", "swipe", "launch_app"}
|
||||
handlers = _mcp_handlers(store, registered_tools=registered)
|
||||
|
||||
# list_skills returns both
|
||||
listed = handlers["list_skills"]()
|
||||
assert listed["ok"] is True
|
||||
ids = {s["id"] for s in listed["skills"]}
|
||||
assert ids == {"xhs-tips", "xhs-search-flow"}
|
||||
|
||||
# search_skills finds the knowledge skill by tag and the flow by name
|
||||
by_tag = handlers["search_skills"](query="social")
|
||||
assert {s["id"] for s in by_tag["skills"]} == {"xhs-tips", "xhs-search-flow"}
|
||||
by_name = handlers["search_skills"](query="Xiaohongshu Search Flow")
|
||||
assert {s["id"] for s in by_name["skills"]} == {"xhs-search-flow"}
|
||||
|
||||
# get_skill returns full content for each kind
|
||||
knowledge = handlers["get_skill"](skill_id="xhs-tips")
|
||||
assert knowledge["ok"] is True
|
||||
assert knowledge["skill"]["kind"] == "knowledge"
|
||||
assert "specific keywords" in knowledge["skill"]["content"]
|
||||
|
||||
flow = handlers["get_skill"](skill_id="xhs-search-flow")
|
||||
assert flow["ok"] is True
|
||||
assert flow["skill"]["kind"] == "flow_template"
|
||||
assert [s["tool_name"] for s in flow["skill"]["steps"]] == [
|
||||
"tap",
|
||||
"input_text",
|
||||
"tap",
|
||||
]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 5.2: revocation — sync-time removal + query-time re-check
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_e2e_revocation_via_sync_cycle_removes_skill(
|
||||
store, fake_client, runner
|
||||
):
|
||||
"""When the Subscription Platform no longer includes a skill in the
|
||||
entitled set, the next sync cycle removes it locally."""
|
||||
fake_client.set_skills(
|
||||
"sub-a",
|
||||
[_knowledge("k1", "Keep Me", "c"), _knowledge("k2", "Revoke Me", "c")],
|
||||
)
|
||||
runner.tick()
|
||||
handlers = _mcp_handlers(store, registered_tools=set())
|
||||
assert {s["id"] for s in handlers["list_skills"]()["skills"]} == {"k1", "k2"}
|
||||
|
||||
# Subscription Platform revokes k2
|
||||
fake_client.set_skills("sub-a", [_knowledge("k1", "Keep Me", "c")])
|
||||
runner.tick()
|
||||
|
||||
after = handlers["list_skills"]()
|
||||
ids = {s["id"] for s in after["skills"]}
|
||||
assert ids == {"k1"}
|
||||
assert handlers["get_skill"](skill_id="k2")["ok"] is False
|
||||
|
||||
|
||||
def test_e2e_revocation_via_query_time_recheck_hides_skill_before_next_sync(
|
||||
store, fake_client, runner
|
||||
):
|
||||
"""Marking a subscription inactive hides its skills on the next query,
|
||||
without waiting for the next sync tick to remove the rows."""
|
||||
fake_client.set_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner.tick()
|
||||
handlers_before = _mcp_handlers(store, registered_tools=set())
|
||||
assert handlers_before["get_skill"](skill_id="k1")["ok"] is True
|
||||
|
||||
# Revoke the subscription locally (e.g., via push notification of revocation)
|
||||
store._set_subscription_state("sub-a", active=False)
|
||||
|
||||
# Same handler config — query re-check excludes sub-a's skills immediately
|
||||
handlers_after = _mcp_handlers(store, registered_tools=set())
|
||||
assert handlers_after["list_skills"]()["skills"] == []
|
||||
assert handlers_after["get_skill"](skill_id="k1")["ok"] is False
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 5.3: resolve via MCP, then drive through device-capability tools
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_e2e_resolved_flow_template_drives_existing_device_tools_one_step_at_a_time(
|
||||
store, fake_client, runner
|
||||
):
|
||||
"""Resolve a flow-template skill via MCP, then execute each resulting
|
||||
step through the existing device-capability tools (Executor + FakeDriver).
|
||||
|
||||
Confirms the Observe-Think-Act loop is preserved: no batched
|
||||
server-side execution primitive is used. Each step is issued
|
||||
individually, exactly as the LLM would.
|
||||
"""
|
||||
# 1. Seed a flow template that uses tap + input_text (real device tools)
|
||||
fake_client.set_skills(
|
||||
"sub-a",
|
||||
[
|
||||
_flow(
|
||||
"login-flow",
|
||||
"Login Flow",
|
||||
steps=[
|
||||
("tap", {"x": 50, "y": 100}),
|
||||
("input_text", {"text": "user@example.com"}),
|
||||
("tap", {"x": 50, "y": 200}),
|
||||
],
|
||||
parameters={},
|
||||
)
|
||||
],
|
||||
)
|
||||
runner.tick()
|
||||
|
||||
# 2. Set up the existing device-capability tool surface (apex-agent-mvp)
|
||||
fake_driver = FakeDriver()
|
||||
manager = DeviceManager()
|
||||
manager.register_device("device-1", lambda: fake_driver)
|
||||
manager.connect("device-1", max_retries=1)
|
||||
device_handlers = tool_handlers(manager=manager)
|
||||
registered_tool_names = set(device_handlers.keys())
|
||||
|
||||
# 3. Resolve the flow template via the MCP handler
|
||||
skill_handlers = _mcp_handlers(store, registered_tools=registered_tool_names)
|
||||
resolved = skill_handlers["resolve_flow_template"](
|
||||
skill_id="login-flow", params={}
|
||||
)
|
||||
assert resolved["ok"] is True
|
||||
steps = resolved["steps"]
|
||||
assert len(steps) == 3
|
||||
|
||||
# 4. Build an Executor with the device-capability tools — the same
|
||||
# Executor the Agent Runtime uses for the Act phase. Bind device_id
|
||||
# via closures so each step executes against the connected device.
|
||||
# Each step is issued individually through executor.execute(step),
|
||||
# exactly as the LLM-driven Observe-Think-Act loop does.
|
||||
device_id = "device-1"
|
||||
executor = Executor(
|
||||
tools={
|
||||
"tap": lambda **kw: device_handlers["tap"](device_id=device_id, **kw),
|
||||
"input_text": lambda **kw: device_handlers["input_text"](
|
||||
device_id=device_id, **kw
|
||||
),
|
||||
},
|
||||
config=ExecutorConfig(max_retries=1, backoff_seconds=0),
|
||||
)
|
||||
|
||||
results = []
|
||||
for index, step in enumerate(steps, start=1):
|
||||
planned = PlannedStep(
|
||||
action=step["tool_name"],
|
||||
description=f"login-flow step {index}",
|
||||
args=step["args"],
|
||||
)
|
||||
result = executor.execute(planned)
|
||||
results.append(result)
|
||||
|
||||
# 5. Every step must succeed through the normal Act loop
|
||||
assert all(r.success for r in results), (
|
||||
f"step failed: {[r.error for r in results if not r.success]}"
|
||||
)
|
||||
assert len(results) == 3
|
||||
|
||||
# 6. The FakeDriver must have received the calls in order — proving the
|
||||
# resolved template actually reached the device via the existing
|
||||
# capability surface (not via a new batch primitive).
|
||||
tool_calls = [c for c in fake_driver.calls if c[0] in ("tap", "input")]
|
||||
# 3 steps → exactly 3 device-level calls (tap, input, tap)
|
||||
assert len(tool_calls) == 3
|
||||
tap_calls = [c for c in tool_calls if c[0] == "tap"]
|
||||
assert tap_calls[0][1] == (50, 100)
|
||||
assert tap_calls[1][1] == (50, 200)
|
||||
input_calls = [c for c in tool_calls if c[0] == "input"]
|
||||
assert input_calls[0][1] == ("user@example.com",)
|
||||
|
||||
|
||||
def test_e2e_resolved_flow_template_uses_parameter_substitution(
|
||||
store, fake_client, runner
|
||||
):
|
||||
"""Resolution substitutes {param} placeholders so the LLM-driven Act loop
|
||||
receives concrete argument values, not templates."""
|
||||
fake_client.set_skills(
|
||||
"sub-a",
|
||||
[
|
||||
_flow(
|
||||
"param-flow",
|
||||
"Parameterized",
|
||||
steps=[
|
||||
("tap", {"x": "{coord_x}", "y": "{coord_y}"}),
|
||||
("input_text", {"text": "fixed-text"}),
|
||||
],
|
||||
parameters={
|
||||
"coord_x": {"type": "number", "required": True},
|
||||
"coord_y": {"type": "number", "required": True},
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
runner.tick()
|
||||
handlers = _mcp_handlers(store, registered_tools={"tap", "input_text"})
|
||||
resolved = handlers["resolve_flow_template"](
|
||||
skill_id="param-flow",
|
||||
params={"coord_x": 42, "coord_y": 99},
|
||||
)
|
||||
assert resolved["ok"] is True
|
||||
steps = resolved["steps"]
|
||||
# Placeholder substituted with provided param values
|
||||
assert steps[0]["args"] == {"x": 42, "y": 99}
|
||||
# Non-parameterized args pass through unchanged
|
||||
assert steps[1]["args"] == {"text": "fixed-text"}
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Tests for api/skill_catalog_mcp.py.
|
||||
|
||||
Covers task 4.5: list/search/get via MCP-style handler calls, resolve_flow_template
|
||||
success and failure cases, semantic error translation, and an explicit
|
||||
assertion that no batch-execute tool is registered (design D5 / task 4.4).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from api.skill_catalog_mcp import (
|
||||
SKILL_TOOL_NAMES,
|
||||
InvalidFlowTemplateError,
|
||||
MissingParameterError,
|
||||
SkillCatalogError,
|
||||
SkillNotFoundError,
|
||||
register_skill_catalog_tools,
|
||||
skill_tool_handlers,
|
||||
)
|
||||
from skills_learning.models import (
|
||||
FlowStep,
|
||||
FlowTemplateSkill,
|
||||
KnowledgeSkill,
|
||||
SkillMetadata,
|
||||
)
|
||||
from storage.skill_catalog import SkillCatalogStore
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Fixtures and helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _knowledge(skill_id: str, name: str, content: str, tags=None) -> KnowledgeSkill:
|
||||
return KnowledgeSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="knowledge",
|
||||
tags=tags or [],
|
||||
source="subscription",
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def _flow(
|
||||
skill_id: str,
|
||||
name: str,
|
||||
*,
|
||||
steps: list[tuple[str, dict]],
|
||||
parameters: dict | None = None,
|
||||
tags=None,
|
||||
) -> FlowTemplateSkill:
|
||||
return FlowTemplateSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="flow_template",
|
||||
tags=tags or [],
|
||||
source="subscription",
|
||||
),
|
||||
steps=[FlowStep(tool_name=t, args=a) for t, a in steps],
|
||||
parameters=parameters or {},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return SkillCatalogStore(db_path=tmp_path / "skills.sqlite3")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def seeded_store(store):
|
||||
store._register_subscription("sub-a")
|
||||
store._apply_sync_replace_all(
|
||||
"sub-a",
|
||||
[
|
||||
_knowledge("k1", "Knowledge One", "body of knowledge", tags=["docs"]),
|
||||
_flow(
|
||||
"f1",
|
||||
"Search Flow",
|
||||
steps=[
|
||||
("tap", {"x": "{x_coord}"}),
|
||||
("input_text", {"text": "{query}"}),
|
||||
],
|
||||
parameters={
|
||||
"x_coord": {"type": "number", "required": True},
|
||||
"query": {"type": "string", "required": True},
|
||||
},
|
||||
tags=["search"],
|
||||
),
|
||||
],
|
||||
)
|
||||
return store
|
||||
|
||||
|
||||
def _handlers(store, *, registered_tools=None, active=None):
|
||||
return skill_tool_handlers(
|
||||
store=store,
|
||||
get_active_subscriptions=lambda: set(active if active is not None else {"sub-a"}),
|
||||
get_registered_tools=(lambda: set(registered_tools or {"tap", "input_text"})),
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# list_skills / search_skills / get_skill via handlers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_list_skills_returns_summaries_without_platform_fields(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["list_skills"]()
|
||||
assert result["ok"] is True
|
||||
assert len(result["skills"]) == 2
|
||||
# Summaries contain only id/name/description/kind/tags — no source/version
|
||||
serialized = repr(result)
|
||||
assert "subscription" not in serialized.lower()
|
||||
assert "parent_version_id" not in serialized
|
||||
assert "originating_goal" not in serialized
|
||||
names = {s["name"] for s in result["skills"]}
|
||||
assert names == {"Knowledge One", "Search Flow"}
|
||||
|
||||
|
||||
def test_list_skills_empty_when_no_active_subscriptions(seeded_store):
|
||||
handlers = _handlers(seeded_store, active=set())
|
||||
result = handlers["list_skills"]()
|
||||
assert result == {"ok": True, "skills": []}
|
||||
|
||||
|
||||
def test_search_skills_finds_by_tag_and_name(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
by_name = handlers["search_skills"](query="Search")
|
||||
assert {s["id"] for s in by_name["skills"]} == {"f1"}
|
||||
|
||||
by_tag = handlers["search_skills"](query="docs")
|
||||
assert {s["id"] for s in by_tag["skills"]} == {"k1"}
|
||||
|
||||
|
||||
def test_get_skill_returns_full_knowledge_content(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["get_skill"](skill_id="k1")
|
||||
assert result["ok"] is True
|
||||
skill = result["skill"]
|
||||
assert skill["kind"] == "knowledge"
|
||||
assert skill["content"] == "body of knowledge"
|
||||
assert skill["name"] == "Knowledge One"
|
||||
|
||||
|
||||
def test_get_skill_returns_full_flow_template(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["get_skill"](skill_id="f1")
|
||||
assert result["ok"] is True
|
||||
skill = result["skill"]
|
||||
assert skill["kind"] == "flow_template"
|
||||
assert [s["tool_name"] for s in skill["steps"]] == ["tap", "input_text"]
|
||||
assert "x_coord" in skill["parameters"]
|
||||
|
||||
|
||||
def test_get_skill_unknown_returns_skill_not_found(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["get_skill"](skill_id="does-not-exist")
|
||||
assert result["ok"] is False
|
||||
assert result["error"] == "skill not found"
|
||||
|
||||
|
||||
def test_get_skill_not_visible_indistinguishable_from_not_found(seeded_store):
|
||||
"""Existence must not leak: not-visible returns the same error as not-found."""
|
||||
handlers = _handlers(seeded_store, active=set()) # no active subs
|
||||
invisible = handlers["get_skill"](skill_id="k1")
|
||||
unknown = handlers["get_skill"](skill_id="never-existed")
|
||||
assert invisible == unknown == {"ok": False, "error": "skill not found"}
|
||||
|
||||
|
||||
def test_get_skill_with_dangling_tool_reference_returns_unavailable(seeded_store):
|
||||
# f1 references tap + input_text; drop input_text from the registered set
|
||||
handlers = _handlers(seeded_store, registered_tools={"tap"})
|
||||
result = handlers["get_skill"](skill_id="f1")
|
||||
assert result["ok"] is False
|
||||
# Per task 2.6, get_skill returns None for dangling refs, which surfaces
|
||||
# as "skill not found" at the MCP layer (no existence leak on the cause).
|
||||
assert result["error"] == "skill not found"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# resolve_flow_template
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_flow_template_succeeds_with_valid_params(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["resolve_flow_template"](
|
||||
skill_id="f1",
|
||||
params={"x_coord": 100, "query": "hello"},
|
||||
)
|
||||
assert result["ok"] is True
|
||||
steps = result["steps"]
|
||||
assert steps[0]["tool_name"] == "tap"
|
||||
assert steps[0]["args"]["x"] == 100 # {x_coord} substituted
|
||||
assert steps[1]["tool_name"] == "input_text"
|
||||
assert steps[1]["args"]["text"] == "hello"
|
||||
|
||||
|
||||
def test_resolve_flow_template_missing_required_param(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["resolve_flow_template"](
|
||||
skill_id="f1",
|
||||
params={"x_coord": 100}, # missing 'query'
|
||||
)
|
||||
assert result["ok"] is False
|
||||
assert result["error"].startswith("missing parameter:")
|
||||
assert "query" in result["error"]
|
||||
|
||||
|
||||
def test_resolve_flow_template_unknown_skill(seeded_store):
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["resolve_flow_template"](
|
||||
skill_id="no-such",
|
||||
params={},
|
||||
)
|
||||
assert result["ok"] is False
|
||||
assert result["error"] == "skill not found"
|
||||
|
||||
|
||||
def test_resolve_flow_template_on_knowledge_skill_returns_unavailable(seeded_store):
|
||||
"""Cannot resolve a knowledge skill as a flow template."""
|
||||
handlers = _handlers(seeded_store)
|
||||
result = handlers["resolve_flow_template"](skill_id="k1", params={})
|
||||
assert result["ok"] is False
|
||||
assert result["error"] == "skill unavailable"
|
||||
|
||||
|
||||
def test_resolve_flow_template_with_dangling_tool_reference(seeded_store):
|
||||
"""A skill whose steps reference an unregistered tool is treated as
|
||||
not-found at resolve time (consistent with get_skill's behavior)."""
|
||||
handlers = _handlers(seeded_store, registered_tools={"tap"}) # missing input_text
|
||||
result = handlers["resolve_flow_template"](
|
||||
skill_id="f1",
|
||||
params={"x_coord": 1, "query": "x"},
|
||||
)
|
||||
assert result["ok"] is False
|
||||
assert result["error"] == "skill not found"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Registration: tools registered on FastMCP server, no batch-execute tool
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fastmcp_tool_names(server) -> set[str]:
|
||||
"""Read the FastMCP tool registry directly (sync, no event loop needed)."""
|
||||
# FastMCP stores tools in ToolManager._tools (mcp>=1.27). Walk the attrs
|
||||
# defensively so the test doesn't break on minor internal refactors.
|
||||
if hasattr(server, "_tool_manager"):
|
||||
manager = server._tool_manager
|
||||
if hasattr(manager, "_tools"):
|
||||
return set(manager._tools.keys())
|
||||
raise AssertionError(
|
||||
"Could not introspect FastMCP tool registry — internal API changed"
|
||||
)
|
||||
|
||||
|
||||
def test_register_skill_catalog_tools_registers_expected_names(seeded_store):
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
server = FastMCP("test")
|
||||
register_skill_catalog_tools(
|
||||
server,
|
||||
store=seeded_store,
|
||||
get_active_subscriptions=lambda: {"sub-a"},
|
||||
get_registered_tools=lambda: {"tap", "input_text"},
|
||||
)
|
||||
names = _fastmcp_tool_names(server)
|
||||
for expected in SKILL_TOOL_NAMES:
|
||||
assert expected in names, f"missing tool: {expected}"
|
||||
|
||||
|
||||
def test_no_batch_execute_tool_is_registered(seeded_store):
|
||||
"""Task 4.4: assert no run_skill_flow / execute / batch tool is registered."""
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
server = FastMCP("test")
|
||||
register_skill_catalog_tools(
|
||||
server,
|
||||
store=seeded_store,
|
||||
get_active_subscriptions=lambda: {"sub-a"},
|
||||
get_registered_tools=lambda: {"tap", "input_text"},
|
||||
)
|
||||
names = _fastmcp_tool_names(server)
|
||||
forbidden_fragments = ("run_", "execute", "batch", "invoke_skill")
|
||||
for name in names:
|
||||
lower = name.lower()
|
||||
for fragment in forbidden_fragments:
|
||||
assert fragment not in lower, (
|
||||
f"Forbidden tool name '{name}' contains '{fragment}' — "
|
||||
"design D5 forbids server-side flow execution tools"
|
||||
)
|
||||
|
||||
|
||||
def test_create_mcp_server_registers_skill_tools_when_store_provided(seeded_store):
|
||||
"""Wire-up: api.mcp.create_mcp_server must register skill tools when
|
||||
skill_catalog_store is provided."""
|
||||
from api.mcp import create_mcp_server
|
||||
|
||||
server = create_mcp_server(
|
||||
skill_catalog_store=seeded_store,
|
||||
skill_active_subscriptions={"sub-a"},
|
||||
)
|
||||
names = _fastmcp_tool_names(server)
|
||||
assert "list_skills" in names
|
||||
assert "tap" in names # existing device tools still present
|
||||
|
||||
|
||||
def test_create_mcp_server_omits_skill_tools_when_no_store():
|
||||
"""Wire-up must not break existing behavior when no store is provided."""
|
||||
from api.mcp import create_mcp_server
|
||||
|
||||
server = create_mcp_server()
|
||||
names = _fastmcp_tool_names(server)
|
||||
assert "list_skills" not in names
|
||||
assert "tap" in names # existing device tools present
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Error class hierarchy
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_skill_error_hierarchy():
|
||||
assert issubclass(SkillNotFoundError, SkillCatalogError)
|
||||
assert issubclass(InvalidFlowTemplateError, SkillCatalogError)
|
||||
assert issubclass(MissingParameterError, SkillCatalogError)
|
||||
@@ -0,0 +1,476 @@
|
||||
"""Tests for api/skill_sync.py.
|
||||
|
||||
Covers task 3.7: create/update/remove sync scenarios, push-triggered
|
||||
immediate sync, query-time revocation re-check, and failure-preserves-cache.
|
||||
All tests use a FakeSubscriptionClient implementing the Protocol — no real
|
||||
HTTP traffic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from api.skill_sync import (
|
||||
HttpSubscriptionClient,
|
||||
SkillSyncRunner,
|
||||
SyncDelta,
|
||||
SyncOutcome,
|
||||
_parse_sync_payload,
|
||||
register_skill_sync_webhook,
|
||||
)
|
||||
from skills_learning.models import (
|
||||
FlowStep,
|
||||
FlowTemplateSkill,
|
||||
KnowledgeSkill,
|
||||
SkillMetadata,
|
||||
)
|
||||
from storage.skill_catalog import SkillCatalogStore
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Test fakes and helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeSubscriptionClient:
|
||||
"""In-process implementation of the SubscriptionClient Protocol."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.skills_by_sub: dict[str, list[Any]] = {}
|
||||
self.delta_overrides: dict[str, SyncDelta] = {}
|
||||
self.next_failure: Exception | None = None
|
||||
self.call_log: list[tuple[str, int | None]] = []
|
||||
|
||||
def set_entitled_skills(self, sub_id: str, skills: list[Any]) -> None:
|
||||
self.skills_by_sub[sub_id] = list(skills)
|
||||
|
||||
def set_delta(self, sub_id: str, delta: SyncDelta) -> None:
|
||||
self.delta_overrides[sub_id] = delta
|
||||
|
||||
def fail_next_with(self, exc: Exception) -> None:
|
||||
self.next_failure = exc
|
||||
|
||||
def fetch_entitled_skills(
|
||||
self,
|
||||
subscription_id: str,
|
||||
since_version: int | None = None,
|
||||
) -> SyncDelta:
|
||||
self.call_log.append((subscription_id, since_version))
|
||||
if self.next_failure is not None:
|
||||
exc = self.next_failure
|
||||
self.next_failure = None
|
||||
raise exc
|
||||
if subscription_id in self.delta_overrides:
|
||||
return self.delta_overrides[subscription_id]
|
||||
skills = self.skills_by_sub.get(subscription_id, [])
|
||||
return SyncDelta(
|
||||
skills=list(skills),
|
||||
removed_ids=[],
|
||||
latest_version=1,
|
||||
is_full_replace=True,
|
||||
)
|
||||
|
||||
|
||||
def _knowledge(skill_id: str, name: str, content: str) -> KnowledgeSkill:
|
||||
return KnowledgeSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="knowledge",
|
||||
source="subscription",
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def _flow(skill_id: str, name: str, steps: list[tuple[str, dict]]) -> FlowTemplateSkill:
|
||||
return FlowTemplateSkill(
|
||||
metadata=SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
kind="flow_template",
|
||||
source="subscription",
|
||||
),
|
||||
steps=[FlowStep(tool_name=t, args=a) for t, a in steps],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return SkillCatalogStore(db_path=tmp_path / "skills.sqlite3")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake():
|
||||
return FakeSubscriptionClient()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runner(store, fake):
|
||||
return SkillSyncRunner(
|
||||
store=store,
|
||||
client=fake,
|
||||
subscriptions=["sub-a"],
|
||||
poll_interval=0.01,
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 3.3 / 3.5: poll loop apply scenarios (create / update / remove)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tick_creates_new_skills_in_catalog(store, fake, runner):
|
||||
fake.set_entitled_skills(
|
||||
"sub-a",
|
||||
[_knowledge("k1", "Knowledge", "body"), _flow("f1", "Flow", [("tap", {})])],
|
||||
)
|
||||
outcomes = runner.tick()
|
||||
assert outcomes["sub-a"].success is True
|
||||
assert outcomes["sub-a"].fetched == 2
|
||||
ids = {m.id for m in store.list_skills({"sub-a"})}
|
||||
assert ids == {"k1", "f1"}
|
||||
|
||||
|
||||
def test_tick_updates_existing_skill_content(store, fake, runner):
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "v1")])
|
||||
runner.tick()
|
||||
fetched = store.get_skill("k1", {"sub-a"})
|
||||
assert isinstance(fetched, KnowledgeSkill)
|
||||
assert fetched.content == "v1"
|
||||
|
||||
# Sync again with new content
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "v2-updated")])
|
||||
runner.tick()
|
||||
fetched = store.get_skill("k1", {"sub-a"})
|
||||
assert isinstance(fetched, KnowledgeSkill)
|
||||
assert fetched.content == "v2-updated"
|
||||
|
||||
|
||||
def test_tick_removes_skills_no_longer_entitled(store, fake, runner):
|
||||
fake.set_entitled_skills(
|
||||
"sub-a",
|
||||
[_knowledge("k1", "K", "c"), _knowledge("k2", "K2", "c")],
|
||||
)
|
||||
runner.tick()
|
||||
assert {m.id for m in store.list_skills({"sub-a"})} == {"k1", "k2"}
|
||||
|
||||
# Full replace with only k1 → k2 must disappear
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner.tick()
|
||||
assert {m.id for m in store.list_skills({"sub-a"})} == {"k1"}
|
||||
|
||||
|
||||
def test_tick_applies_incremental_delta_via_upsert_and_remove(store, fake, runner):
|
||||
# Seed with full sync
|
||||
fake.set_entitled_skills(
|
||||
"sub-a",
|
||||
[_knowledge("k1", "K1", "c1"), _knowledge("k2", "K2", "c2")],
|
||||
)
|
||||
runner.tick()
|
||||
|
||||
# Now simulate an incremental delta: k1 content updated, k2 removed, k3 added
|
||||
fake.set_delta(
|
||||
"sub-a",
|
||||
SyncDelta(
|
||||
skills=[_knowledge("k1", "K1", "c1-updated"), _knowledge("k3", "K3", "c3")],
|
||||
removed_ids=["k2"],
|
||||
latest_version=5,
|
||||
is_full_replace=False,
|
||||
),
|
||||
)
|
||||
outcomes = runner.tick()
|
||||
assert outcomes["sub-a"].success is True
|
||||
assert outcomes["sub-a"].fetched == 2
|
||||
|
||||
ids = {m.id for m in store.list_skills({"sub-a"})}
|
||||
assert ids == {"k1", "k3"}
|
||||
fetched = store.get_skill("k1", {"sub-a"})
|
||||
assert isinstance(fetched, KnowledgeSkill)
|
||||
assert fetched.content == "c1-updated"
|
||||
|
||||
|
||||
def test_tick_skips_unknown_subscriptions_in_outcome(store, fake):
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner = SkillSyncRunner(
|
||||
store=store,
|
||||
client=fake,
|
||||
subscriptions=["sub-a", "sub-b"],
|
||||
poll_interval=0.01,
|
||||
)
|
||||
outcomes = runner.tick()
|
||||
# sub-b has no skills set → empty delta, success, fetched=0
|
||||
assert outcomes["sub-b"].success is True
|
||||
assert outcomes["sub-b"].fetched == 0
|
||||
assert {m.id for m in store.list_skills({"sub-a"})} == {"k1"}
|
||||
assert store.list_skills({"sub-b"}) == []
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 3.6: sync failure preserves cache + records last_error
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tick_failure_preserves_cache_and_records_error(store, fake, runner):
|
||||
# Initial successful sync
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner.tick()
|
||||
assert store.get_skill("k1", {"sub-a"}) is not None
|
||||
|
||||
# Next tick fails — cache must remain, error recorded
|
||||
fake.fail_next_with(RuntimeError("503 Service Unavailable"))
|
||||
outcomes = runner.tick()
|
||||
assert outcomes["sub-a"].success is False
|
||||
assert "503 Service Unavailable" in (outcomes["sub-a"].error or "")
|
||||
assert outcomes["sub-a"].fetched == 0
|
||||
|
||||
# Cache preserved
|
||||
fetched = store.get_skill("k1", {"sub-a"})
|
||||
assert isinstance(fetched, KnowledgeSkill)
|
||||
assert fetched.content == "c"
|
||||
|
||||
# last_error recorded in subscriptions table
|
||||
with store._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT last_error FROM subscriptions WHERE id = ?", ("sub-a",)
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
assert "503 Service Unavailable" in (row["last_error"] or "")
|
||||
|
||||
|
||||
def test_tick_failure_then_recovery_clears_error(store, fake, runner):
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner.tick()
|
||||
|
||||
fake.fail_next_with(ConnectionError("network down"))
|
||||
runner.tick()
|
||||
# Successful sync after failure must clear last_error
|
||||
runner.tick()
|
||||
with store._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT last_error FROM subscriptions WHERE id = ?", ("sub-a",)
|
||||
).fetchone()
|
||||
assert row["last_error"] is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 3.5: query-time revocation before next sync
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_revocation_takes_effect_immediately_without_resync(store, fake, runner):
|
||||
"""Marking a subscription inactive hides its skills on the next query,
|
||||
even if no further sync tick has run."""
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner.tick()
|
||||
assert store.get_skill("k1", {"sub-a"}) is not None
|
||||
|
||||
# Revoke sub-a without syncing again
|
||||
store._set_subscription_state("sub-a", active=False)
|
||||
|
||||
# Query-time re-check (storage._effective_active) must hide the skill
|
||||
assert store.list_skills({"sub-a"}) == []
|
||||
assert store.get_skill("k1", {"sub-a"}) is None
|
||||
# Union with another (still-active) subscription still excludes sub-a's skills
|
||||
store._register_subscription("sub-b")
|
||||
assert store.list_skills({"sub-a", "sub-b"}) == []
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Task 3.4: push-triggered (webhook) immediate sync
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_webhook_triggers_immediate_sync_of_all_subscriptions(store, fake):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner = SkillSyncRunner(
|
||||
store=store,
|
||||
client=fake,
|
||||
subscriptions=["sub-a"],
|
||||
poll_interval=999.0, # never auto-poll
|
||||
)
|
||||
app = FastAPI()
|
||||
register_skill_sync_webhook(app, runner)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/webhooks/skill-sync")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["ok"] is True
|
||||
assert body["outcomes"]["sub-a"]["success"] is True
|
||||
# Catalog now reflects the synced skill without any tick() call from us
|
||||
assert {m.id for m in store.list_skills({"sub-a"})} == {"k1"}
|
||||
|
||||
|
||||
def test_webhook_with_subscription_id_syncs_only_that_subscription(store, fake):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
fake.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
fake.set_entitled_skills("sub-b", [_knowledge("k2", "K2", "c")])
|
||||
runner = SkillSyncRunner(
|
||||
store=store,
|
||||
client=fake,
|
||||
subscriptions=["sub-a", "sub-b"],
|
||||
poll_interval=999.0,
|
||||
)
|
||||
app = FastAPI()
|
||||
register_skill_sync_webhook(app, runner)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/webhooks/skill-sync", json={"subscription_id": "sub-a"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert set(body["outcomes"].keys()) == {"sub-a"}
|
||||
# Only sub-a was synced
|
||||
assert {m.id for m in store.list_skills({"sub-a"})} == {"k1"}
|
||||
# sub-b was not synced in this call
|
||||
assert store.list_skills({"sub-b"}) == []
|
||||
|
||||
|
||||
def test_register_webhook_rejects_non_fastapi_app():
|
||||
runner = object() # not a SkillSyncRunner, not a FastAPI app
|
||||
with pytest.raises(TypeError):
|
||||
register_skill_sync_webhook("not-an-app", runner) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Background poll loop
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_start_background_polls_until_stopped(store, fake):
|
||||
call_count = {"n": 0}
|
||||
|
||||
class CountingClient(FakeSubscriptionClient):
|
||||
def fetch_entitled_skills(self, subscription_id, since_version=None):
|
||||
call_count["n"] += 1
|
||||
return super().fetch_entitled_skills(subscription_id, since_version)
|
||||
|
||||
client = CountingClient()
|
||||
client.set_entitled_skills("sub-a", [_knowledge("k1", "K", "c")])
|
||||
runner = SkillSyncRunner(
|
||||
store=store,
|
||||
client=client,
|
||||
subscriptions=["sub-a"],
|
||||
poll_interval=0.01,
|
||||
)
|
||||
runner.start_background()
|
||||
try:
|
||||
# Allow at least 2 polls
|
||||
import time
|
||||
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
runner.stop_background()
|
||||
assert call_count["n"] >= 2
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# HttpSubscriptionClient: payload parsing + httpx transport (mocked)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_sync_payload_handles_knowledge_and_flow():
|
||||
payload = {
|
||||
"skills": [
|
||||
{"id": "k1", "name": "K", "kind": "knowledge", "content": "body"},
|
||||
{
|
||||
"id": "f1",
|
||||
"name": "F",
|
||||
"kind": "flow_template",
|
||||
"steps": [{"tool_name": "tap", "args": {"x": 1}}],
|
||||
"parameters": {"x": {"type": "int"}},
|
||||
},
|
||||
],
|
||||
"removed_ids": ["old1"],
|
||||
"latest_version": 7,
|
||||
"is_full_replace": False,
|
||||
}
|
||||
delta = _parse_sync_payload(payload)
|
||||
assert len(delta.skills) == 2
|
||||
assert isinstance(delta.skills[0], KnowledgeSkill)
|
||||
assert isinstance(delta.skills[1], FlowTemplateSkill)
|
||||
assert delta.skills[1].steps[0].tool_name == "tap"
|
||||
assert delta.removed_ids == ["old1"]
|
||||
assert delta.latest_version == 7
|
||||
assert delta.is_full_replace is False
|
||||
|
||||
|
||||
def test_parse_sync_payload_defaults_to_full_replace():
|
||||
delta = _parse_sync_payload({"skills": []})
|
||||
assert delta.skills == []
|
||||
assert delta.removed_ids == []
|
||||
assert delta.latest_version is None
|
||||
assert delta.is_full_replace is True
|
||||
|
||||
|
||||
def test_http_client_uses_injected_httpx_client_and_parses_response(monkeypatch):
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class FakeResponse:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, Any]:
|
||||
return {"skills": [{"id": "k1", "name": "K", "kind": "knowledge"}]}
|
||||
|
||||
class FakeHttpxClient:
|
||||
def get(self, url, *, params=None, headers=None, timeout=None):
|
||||
captured["url"] = url
|
||||
captured["params"] = params
|
||||
captured["headers"] = headers
|
||||
captured["timeout"] = timeout
|
||||
return FakeResponse()
|
||||
|
||||
def close(self) -> None:
|
||||
captured["closed"] = True
|
||||
|
||||
fake_http = FakeHttpxClient()
|
||||
client = HttpSubscriptionClient(
|
||||
base_url="https://subscription.example.com/",
|
||||
auth_token="secret",
|
||||
client=fake_http,
|
||||
)
|
||||
delta = client.fetch_entitled_skills("sub-a")
|
||||
assert captured["url"] == "https://subscription.example.com/subscriptions/sub-a/skills"
|
||||
assert captured["headers"] == {"Authorization": "Bearer secret"}
|
||||
assert len(delta.skills) == 1
|
||||
assert isinstance(delta.skills[0], KnowledgeSkill)
|
||||
client.close()
|
||||
assert captured.get("closed") is True
|
||||
|
||||
|
||||
def test_http_client_raises_on_status_error_propagates_to_runner(store):
|
||||
"""HTTP errors surface as a failed SyncOutcome, not a raised exception."""
|
||||
|
||||
class BoomResponse:
|
||||
def raise_for_status(self):
|
||||
raise RuntimeError("simulated 500")
|
||||
|
||||
class FailingClient:
|
||||
def get(self, url, *, params=None, headers=None, timeout=None):
|
||||
return BoomResponse()
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
http_client = HttpSubscriptionClient(
|
||||
base_url="https://subscription.example.com",
|
||||
client=FailingClient(),
|
||||
)
|
||||
runner = SkillSyncRunner(
|
||||
store=store,
|
||||
client=http_client,
|
||||
subscriptions=["sub-a"],
|
||||
)
|
||||
outcomes = runner.tick()
|
||||
assert outcomes["sub-a"].success is False
|
||||
assert "simulated 500" in (outcomes["sub-a"].error or "")
|
||||
Reference in New Issue
Block a user