multi agent

tests/test_agents_collab_runner.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+from agents.collab_runner import CollaborativeTaskRunner, CollaborativeTaskRunnerConfig
+from agents.config import CollaborationConfig
+from agents.models import Observation, ReflectionAction, ReflectionOutcome, VerificationVerdict
+from core.models import Bounds, Scene, SceneElement, Task
+from runtime.executor import StepResult
+from runtime.planner import PlannedStep
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=1080,
+        height=1920,
+        elements=[SceneElement(id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="OK")],
+    )
+
+
+def _task() -> Task:
+    return Task(goal="Test goal", device_id="dev1")
+
+
+def _planned_step() -> PlannedStep:
+    return PlannedStep(action="describe_screen", description="Observe", args={})
+
+
+def _success_step_result() -> StepResult:
+    return StepResult(step=_planned_step(), success=True, attempts=1, result="ok")
+
+
+def _observation() -> Observation:
+    return Observation(scene_summary="Screen 1080x1920")
+
+
+def _achieved_verdict() -> VerificationVerdict:
+    return VerificationVerdict(result="achieved", reasoning="Effect observed.")
+
+
+def _not_achieved_verdict() -> VerificationVerdict:
+    return VerificationVerdict(result="not_achieved", reasoning="No change.")
+
+
+def _replan_outcome() -> ReflectionOutcome:
+    return ReflectionOutcome(replan=True, reasoning="No recovery.")
+
+
+def _recovery_outcome() -> ReflectionOutcome:
+    return ReflectionOutcome(
+        replan=False,
+        action=ReflectionAction(action="swipe", description="Scroll", args={"direction": "up"}),
+        reasoning="Try scrolling.",
+    )
+
+
+def test_completes_without_verification_failure() -> None:
+    """Task completes normally when verifier reports achieved."""
+    planner = MagicMock()
+    planner.plan.return_value = [_planned_step()]
+    planner.goal_reached.return_value = True
+
+    executor = MagicMock()
+    executor.execute.return_value = _success_step_result()
+
+    observer = MagicMock()
+    observer.observe.return_value = _observation()
+
+    verifier = MagicMock()
+    verifier.verify.return_value = _achieved_verdict()
+
+    reflector = MagicMock()
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        observer=observer,
+        verifier=verifier,
+        reflector=reflector,
+        config=CollaborativeTaskRunnerConfig(max_steps=5),
+        collaboration_config=CollaborationConfig(enabled=True, max_recovery_attempts=3),
+    )
+
+    with patch("tools.describe_screen.describe_screen", return_value=_scene()):
+        task = runner.run(_task())
+
+    assert task.status == "completed"
+    reflector.reflect.assert_not_called()
+
+
+def test_recovers_via_reflector_action() -> None:
+    """Task recovers when reflector proposes a distinct action."""
+    call_count = 0
+
+    def verifier_side_effect(**kwargs: Any) -> VerificationVerdict:
+        nonlocal call_count
+        call_count += 1
+        if call_count == 1:
+            return _not_achieved_verdict()
+        return _achieved_verdict()
+
+    planner = MagicMock()
+    planner.plan.return_value = [_planned_step()]
+    planner.goal_reached.side_effect = [False, True]
+
+    executor = MagicMock()
+    executor.execute.return_value = _success_step_result()
+
+    observer = MagicMock()
+    observer.observe.return_value = _observation()
+
+    verifier = MagicMock()
+    verifier.verify.side_effect = verifier_side_effect
+
+    reflector = MagicMock()
+    reflector.reflect.return_value = _recovery_outcome()
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        observer=observer,
+        verifier=verifier,
+        reflector=reflector,
+        config=CollaborativeTaskRunnerConfig(max_steps=5),
+        collaboration_config=CollaborationConfig(enabled=True, max_recovery_attempts=3),
+    )
+
+    with patch("tools.describe_screen.describe_screen", return_value=_scene()):
+        task = runner.run(_task())
+
+    assert task.status == "completed"
+    reflector.reflect.assert_called_once()
+    executor.execute.assert_called()  # recovery action also executed
+
+
+def test_exhausts_recovery_ceiling() -> None:
+    """Task fails when reflection-recovery ceiling is reached."""
+    planner = MagicMock()
+    planner.plan.return_value = [_planned_step()]
+    planner.goal_reached.return_value = False
+
+    executor = MagicMock()
+    executor.execute.return_value = _success_step_result()
+
+    observer = MagicMock()
+    observer.observe.return_value = _observation()
+
+    verifier = MagicMock()
+    verifier.verify.return_value = _not_achieved_verdict()
+
+    reflector = MagicMock()
+    reflector.reflect.return_value = _replan_outcome()
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        observer=observer,
+        verifier=verifier,
+        reflector=reflector,
+        config=CollaborativeTaskRunnerConfig(max_steps=10, max_recovery_attempts=2),
+        collaboration_config=CollaborationConfig(enabled=True, max_recovery_attempts=2),
+    )
+
+    with patch("tools.describe_screen.describe_screen", return_value=_scene()):
+        task = runner.run(_task())
+
+    assert task.status == "failed"
+    assert "ceiling" in (task.failure_reason or "").lower()
+
+
+def test_disabled_collaboration_runs_plain() -> None:
+    """When collaboration is disabled, delegates to plain TaskRunner."""
+    planner = MagicMock()
+    executor = MagicMock()
+
+    mock_task_runner = MagicMock()
+    expected_task = _task()
+    expected_task.status = "completed"
+    mock_task_runner.run.return_value = expected_task
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        task_runner=mock_task_runner,
+        collaboration_config=CollaborationConfig(enabled=False),
+    )
+
+    result = runner.run(_task())
+    assert result.status == "completed"
+    mock_task_runner.run.assert_called_once()

tests/test_agents_e2e.py

@@ -0,0 +1,141 @@
+"""End-to-end tests for multi-agent collaboration (task 8.1)."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from agents.collab_runner import CollaborativeTaskRunner, CollaborativeTaskRunnerConfig
+from agents.config import CollaborationConfig
+from agents.models import Observation, ReflectionAction, ReflectionOutcome, VerificationVerdict
+from agents.observer import Observer
+from agents.reflector import Reflector
+from agents.verifier import Verifier
+from core.models import Bounds, Scene, SceneElement, Task
+from runtime.executor import StepResult
+from runtime.planner import PlannedStep
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=1080,
+        height=1920,
+        elements=[
+            SceneElement(id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="Send"),
+        ],
+    )
+
+
+def _task() -> Task:
+    return Task(goal="Send a message", device_id="dev1")
+
+
+def test_full_loop_verification_succeeds() -> None:
+    """Full collaborative loop: plan → execute → verify (achieved) → complete."""
+    planner = MagicMock()
+    step = PlannedStep(action="tap", description="Tap send", args={"element_id": "btn1"})
+    planner.plan.return_value = [step]
+    planner.goal_reached.side_effect = [False, True]
+
+    executor = MagicMock()
+    result = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
+    executor.execute.return_value = result
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        observer=Observer(),
+        verifier=Verifier(),
+        reflector=Reflector(),
+        config=CollaborativeTaskRunnerConfig(max_steps=5),
+        collaboration_config=CollaborationConfig(enabled=True, max_recovery_attempts=3),
+    )
+
+    scene = _scene()
+    with patch("tools.describe_screen.describe_screen", return_value=scene):
+        task = runner.run(_task())
+
+    assert task.status == "completed"
+
+
+def test_full_loop_with_reflection_recovery() -> None:
+    """Full loop: plan → execute → verify (not achieved) → reflect (recovery) → execute recovery → verify (achieved)."""
+    planner = MagicMock()
+    step = PlannedStep(action="tap", description="Tap send", args={"element_id": "btn1"})
+    planner.plan.return_value = [step]
+    planner.goal_reached.side_effect = [False, False, True]
+
+    executor = MagicMock()
+    success = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
+    executor.execute.return_value = success
+
+    # First verify: not achieved; second verify: achieved
+    verify_call_count = 0
+
+    def verifier_verify(**kwargs):
+        nonlocal verify_call_count
+        verify_call_count += 1
+        if verify_call_count == 1:
+            return VerificationVerdict(result="not_achieved", reasoning="No change.")
+        return VerificationVerdict(result="achieved", reasoning="Done.")
+
+    verifier = MagicMock()
+    verifier.verify.side_effect = verifier_verify
+
+    reflector = MagicMock()
+    reflector.reflect.return_value = ReflectionOutcome(
+        replan=False,
+        action=ReflectionAction(action="swipe", description="Scroll", args={"direction": "up"}),
+        reasoning="Try scrolling.",
+    )
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        observer=Observer(),
+        verifier=verifier,
+        reflector=reflector,
+        config=CollaborativeTaskRunnerConfig(max_steps=5),
+        collaboration_config=CollaborationConfig(enabled=True, max_recovery_attempts=3),
+    )
+
+    scene = _scene()
+    with patch("tools.describe_screen.describe_screen", return_value=scene):
+        task = runner.run(_task())
+
+    assert task.status == "completed"
+    reflector.reflect.assert_called_once()
+
+
+def test_full_loop_exhausts_ceiling() -> None:
+    """Full loop fails when recovery ceiling is exhausted."""
+    planner = MagicMock()
+    step = PlannedStep(action="tap", description="Tap send", args={"element_id": "btn1"})
+    planner.plan.return_value = [step]
+    planner.goal_reached.return_value = False
+
+    executor = MagicMock()
+    success = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
+    executor.execute.return_value = success
+
+    verifier = MagicMock()
+    verifier.verify.return_value = VerificationVerdict(result="not_achieved", reasoning="No change.")
+
+    reflector = MagicMock()
+    reflector.reflect.return_value = ReflectionOutcome(replan=True, reasoning="Replan.")
+
+    runner = CollaborativeTaskRunner(
+        planner=planner,
+        executor=executor,
+        observer=Observer(),
+        verifier=verifier,
+        reflector=reflector,
+        config=CollaborativeTaskRunnerConfig(max_steps=10, max_recovery_attempts=2),
+        collaboration_config=CollaborationConfig(enabled=True, max_recovery_attempts=2),
+    )
+
+    scene = _scene()
+    with patch("tools.describe_screen.describe_screen", return_value=scene):
+        task = runner.run(_task())
+
+    assert task.status == "failed"
+    assert "ceiling" in (task.failure_reason or "").lower()

tests/test_agents_integration.py

@@ -0,0 +1,71 @@
+"""Integration test for Verifier/Reflector against the real LLM client (task 8.2).
+Skippable without network/API credentials."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from agents.models import Observation, VerificationVerdict
+from agents.reflector import Reflector
+from agents.verifier import Verifier
+from runtime.executor import StepResult
+from runtime.planner import PlannedStep
+from semantic.llm_client import AnthropicSemanticClient
+
+_has_api_key = bool(os.environ.get("ANTHROPIC_API_KEY"))
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(not _has_api_key, reason="ANTHROPIC_API_KEY not set")
+def test_verifier_against_real_llm() -> None:
+    client = AnthropicSemanticClient()
+    verifier = Verifier(client=client)
+
+    pre = Observation(
+        scene_summary="Screen showing a chat app with a 'Reply' button visible.",
+        semantic_page="chat",
+        semantic_intents=["send_message"],
+    )
+    post = Observation(
+        scene_summary="Screen showing a chat app with 'Message sent' confirmation visible.",
+        semantic_page="chat",
+        semantic_intents=["send_message"],
+    )
+    step = PlannedStep(action="tap", description="Tap the send button", args={})
+    result = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
+
+    verdict = verifier.verify(
+        pre_observation=pre,
+        post_observation=post,
+        planned_step=step,
+        step_result=result,
+    )
+    assert verdict.result in ("achieved", "not_achieved")
+    assert verdict.reasoning
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(not _has_api_key, reason="ANTHROPIC_API_KEY not set")
+def test_reflector_against_real_llm() -> None:
+    client = AnthropicSemanticClient()
+    reflector = Reflector(client=client)
+
+    observation = Observation(
+        scene_summary="Screen unchanged after tapping send button.",
+        semantic_page="chat",
+        semantic_intents=["send_message"],
+    )
+    step = PlannedStep(action="tap", description="Tap the send button", args={})
+    result = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
+    verdict = VerificationVerdict(result="not_achieved", reasoning="Screen unchanged.")
+
+    outcome = reflector.reflect(
+        observation=observation,
+        planned_step=step,
+        step_result=result,
+        verdict=verdict,
+    )
+    assert outcome.replan is True or outcome.action is not None
+    assert outcome.reasoning

tests/test_agents_models.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from agents.models import (
+    Observation,
+    ReflectionAction,
+    ReflectionOutcome,
+    VerificationVerdict,
+)
+
+
+def test_observation_round_trip() -> None:
+    obs = Observation(
+        scene_summary="Screen 1080x1920\n  [button] Send",
+        semantic_page="chat",
+        semantic_intents=["send_message"],
+        world_app="com.example.messenger",
+        world_page="conversation",
+        world_variables={"draft": "hello"},
+        raw_scene={"screen": {"width": 1080, "height": 1920}, "elements": []},
+    )
+    data = obs.to_dict()
+    restored = Observation.from_dict(data)
+    assert restored == obs
+
+
+def test_observation_minimal_round_trip() -> None:
+    obs = Observation(scene_summary="empty")
+    data = obs.to_dict()
+    restored = Observation.from_dict(data)
+    assert restored.scene_summary == "empty"
+    assert restored.semantic_page is None
+    assert restored.semantic_intents == []
+    assert restored.world_app is None
+    assert restored.world_page is None
+    assert restored.world_variables == {}
+    assert restored.raw_scene is None
+
+
+def test_verification_verdict_achieved_round_trip() -> None:
+    verdict = VerificationVerdict(result="achieved", reasoning="Effect observed.")
+    data = verdict.to_dict()
+    restored = VerificationVerdict.from_dict(data)
+    assert restored == verdict
+
+
+def test_verification_verdict_not_achieved_round_trip() -> None:
+    verdict = VerificationVerdict(result="not_achieved", reasoning="Scene unchanged.")
+    data = verdict.to_dict()
+    restored = VerificationVerdict.from_dict(data)
+    assert restored == verdict
+
+
+def test_reflection_action_round_trip() -> None:
+    action = ReflectionAction(
+        action="tap",
+        description="Tap the send button",
+        args={"element_id": "btn_send"},
+    )
+    data = action.to_dict()
+    restored = ReflectionAction.from_dict(data)
+    assert restored == action
+
+
+def test_reflection_outcome_with_action_round_trip() -> None:
+    action = ReflectionAction(action="tap", description="Retry tap", args={})
+    outcome = ReflectionOutcome(replan=False, action=action, reasoning="Try again.")
+    data = outcome.to_dict()
+    restored = ReflectionOutcome.from_dict(data)
+    assert restored == outcome
+
+
+def test_reflection_outcome_replan_round_trip() -> None:
+    outcome = ReflectionOutcome(replan=True, reasoning="No recovery possible.")
+    data = outcome.to_dict()
+    restored = ReflectionOutcome.from_dict(data)
+    assert restored == outcome
+    assert restored.action is None

tests/test_agents_no_coupling.py

@@ -0,0 +1,36 @@
+"""Verify that the agents/ package does not introduce accidental coupling
+into the existing runtime/ modules (task 7.2)."""
+
+from __future__ import annotations
+
+import importlib
+
+
+def test_runtime_planner_unchanged() -> None:
+    """Planner should not import from agents/."""
+    module = importlib.import_module("runtime.planner")
+    source = open(module.__file__).read()  # type: ignore[arg-type]
+    assert "agents" not in source
+
+
+def test_runtime_executor_unchanged() -> None:
+    """Executor should not import from agents/."""
+    module = importlib.import_module("runtime.executor")
+    source = open(module.__file__).read()  # type: ignore[arg-type]
+    assert "agents" not in source
+
+
+def test_runtime_task_unchanged() -> None:
+    """TaskRunner should not import from agents/."""
+    module = importlib.import_module("runtime.task")
+    source = open(module.__file__).read()  # type: ignore[arg-type]
+    assert "agents" not in source
+
+
+def test_agents_imports_runtime() -> None:
+    """agents/ should import from runtime/, not the other way around."""
+    collab = importlib.import_module("agents.collab_runner")
+    source = open(collab.__file__).read()  # type: ignore[arg-type]
+    assert "runtime.task" in source
+    assert "runtime.planner" in source
+    assert "runtime.executor" in source

tests/test_agents_observer.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from collections import deque
+
+from agents.observer import Observer
+from core.models import Bounds, Scene, SceneElement
+from semantic.models import SemanticScene, SemanticWidget
+from world.models import WorldState
+
+
+def _make_scene() -> Scene:
+    return Scene(
+        width=1080,
+        height=1920,
+        elements=[
+            SceneElement(id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="Send"),
+            SceneElement(id="txt1", type="text", bounds=Bounds(0, 0, 500, 30), text="Hello"),
+        ],
+    )
+
+
+def _make_semantic_scene() -> SemanticScene:
+    return SemanticScene(
+        page="chat",
+        intents=["send_message"],
+        widgets=[SemanticWidget(element_id="btn1", purpose="send button")],
+    )
+
+
+def _make_world() -> WorldState:
+    state = WorldState(
+        current_app="com.example.messenger",
+        current_page="conversation",
+        variables={"draft": "hello"},
+    )
+    return state
+
+
+def test_observe_with_semantic_and_world() -> None:
+    obs = Observer().observe(
+        scene=_make_scene(),
+        semantic_scene=_make_semantic_scene(),
+        world=_make_world(),
+    )
+    assert obs.semantic_page == "chat"
+    assert obs.semantic_intents == ["send_message"]
+    assert obs.world_app == "com.example.messenger"
+    assert obs.world_page == "conversation"
+    assert obs.world_variables == {"draft": "hello"}
+    assert obs.raw_scene is not None
+
+
+def test_observe_without_semantic_or_world() -> None:
+    obs = Observer().observe(scene=_make_scene())
+    assert obs.semantic_page is None
+    assert obs.semantic_intents == []
+    assert obs.world_app is None
+    assert obs.world_page is None
+    assert obs.world_variables == {}
+    assert obs.scene_summary is not None
+    assert obs.raw_scene is not None
+
+
+def test_observe_with_semantic_only() -> None:
+    obs = Observer().observe(
+        scene=_make_scene(),
+        semantic_scene=_make_semantic_scene(),
+    )
+    assert obs.semantic_page == "chat"
+    assert obs.world_app is None
+
+
+def test_observe_with_world_only() -> None:
+    obs = Observer().observe(
+        scene=_make_scene(),
+        world=_make_world(),
+    )
+    assert obs.semantic_page is None
+    assert obs.world_app == "com.example.messenger"
+
+
+def test_observe_scene_summary_includes_elements() -> None:
+    obs = Observer().observe(scene=_make_scene())
+    assert "1080x1920" in obs.scene_summary
+    assert "Send" in obs.scene_summary

tests/test_agents_reflector.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from agents.models import Observation, ReflectionOutcome, VerificationVerdict
+from agents.reflector import Reflector
+from runtime.executor import StepResult
+from runtime.planner import PlannedStep
+
+
+def _observation() -> Observation:
+    return Observation(scene_summary="Screen 1080x1920\n  [button] Reply")
+
+
+def _planned_step() -> PlannedStep:
+    return PlannedStep(
+        action="tap",
+        description="Tap the send button",
+        args={"element_id": "btn_send"},
+    )
+
+
+def _success_result() -> StepResult:
+    return StepResult(
+        step=_planned_step(),
+        success=True,
+        attempts=1,
+        result={"tapped": True},
+    )
+
+
+def _not_achieved_verdict() -> VerificationVerdict:
+    return VerificationVerdict(result="not_achieved", reasoning="Scene unchanged.")
+
+
+def test_reflect_fallback_returns_replan() -> None:
+    reflector = Reflector()
+    outcome = reflector.reflect(
+        observation=_observation(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+        verdict=_not_achieved_verdict(),
+    )
+    assert outcome.replan is True
+    assert outcome.action is None
+    assert "fallback" in outcome.reasoning.lower()
+
+
+class _FakeClient:
+    """A fake LLM client that returns a canned outcome."""
+
+    def __init__(self, outcome: dict) -> None:
+        self._outcome = outcome
+        self._calls: list[dict] = []
+
+    def _create_message(self, payload: dict, *, timeout: float) -> dict:
+        self._calls.append(payload)
+        return self._outcome
+
+
+def test_reflect_recovery_action_via_llm() -> None:
+    fake = _FakeClient({
+        "replan": False,
+        "action": {
+            "action": "swipe",
+            "description": "Scroll down to find send button",
+            "args": {"direction": "up"},
+        },
+        "reasoning": "Button may be off-screen.",
+    })
+    reflector = Reflector(client=fake)  # type: ignore[arg-type]
+    outcome = reflector.reflect(
+        observation=_observation(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+        verdict=_not_achieved_verdict(),
+    )
+    assert outcome.replan is False
+    assert outcome.action is not None
+    assert outcome.action.action == "swipe"
+    assert len(fake._calls) == 1
+
+
+def test_reflect_replan_via_llm() -> None:
+    fake = _FakeClient({
+        "replan": True,
+        "action": None,
+        "reasoning": "No bounded recovery possible.",
+    })
+    reflector = Reflector(client=fake)  # type: ignore[arg-type]
+    outcome = reflector.reflect(
+        observation=_observation(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+        verdict=_not_achieved_verdict(),
+    )
+    assert outcome.replan is True
+    assert outcome.action is None
+
+
+def test_reflect_rejects_identical_action() -> None:
+    """If the LLM proposes the exact same step, reflector should force replan."""
+    fake = _FakeClient({
+        "replan": False,
+        "action": {
+            "action": "tap",
+            "description": "Tap the send button",
+            "args": {"element_id": "btn_send"},
+        },
+        "reasoning": "Try again.",
+    })
+    reflector = Reflector(client=fake)  # type: ignore[arg-type]
+    outcome = reflector.reflect(
+        observation=_observation(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+        verdict=_not_achieved_verdict(),
+    )
+    assert outcome.replan is True
+    assert "identical" in outcome.reasoning.lower()
+
+
+def test_reflect_degrades_on_client_failure() -> None:
+    from semantic.llm_client import EnrichmentUnavailable
+
+    class _UnavailableClient:
+        def _create_message(self, *args, **kwargs):  # type: ignore[no-untyped-def]
+            raise EnrichmentUnavailable("timeout")
+
+    reflector = Reflector(client=_UnavailableClient())  # type: ignore[arg-type]
+    outcome = reflector.reflect(
+        observation=_observation(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+        verdict=_not_achieved_verdict(),
+    )
+    assert outcome.replan is True
+    assert "fallback" in outcome.reasoning.lower()

tests/test_agents_verifier.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+from agents.models import Observation, VerificationVerdict
+from agents.verifier import Verifier
+from runtime.executor import StepResult
+from runtime.planner import PlannedStep
+
+
+def _pre_observation() -> Observation:
+    return Observation(scene_summary="Screen 1080x1920\n  [button] Reply")
+
+
+def _post_observation_changed() -> Observation:
+    return Observation(scene_summary="Screen 1080x1920\n  [text] Message sent")
+
+
+def _post_observation_unchanged() -> Observation:
+    return Observation(scene_summary="Screen 1080x1920\n  [button] Reply")
+
+
+def _planned_step() -> PlannedStep:
+    return PlannedStep(
+        action="tap",
+        description="Tap the send button",
+        args={"element_id": "btn_send"},
+    )
+
+
+def _success_result() -> StepResult:
+    return StepResult(
+        step=_planned_step(),
+        success=True,
+        attempts=1,
+        result={"tapped": True},
+    )
+
+
+def _failure_result() -> StepResult:
+    return StepResult(
+        step=_planned_step(),
+        success=False,
+        attempts=3,
+        error="element not found",
+    )
+
+
+def test_verify_not_achieved_on_mechanical_failure() -> None:
+    verifier = Verifier()
+    verdict = verifier.verify(
+        pre_observation=_pre_observation(),
+        post_observation=_post_observation_changed(),
+        planned_step=_planned_step(),
+        step_result=_failure_result(),
+    )
+    assert verdict.result == "not_achieved"
+    assert "failed mechanically" in verdict.reasoning.lower()
+
+
+def test_verify_achieved_fallback_scene_changed() -> None:
+    verifier = Verifier()
+    verdict = verifier.verify(
+        pre_observation=_pre_observation(),
+        post_observation=_post_observation_changed(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+    )
+    assert verdict.result == "achieved"
+
+
+def test_verify_not_achieved_fallback_scene_unchanged() -> None:
+    verifier = Verifier()
+    verdict = verifier.verify(
+        pre_observation=_pre_observation(),
+        post_observation=_post_observation_unchanged(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+    )
+    assert verdict.result == "not_achieved"
+
+
+class _FakeClient:
+    """A fake LLM client that returns a canned verdict."""
+
+    def __init__(self, verdict: dict) -> None:
+        self._verdict = verdict
+        self._calls: list[dict] = []
+
+    def _create_message(self, payload: dict, *, timeout: float) -> dict:
+        self._calls.append(payload)
+        return self._verdict
+
+
+def test_verify_achieved_via_llm() -> None:
+    fake = _FakeClient({"result": "achieved", "reasoning": "Effect observed."})
+    verifier = Verifier(client=fake)  # type: ignore[arg-type]
+    verdict = verifier.verify(
+        pre_observation=_pre_observation(),
+        post_observation=_post_observation_changed(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+    )
+    assert verdict.result == "achieved"
+    assert len(fake._calls) == 1
+
+
+def test_verify_not_achieved_via_llm() -> None:
+    fake = _FakeClient({"result": "not_achieved", "reasoning": "No change."})
+    verifier = Verifier(client=fake)  # type: ignore[arg-type]
+    verdict = verifier.verify(
+        pre_observation=_pre_observation(),
+        post_observation=_post_observation_unchanged(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+    )
+    assert verdict.result == "not_achieved"
+
+
+def test_verify_degrades_on_client_failure() -> None:
+    class _BrokenClient:
+        def _create_message(self, *args, **kwargs):  # type: ignore[no-untyped-def]
+            raise RuntimeError("connection failed")
+
+    from semantic.llm_client import EnrichmentUnavailable
+
+    class _UnavailableClient:
+        def _create_message(self, *args, **kwargs):  # type: ignore[no-untyped-def]
+            raise EnrichmentUnavailable("timeout")
+
+    verifier = Verifier(client=_UnavailableClient())  # type: ignore[arg-type]
+    verdict = verifier.verify(
+        pre_observation=_pre_observation(),
+        post_observation=_post_observation_changed(),
+        planned_step=_planned_step(),
+        step_result=_success_result(),
+    )
+    assert verdict.result == "achieved"
+    assert "fallback" in verdict.reasoning.lower()

tests/test_smoke.py

@@ -5,6 +5,7 @@ import importlib
 
 def test_imports_new_packages() -> None:
     for package in (
+        "agents",
         "api",
         "core",
         "device",