72 lines
2.4 KiB
Python
72 lines
2.4 KiB
Python
"""Integration test for Verifier/Reflector against the real LLM client (task 8.2).
|
|
Skippable without network/API credentials."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from agents.models import Observation, VerificationVerdict
|
|
from agents.reflector import Reflector
|
|
from agents.verifier import Verifier
|
|
from runtime.executor import StepResult
|
|
from runtime.planner import PlannedStep
|
|
from semantic.llm_client import AnthropicSemanticClient
|
|
|
|
_has_api_key = bool(os.environ.get("ANTHROPIC_API_KEY"))
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.skipif(not _has_api_key, reason="ANTHROPIC_API_KEY not set")
|
|
def test_verifier_against_real_llm() -> None:
|
|
client = AnthropicSemanticClient()
|
|
verifier = Verifier(client=client)
|
|
|
|
pre = Observation(
|
|
scene_summary="Screen showing a chat app with a 'Reply' button visible.",
|
|
semantic_page="chat",
|
|
semantic_intents=["send_message"],
|
|
)
|
|
post = Observation(
|
|
scene_summary="Screen showing a chat app with 'Message sent' confirmation visible.",
|
|
semantic_page="chat",
|
|
semantic_intents=["send_message"],
|
|
)
|
|
step = PlannedStep(action="tap", description="Tap the send button", args={})
|
|
result = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
|
|
|
|
verdict = verifier.verify(
|
|
pre_observation=pre,
|
|
post_observation=post,
|
|
planned_step=step,
|
|
step_result=result,
|
|
)
|
|
assert verdict.result in ("achieved", "not_achieved")
|
|
assert verdict.reasoning
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.skipif(not _has_api_key, reason="ANTHROPIC_API_KEY not set")
|
|
def test_reflector_against_real_llm() -> None:
|
|
client = AnthropicSemanticClient()
|
|
reflector = Reflector(client=client)
|
|
|
|
observation = Observation(
|
|
scene_summary="Screen unchanged after tapping send button.",
|
|
semantic_page="chat",
|
|
semantic_intents=["send_message"],
|
|
)
|
|
step = PlannedStep(action="tap", description="Tap the send button", args={})
|
|
result = StepResult(step=step, success=True, attempts=1, result={"tapped": True})
|
|
verdict = VerificationVerdict(result="not_achieved", reasoning="Screen unchanged.")
|
|
|
|
outcome = reflector.reflect(
|
|
observation=observation,
|
|
planned_step=step,
|
|
step_result=result,
|
|
verdict=verdict,
|
|
)
|
|
assert outcome.replan is True or outcome.action is not None
|
|
assert outcome.reasoning
|