"""Integration test for Verifier/Reflector against the real LLM client (task 8.2). Skippable without network/API credentials.""" from __future__ import annotations import os import pytest from agents.models import Observation, VerificationVerdict from agents.reflector import Reflector from agents.verifier import Verifier from runtime.executor import StepResult from runtime.planner import PlannedStep from semantic.llm_client import AnthropicSemanticClient _has_api_key = bool(os.environ.get("ANTHROPIC_API_KEY")) @pytest.mark.integration @pytest.mark.skipif(not _has_api_key, reason="ANTHROPIC_API_KEY not set") def test_verifier_against_real_llm() -> None: client = AnthropicSemanticClient() verifier = Verifier(client=client) pre = Observation( scene_summary="Screen showing a chat app with a 'Reply' button visible.", semantic_page="chat", semantic_intents=["send_message"], ) post = Observation( scene_summary="Screen showing a chat app with 'Message sent' confirmation visible.", semantic_page="chat", semantic_intents=["send_message"], ) step = PlannedStep(action="tap", description="Tap the send button", args={}) result = StepResult(step=step, success=True, attempts=1, result={"tapped": True}) verdict = verifier.verify( pre_observation=pre, post_observation=post, planned_step=step, step_result=result, ) assert verdict.result in ("achieved", "not_achieved") assert verdict.reasoning @pytest.mark.integration @pytest.mark.skipif(not _has_api_key, reason="ANTHROPIC_API_KEY not set") def test_reflector_against_real_llm() -> None: client = AnthropicSemanticClient() reflector = Reflector(client=client) observation = Observation( scene_summary="Screen unchanged after tapping send button.", semantic_page="chat", semantic_intents=["send_message"], ) step = PlannedStep(action="tap", description="Tap the send button", args={}) result = StepResult(step=step, success=True, attempts=1, result={"tapped": True}) verdict = VerificationVerdict(result="not_achieved", reasoning="Screen unchanged.") outcome = reflector.reflect( observation=observation, planned_step=step, step_result=result, verdict=verdict, ) assert outcome.replan is True or outcome.action is not None assert outcome.reasoning