multi agent
This commit is contained in:
@@ -0,0 +1 @@
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
from agents.config import CollaborationConfig, load_config
|
||||
from agents.models import Observation, VerificationVerdict
|
||||
from agents.observer import Observer
|
||||
from agents.reflector import Reflector
|
||||
from agents.verifier import Verifier
|
||||
from core.models import Scene, Task, utc_now
|
||||
from runtime.context import TaskContext
|
||||
from runtime.executor import Executor
|
||||
from runtime.planner import PlannedStep, Planner
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from semantic.llm_client import AnthropicSemanticClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollaborativeTaskRunnerConfig:
|
||||
max_steps: int = 20
|
||||
max_recovery_attempts: int = 3
|
||||
|
||||
|
||||
class CollaborativeTaskRunner:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
planner: Planner | None = None,
|
||||
executor: Executor | None = None,
|
||||
task_runner: TaskRunner | None = None,
|
||||
observer: Observer | None = None,
|
||||
verifier: Verifier | None = None,
|
||||
reflector: Reflector | None = None,
|
||||
config: CollaborativeTaskRunnerConfig | None = None,
|
||||
collaboration_config: CollaborationConfig | None = None,
|
||||
llm_client: AnthropicSemanticClient | None = None,
|
||||
) -> None:
|
||||
self.planner = planner or Planner()
|
||||
self.executor = executor or Executor()
|
||||
self.task_runner = task_runner
|
||||
self.observer = observer or Observer()
|
||||
self.verifier = verifier or Verifier(client=llm_client)
|
||||
self.reflector = reflector or Reflector(client=llm_client)
|
||||
self.config = config or CollaborativeTaskRunnerConfig()
|
||||
self.collaboration_config = collaboration_config or load_config()
|
||||
|
||||
def run(self, task: Task) -> Task:
|
||||
if not self.collaboration_config.enabled:
|
||||
return self._run_plain(task)
|
||||
return self._run_collaborative(task)
|
||||
|
||||
def _run_plain(self, task: Task) -> Task:
|
||||
runner = self.task_runner or TaskRunner(
|
||||
planner=self.planner,
|
||||
executor=self.executor,
|
||||
config=TaskRunnerConfig(max_steps=self.config.max_steps),
|
||||
)
|
||||
return runner.run(task)
|
||||
|
||||
def _run_collaborative(self, task: Task) -> Task:
|
||||
context = TaskContext(task_id=task.id, goal=task.goal)
|
||||
task.status = "running" # type: ignore[assignment]
|
||||
task.updated_at = utc_now()
|
||||
recovery_attempts = 0
|
||||
|
||||
for _ in range(self.config.max_steps):
|
||||
scene = self._observe_scene(task.device_id)
|
||||
context.add_scene(scene)
|
||||
|
||||
pre_observation = self.observer.observe(
|
||||
scene=scene,
|
||||
world=context.world,
|
||||
)
|
||||
|
||||
steps = self.planner.plan(
|
||||
goal=task.goal,
|
||||
scene=scene,
|
||||
context=context,
|
||||
)
|
||||
if not steps or self.planner.goal_reached(
|
||||
goal=task.goal,
|
||||
scene=scene,
|
||||
context=context,
|
||||
):
|
||||
task.status = "completed" # type: ignore[assignment]
|
||||
task.updated_at = utc_now()
|
||||
task.completed_at = utc_now()
|
||||
return task
|
||||
|
||||
for step in steps:
|
||||
executable_step = self._step_for_device(step, task.device_id)
|
||||
result = self.executor.execute(executable_step, context=context)
|
||||
context.add_step_result(result)
|
||||
|
||||
post_scene = self._observe_scene(task.device_id)
|
||||
post_observation = self.observer.observe(
|
||||
scene=post_scene,
|
||||
world=context.world,
|
||||
)
|
||||
|
||||
verdict = self.verifier.verify(
|
||||
pre_observation=pre_observation,
|
||||
post_observation=post_observation,
|
||||
planned_step=step,
|
||||
step_result=result,
|
||||
)
|
||||
|
||||
if verdict.result == "achieved":
|
||||
continue
|
||||
|
||||
if recovery_attempts >= self.collaboration_config.max_recovery_attempts:
|
||||
task.status = "failed" # type: ignore[assignment]
|
||||
task.updated_at = utc_now()
|
||||
task.completed_at = utc_now()
|
||||
task.failure_reason = (
|
||||
f"Reflection recovery ceiling reached "
|
||||
f"({self.collaboration_config.max_recovery_attempts} attempts)"
|
||||
)
|
||||
return task
|
||||
|
||||
outcome = self.reflector.reflect(
|
||||
observation=post_observation,
|
||||
planned_step=step,
|
||||
step_result=result,
|
||||
verdict=verdict,
|
||||
)
|
||||
recovery_attempts += 1
|
||||
|
||||
if outcome.replan:
|
||||
break
|
||||
|
||||
if outcome.action is not None:
|
||||
recovery_step = PlannedStep(
|
||||
action=outcome.action.action,
|
||||
description=outcome.action.description,
|
||||
args=outcome.action.args,
|
||||
)
|
||||
recovery_result = self.executor.execute(
|
||||
self._step_for_device(recovery_step, task.device_id),
|
||||
context=context,
|
||||
)
|
||||
context.add_step_result(recovery_result)
|
||||
if not recovery_result.success:
|
||||
task.status = "failed" # type: ignore[assignment]
|
||||
task.updated_at = utc_now()
|
||||
task.completed_at = utc_now()
|
||||
task.failure_reason = (
|
||||
f"Recovery action failed: {recovery_result.error}"
|
||||
)
|
||||
return task
|
||||
|
||||
task.status = "failed" # type: ignore[assignment]
|
||||
task.updated_at = utc_now()
|
||||
task.completed_at = utc_now()
|
||||
task.failure_reason = f"max steps exceeded: {self.config.max_steps}"
|
||||
return task
|
||||
|
||||
def _observe_scene(self, device_id: str) -> Scene:
|
||||
from tools.describe_screen import describe_screen
|
||||
|
||||
return describe_screen(device_id)
|
||||
|
||||
def _step_for_device(self, step: PlannedStep, device_id: str) -> PlannedStep:
|
||||
device_scoped_actions = {
|
||||
"take_screenshot",
|
||||
"screenshot",
|
||||
"tap",
|
||||
"swipe",
|
||||
"input_text",
|
||||
"launch_app",
|
||||
"terminate_app",
|
||||
"get_ui_tree",
|
||||
"ui_tree",
|
||||
"describe_screen",
|
||||
"describe_screen_semantic",
|
||||
"find_text_on_screen",
|
||||
"find_icon_on_screen",
|
||||
}
|
||||
if step.action not in device_scoped_actions or "device_id" in step.args:
|
||||
return step
|
||||
return replace(step, args={**step.args, "device_id": device_id})
|
||||
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
ENABLED_ENV = "MULTI_AGENT_COLLABORATION_ENABLED"
|
||||
MAX_RECOVERY_ENV = "MULTI_AGENT_MAX_RECOVERY_ATTEMPTS"
|
||||
DEFAULT_MAX_RECOVERY_ATTEMPTS = 3
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CollaborationConfig:
|
||||
enabled: bool = False
|
||||
max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS
|
||||
|
||||
|
||||
def load_config(env: Mapping[str, str] | None = None) -> CollaborationConfig:
|
||||
values = env or os.environ
|
||||
return CollaborationConfig(
|
||||
enabled=_parse_bool(values.get(ENABLED_ENV), default=False),
|
||||
max_recovery_attempts=_parse_int(
|
||||
values.get(MAX_RECOVERY_ENV), default=DEFAULT_MAX_RECOVERY_ATTEMPTS
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _parse_bool(value: str | None, *, default: bool) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on", "enabled"}
|
||||
|
||||
|
||||
def _parse_int(value: str | None, *, default: int) -> int:
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
result = int(value)
|
||||
except ValueError:
|
||||
return default
|
||||
return result if result > 0 else default
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
@dataclass
|
||||
class Observation:
|
||||
scene_summary: str
|
||||
semantic_page: str | None = None
|
||||
semantic_intents: list[str] = field(default_factory=list)
|
||||
world_app: str | None = None
|
||||
world_page: str | None = None
|
||||
world_variables: dict[str, Any] = field(default_factory=dict)
|
||||
raw_scene: dict[str, Any] | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"scene_summary": self.scene_summary,
|
||||
"semantic_page": self.semantic_page,
|
||||
"semantic_intents": list(self.semantic_intents),
|
||||
"world_app": self.world_app,
|
||||
"world_page": self.world_page,
|
||||
"world_variables": dict(self.world_variables),
|
||||
"raw_scene": self.raw_scene,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> Observation:
|
||||
return cls(
|
||||
scene_summary=str(data["scene_summary"]),
|
||||
semantic_page=data.get("semantic_page"),
|
||||
semantic_intents=list(data.get("semantic_intents", [])),
|
||||
world_app=data.get("world_app"),
|
||||
world_page=data.get("world_page"),
|
||||
world_variables=dict(data.get("world_variables", {})),
|
||||
raw_scene=data.get("raw_scene"),
|
||||
)
|
||||
|
||||
|
||||
VerdictResult = Literal["achieved", "not_achieved"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class VerificationVerdict:
|
||||
result: VerdictResult
|
||||
reasoning: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"result": self.result,
|
||||
"reasoning": self.reasoning,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> VerificationVerdict:
|
||||
result = data["result"]
|
||||
if result not in ("achieved", "not_achieved"):
|
||||
raise ValueError(f"invalid verdict result: {result}")
|
||||
return cls(
|
||||
result=result,
|
||||
reasoning=str(data["reasoning"]),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReflectionAction:
|
||||
action: str
|
||||
description: str
|
||||
args: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"action": self.action,
|
||||
"description": self.description,
|
||||
"args": dict(self.args),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> ReflectionAction:
|
||||
return cls(
|
||||
action=str(data["action"]),
|
||||
description=str(data["description"]),
|
||||
args=dict(data.get("args", {})),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReflectionOutcome:
|
||||
replan: bool = False
|
||||
action: ReflectionAction | None = None
|
||||
reasoning: str = ""
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"replan": self.replan,
|
||||
"action": self.action.to_dict() if self.action else None,
|
||||
"reasoning": self.reasoning,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> ReflectionOutcome:
|
||||
action_data = data.get("action")
|
||||
return cls(
|
||||
replan=bool(data.get("replan", False)),
|
||||
action=ReflectionAction.from_dict(action_data) if action_data else None,
|
||||
reasoning=str(data.get("reasoning", "")),
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agents.models import Observation
|
||||
from core.models import Scene, Step
|
||||
from runtime.executor import StepResult
|
||||
from runtime.planner import PlannedStep
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic.models import SemanticScene
|
||||
from world.models import WorldState
|
||||
|
||||
|
||||
class Observer:
|
||||
def observe(
|
||||
self,
|
||||
*,
|
||||
scene: Scene,
|
||||
step: PlannedStep | None = None,
|
||||
result: StepResult | None = None,
|
||||
semantic_scene: SemanticScene | None = None,
|
||||
world: WorldState | None = None,
|
||||
) -> Observation:
|
||||
scene_summary = self._build_scene_summary(scene)
|
||||
semantic_page = None
|
||||
semantic_intents: list[str] = []
|
||||
if semantic_scene is not None:
|
||||
semantic_page = semantic_scene.page
|
||||
semantic_intents = list(semantic_scene.intents)
|
||||
|
||||
world_app = None
|
||||
world_page = None
|
||||
world_variables: dict[str, Any] = {}
|
||||
if world is not None:
|
||||
world_app = world.current_app
|
||||
world_page = world.current_page
|
||||
world_variables = dict(world.variables)
|
||||
|
||||
return Observation(
|
||||
scene_summary=scene_summary,
|
||||
semantic_page=semantic_page,
|
||||
semantic_intents=semantic_intents,
|
||||
world_app=world_app,
|
||||
world_page=world_page,
|
||||
world_variables=world_variables,
|
||||
raw_scene=scene.to_dict(),
|
||||
)
|
||||
|
||||
def _build_scene_summary(self, scene: Scene) -> str:
|
||||
parts = [f"Screen {scene.width}x{scene.height}"]
|
||||
for element in scene.elements[:10]:
|
||||
text = element.text or element.type
|
||||
parts.append(f" [{element.type}] {text}")
|
||||
return "\n".join(parts)
|
||||
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from agents.models import Observation, ReflectionAction, ReflectionOutcome, VerificationVerdict
|
||||
from runtime.executor import StepResult
|
||||
from runtime.planner import PlannedStep
|
||||
from semantic.llm_client import AnthropicSemanticClient, EnrichmentUnavailable
|
||||
|
||||
REFLECTOR_SYSTEM_PROMPT = """You are a reflection agent. A step failed to achieve its intended effect despite mechanical success. Your job is to propose either:
|
||||
1. A distinct recovery action (different from the failed step) that might fix the issue
|
||||
2. A replan request if no bounded corrective action is identifiable
|
||||
|
||||
You must NEVER simply re-issue the identical failed step.
|
||||
|
||||
Respond with JSON: {"replan": bool, "action": null or {"action": "...", "description": "...", "args": {...}}, "reasoning": "explanation"}"""
|
||||
|
||||
REFLECTOR_JSON_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["replan", "reasoning"],
|
||||
"properties": {
|
||||
"replan": {"type": "boolean"},
|
||||
"action": {
|
||||
"type": ["object", "null"],
|
||||
"properties": {
|
||||
"action": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"args": {"type": "object"},
|
||||
},
|
||||
},
|
||||
"reasoning": {"type": "string", "minLength": 1},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Reflector:
|
||||
def __init__(self, *, client: AnthropicSemanticClient | None = None) -> None:
|
||||
self._client = client
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
*,
|
||||
observation: Observation,
|
||||
planned_step: PlannedStep,
|
||||
step_result: StepResult,
|
||||
verdict: VerificationVerdict,
|
||||
) -> ReflectionOutcome:
|
||||
if self._client is None:
|
||||
return self._fallback_reflect(
|
||||
observation=observation,
|
||||
planned_step=planned_step,
|
||||
step_result=step_result,
|
||||
verdict=verdict,
|
||||
)
|
||||
|
||||
try:
|
||||
return self._llm_reflect(
|
||||
observation=observation,
|
||||
planned_step=planned_step,
|
||||
step_result=step_result,
|
||||
verdict=verdict,
|
||||
)
|
||||
except EnrichmentUnavailable:
|
||||
return self._fallback_reflect(
|
||||
observation=observation,
|
||||
planned_step=planned_step,
|
||||
step_result=step_result,
|
||||
verdict=verdict,
|
||||
)
|
||||
|
||||
def _llm_reflect(
|
||||
self,
|
||||
*,
|
||||
observation: Observation,
|
||||
planned_step: PlannedStep,
|
||||
step_result: StepResult,
|
||||
verdict: VerificationVerdict,
|
||||
) -> ReflectionOutcome:
|
||||
user_prompt = json.dumps(
|
||||
{
|
||||
"observation": observation.to_dict(),
|
||||
"planned_step": {
|
||||
"action": planned_step.action,
|
||||
"description": planned_step.description,
|
||||
"args": planned_step.args,
|
||||
"expected_text": planned_step.expected_text,
|
||||
},
|
||||
"step_result": step_result.to_dict(),
|
||||
"verdict": verdict.to_dict(),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
response = self._client._create_message( # type: ignore[union-attr]
|
||||
{"prompt": user_prompt, "system": REFLECTOR_SYSTEM_PROMPT},
|
||||
timeout=10.0,
|
||||
)
|
||||
payload = _extract_outcome(response)
|
||||
outcome = ReflectionOutcome.from_dict(payload)
|
||||
if not outcome.replan and outcome.action is not None:
|
||||
if outcome.action.action == planned_step.action and outcome.action.args == planned_step.args:
|
||||
return ReflectionOutcome(
|
||||
replan=True,
|
||||
reasoning="Reflector proposed identical action; requesting replan instead.",
|
||||
)
|
||||
return outcome
|
||||
|
||||
def _fallback_reflect(
|
||||
self,
|
||||
*,
|
||||
observation: Observation,
|
||||
planned_step: PlannedStep,
|
||||
step_result: StepResult,
|
||||
verdict: VerificationVerdict,
|
||||
) -> ReflectionOutcome:
|
||||
return ReflectionOutcome(
|
||||
replan=True,
|
||||
reasoning=f"Step '{planned_step.description}' failed semantically; replanning (fallback).",
|
||||
)
|
||||
|
||||
|
||||
def _extract_outcome(response: Any) -> dict[str, Any]:
|
||||
if isinstance(response, dict) and "replan" in response:
|
||||
return response
|
||||
content = getattr(response, "content", None)
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
text = getattr(block, "text", None)
|
||||
if isinstance(text, str):
|
||||
try:
|
||||
decoded = json.loads(text)
|
||||
if isinstance(decoded, dict) and "replan" in decoded:
|
||||
return decoded
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(block, dict) and "replan" in block:
|
||||
return block
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
decoded = json.loads(content)
|
||||
if isinstance(decoded, dict) and "replan" in decoded:
|
||||
return decoded
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
raise ValueError("could not extract reflection outcome from LLM response")
|
||||
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from agents.models import Observation, VerificationVerdict
|
||||
from runtime.executor import StepResult
|
||||
from runtime.planner import PlannedStep
|
||||
from semantic.llm_client import AnthropicSemanticClient, EnrichmentUnavailable
|
||||
|
||||
VERIFIER_SYSTEM_PROMPT = """You are a verification agent. Your job is to determine whether a planned step achieved its intended effect.
|
||||
|
||||
You will receive:
|
||||
1. A pre-step observation of the device state
|
||||
2. A post-step observation of the device state
|
||||
3. The planned step (action, description, expected outcome)
|
||||
4. The step result (whether the tool call succeeded mechanically)
|
||||
|
||||
Analyze whether the intended effect of the step actually occurred by comparing the pre and post observations against the step's stated intent.
|
||||
|
||||
Respond with JSON: {"result": "achieved" or "not_achieved", "reasoning": "explanation"}"""
|
||||
|
||||
VERIFIER_JSON_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["result", "reasoning"],
|
||||
"properties": {
|
||||
"result": {"type": "string", "enum": ["achieved", "not_achieved"]},
|
||||
"reasoning": {"type": "string", "minLength": 1},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Verifier:
|
||||
def __init__(self, *, client: AnthropicSemanticClient | None = None) -> None:
|
||||
self._client = client
|
||||
|
||||
def verify(
|
||||
self,
|
||||
*,
|
||||
pre_observation: Observation,
|
||||
post_observation: Observation,
|
||||
planned_step: PlannedStep,
|
||||
step_result: StepResult,
|
||||
) -> VerificationVerdict:
|
||||
if not step_result.success:
|
||||
return VerificationVerdict(
|
||||
result="not_achieved",
|
||||
reasoning=f"Step failed mechanically: {step_result.error}",
|
||||
)
|
||||
|
||||
if self._client is None:
|
||||
return self._fallback_verify(
|
||||
pre_observation=pre_observation,
|
||||
post_observation=post_observation,
|
||||
planned_step=planned_step,
|
||||
step_result=step_result,
|
||||
)
|
||||
|
||||
try:
|
||||
return self._llm_verify(
|
||||
pre_observation=pre_observation,
|
||||
post_observation=post_observation,
|
||||
planned_step=planned_step,
|
||||
step_result=step_result,
|
||||
)
|
||||
except EnrichmentUnavailable:
|
||||
return self._fallback_verify(
|
||||
pre_observation=pre_observation,
|
||||
post_observation=post_observation,
|
||||
planned_step=planned_step,
|
||||
step_result=step_result,
|
||||
)
|
||||
|
||||
def _llm_verify(
|
||||
self,
|
||||
*,
|
||||
pre_observation: Observation,
|
||||
post_observation: Observation,
|
||||
planned_step: PlannedStep,
|
||||
step_result: StepResult,
|
||||
) -> VerificationVerdict:
|
||||
user_prompt = json.dumps(
|
||||
{
|
||||
"pre_observation": pre_observation.to_dict(),
|
||||
"post_observation": post_observation.to_dict(),
|
||||
"planned_step": {
|
||||
"action": planned_step.action,
|
||||
"description": planned_step.description,
|
||||
"args": planned_step.args,
|
||||
"expected_text": planned_step.expected_text,
|
||||
},
|
||||
"step_result": step_result.to_dict(),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
response = self._client._create_message( # type: ignore[union-attr]
|
||||
{"prompt": user_prompt, "system": VERIFIER_SYSTEM_PROMPT},
|
||||
timeout=10.0,
|
||||
)
|
||||
payload = _extract_verdict(response)
|
||||
return VerificationVerdict.from_dict(payload)
|
||||
|
||||
def _fallback_verify(
|
||||
self,
|
||||
*,
|
||||
pre_observation: Observation,
|
||||
post_observation: Observation,
|
||||
planned_step: PlannedStep,
|
||||
step_result: StepResult,
|
||||
) -> VerificationVerdict:
|
||||
if pre_observation.scene_summary == post_observation.scene_summary:
|
||||
return VerificationVerdict(
|
||||
result="not_achieved",
|
||||
reasoning="Scene unchanged after step execution (fallback heuristic).",
|
||||
)
|
||||
return VerificationVerdict(
|
||||
result="achieved",
|
||||
reasoning="Scene changed after step execution (fallback heuristic).",
|
||||
)
|
||||
|
||||
|
||||
def _extract_verdict(response: Any) -> dict[str, Any]:
|
||||
if isinstance(response, dict) and "result" in response:
|
||||
return response
|
||||
content = getattr(response, "content", None)
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
text = getattr(block, "text", None)
|
||||
if isinstance(text, str):
|
||||
try:
|
||||
decoded = json.loads(text)
|
||||
if isinstance(decoded, dict) and "result" in decoded:
|
||||
return decoded
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(block, dict) and "result" in block:
|
||||
return block
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
decoded = json.loads(content)
|
||||
if isinstance(decoded, dict) and "result" in decoded:
|
||||
return decoded
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
raise ValueError("could not extract verification verdict from LLM response")
|
||||
Reference in New Issue
Block a user