agentic-mobile-control/apps/cloud-api/tests/test_app.py

from __future__ import annotations

import time
from datetime import timedelta

import pytest
from fastapi.testclient import TestClient

import cloud_api.app as app_module
from cloud_api.app import create_app
from cloud.auth import BearerCredential, EnrollmentCredential, digest_token
from cloud.control_config import CloudConfigurationError, CloudControlConfig
from cloud.database import CloudDatabase
from cloud.pool import PooledDevice
from cloud.scheduler import ScheduledTask, TaskConstraints
from core.models import utc_now


def test_create_app_returns_independent_cloud_application() -> None:
    app = create_app()

    assert app.title == "Device Cloud API"
    assert callable(create_app)
    paths = set(app.openapi()["paths"])
    assert "/v1/tasks" in paths
    assert "/internal/v1/hosts/{host_id}/heartbeat" in paths
    assert "/internal/v1/enrollments" in paths
    assert "/internal/v1/hosts/{host_id}/devices/enroll" in paths


def test_managed_host_enrollment_device_mapping_and_restart_authentication(
    tmp_path,
) -> None:
    database_url = f"sqlite:///{(tmp_path / 'enrollment.sqlite3').as_posix()}"
    enrollment_token = "one-time-enrollment-token"
    host_token = "host-token-" + ("x" * 40)
    config = CloudControlConfig(
        database_url=database_url,
        credentials=(
            BearerCredential(
                principal_id="operator",
                token="operator-token",
                scopes=frozenset({"pool:read"}),
            ),
        ),
        enrollment_credentials=(
            EnrollmentCredential(
                principal_id="installer-a",
                token=enrollment_token,
            ),
        ),
    )
    enrollment_payload = {
        "agent_instance_id": "agent-instance-a",
        "host_token": host_token,
        "display_name": "Edge Mac",
    }

    app = create_app(config=config)
    with TestClient(app) as client:
        enrolled = client.post(
            "/internal/v1/enrollments",
            headers={"Authorization": f"Bearer {enrollment_token}"},
            json=enrollment_payload,
        )
        assert enrolled.status_code == 201
        host_id = enrolled.json()["host_id"]
        assert host_id.startswith("host-")

        retried = client.post(
            "/internal/v1/enrollments",
            headers={"Authorization": f"Bearer {enrollment_token}"},
            json=enrollment_payload,
        )
        assert retried.status_code == 201
        assert retried.json()["host_id"] == host_id

        reused = client.post(
            "/internal/v1/enrollments",
            headers={"Authorization": f"Bearer {enrollment_token}"},
            json={
                **enrollment_payload,
                "agent_instance_id": "agent-instance-b",
            },
        )
        assert reused.status_code == 409

        device = client.post(
            f"/internal/v1/hosts/{host_id}/devices/enroll",
            headers={"Authorization": f"Bearer {host_token}"},
            json={
                "local_device_id": "local-device-a",
                "driver_type": "wda",
                "name": "iPhone",
                "capability_tags": ["ios"],
            },
        )
        assert device.status_code == 201
        device_id = device.json()["device_id"]
        assert device_id.startswith("device-")

        device_retry = client.post(
            f"/internal/v1/hosts/{host_id}/devices/enroll",
            headers={"Authorization": f"Bearer {host_token}"},
            json={
                "local_device_id": "local-device-a",
                "driver_type": "wda",
                "name": "Renamed iPhone",
                "capability_tags": ["ios", "physical"],
            },
        )
        assert device_retry.json()["device_id"] == device_id

        rejected_snapshot = client.put(
            f"/internal/v1/hosts/{host_id}/heartbeat",
            headers={"Authorization": f"Bearer {host_token}"},
            json={
                "host_id": host_id,
                "devices": [
                    {
                        "device_id": "caller-selected-device",
                        "driver_type": "wda",
                        "status": "idle",
                    }
                ],
            },
        )
        assert rejected_snapshot.status_code == 409

        heartbeat = client.put(
            f"/internal/v1/hosts/{host_id}/heartbeat",
            headers={"Authorization": f"Bearer {host_token}"},
            json={
                "host_id": host_id,
                "devices": [
                    {
                        "device_id": device_id,
                        "driver_type": "wda",
                        "status": "idle",
                    }
                ],
            },
        )
        assert heartbeat.status_code == 200

        public_attempt = client.get(
            "/v1/devices",
            headers={"Authorization": f"Bearer {host_token}"},
        )
        assert public_attempt.status_code == 403

    restarted = create_app(config=config)
    with TestClient(restarted) as client:
        heartbeat = client.put(
            f"/internal/v1/hosts/{host_id}/heartbeat",
            headers={"Authorization": f"Bearer {host_token}"},
            json={"host_id": host_id, "devices": []},
        )
        assert heartbeat.status_code == 200

        assert (
            restarted.state.cloud_services.repository.authenticate_enrolled_host(
                digest_token(host_token)
            )
            == host_id
        )
        assert restarted.state.cloud_services.repository.revoke_enrolled_host(
            host_id,
            revoked_at=utc_now(),
        )
        rejected = client.put(
            f"/internal/v1/hosts/{host_id}/heartbeat",
            headers={"Authorization": f"Bearer {host_token}"},
            json={"host_id": host_id, "devices": []},
        )
        assert rejected.status_code == 401


def test_cloud_application_owns_database_lifecycle() -> None:
    events: list[str] = []

    class FakeRepository:
        def health_check(self) -> None:
            events.append("healthy")

        def list_queued_tasks(self) -> list[object]:
            events.append("scheduled")
            return []

        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
            events.append("reaped")
            return []

    class FakeDatabase:
        repository = FakeRepository()

        def close(self) -> None:
            events.append("closed")

    fake_database = FakeDatabase()
    app = create_app(
        config=CloudControlConfig(database_url="sqlite:///:memory:"),
        database_factory=lambda _config: fake_database,  # type: ignore[arg-type,return-value]
    )

    with TestClient(app):
        assert app.state.database is fake_database
        assert (
            app.state.cloud_services.pool.store is app.state.cloud_services.repository
        )
        assert "healthy" in events

    assert events[-1] == "closed"


def test_repository_proxy_is_available_only_during_lifespan() -> None:
    class FakeRepository:
        def health_check(self) -> None:
            return None

        def list_queued_tasks(self) -> list[object]:
            return []

        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
            return []

    class FakeDatabase:
        repository = FakeRepository()

        def close(self) -> None:
            return None

    app = create_app(
        config=CloudControlConfig(database_url="sqlite:///:memory:"),
        database_factory=lambda _config: FakeDatabase(),  # type: ignore[arg-type,return-value]
    )

    with TestClient(app):
        app.state.cloud_services.repository.health_check()

    with pytest.raises(RuntimeError, match="outside app lifespan"):
        app.state.cloud_services.repository.health_check()


def test_lifespan_runs_scheduler_and_reaper_until_shutdown() -> None:
    events: list[str] = []

    class FakeRepository:
        def health_check(self) -> None:
            events.append("healthy")

        def list_queued_tasks(self) -> list[object]:
            events.append("scheduled")
            return []

        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
            events.append(f"reaped:{max_attempts}")
            return []

    class FakeDatabase:
        repository = FakeRepository()

        def close(self) -> None:
            events.append("closed")

    app = create_app(
        config=CloudControlConfig(
            database_url="sqlite:///:memory:",
            scheduler_interval_seconds=0.01,
            lease_reaper_interval_seconds=0.01,
            max_task_attempts=4,
        ),
        database_factory=lambda _config: FakeDatabase(),  # type: ignore[arg-type,return-value]
    )

    with TestClient(app):
        time.sleep(0.04)
        assert events.count("scheduled") >= 2
        assert events.count("reaped:4") >= 2
        assert all(not task.done() for task in app.state.worker_tasks)

    assert all(task.done() for task in app.state.worker_tasks)
    assert events[-1] == "closed"


def test_lifecycle_workers_log_failures_and_continue(monkeypatch) -> None:
    logged_workers: list[str] = []

    def record_exception(_message: str, *, extra: dict[str, str]) -> None:
        logged_workers.append(extra["worker"])

    monkeypatch.setattr(app_module.logger, "exception", record_exception)
    scheduler_calls = 0
    reaper_calls = 0

    class FakeRepository:
        def health_check(self) -> None:
            return None

        def list_queued_tasks(self) -> list[object]:
            nonlocal scheduler_calls
            scheduler_calls += 1
            if scheduler_calls == 1:
                raise RuntimeError("scheduler transient failure")
            return []

        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
            nonlocal reaper_calls
            reaper_calls += 1
            if reaper_calls == 1:
                raise RuntimeError("reaper transient failure")
            return []

    class FakeDatabase:
        repository = FakeRepository()

        def close(self) -> None:
            return None

    app = create_app(
        config=CloudControlConfig(
            database_url="sqlite:///:memory:",
            scheduler_interval_seconds=0.01,
            lease_reaper_interval_seconds=0.01,
        ),
        database_factory=lambda _config: FakeDatabase(),  # type: ignore[arg-type,return-value]
    )

    with TestClient(app):
        time.sleep(0.04)
        assert scheduler_calls >= 2
        assert reaper_calls >= 2
        assert all(not task.done() for task in app.state.worker_tasks)

    assert set(logged_workers) >= {
        "scheduler",
        "lease_reaper",
    }


def test_liveness_stays_up_when_database_readiness_fails() -> None:
    database_healthy = True

    class FakeRepository:
        def health_check(self) -> None:
            if not database_healthy:
                raise RuntimeError("database unavailable with secret credentials")

        def list_queued_tasks(self) -> list[object]:
            return []

        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
            return []

    class FakeDatabase:
        repository = FakeRepository()

        def close(self) -> None:
            return None

    app = create_app(
        config=CloudControlConfig(database_url="sqlite:///:memory:"),
        database_factory=lambda _config: FakeDatabase(),  # type: ignore[arg-type,return-value]
    )

    with TestClient(app) as client:
        assert client.get("/health/live").json() == {"status": "live"}
        ready = client.get("/health/ready")
        assert ready.status_code == 200
        assert all(ready.json()["checks"].values())

        database_healthy = False
        assert client.get("/health/live").status_code == 200
        not_ready = client.get("/health/ready")
        assert not_ready.status_code == 503
        assert not_ready.json()["checks"]["database"] is False
        assert "secret credentials" not in not_ready.text


def test_readiness_reports_stopped_worker() -> None:
    class FakeRepository:
        def health_check(self) -> None:
            return None

        def list_queued_tasks(self) -> list[object]:
            return []

        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
            return []

    class FakeDatabase:
        repository = FakeRepository()

        def close(self) -> None:
            return None

    class StoppedWorker:
        def done(self) -> bool:
            return True

    app = create_app(
        config=CloudControlConfig(database_url="sqlite:///:memory:"),
        database_factory=lambda _config: FakeDatabase(),  # type: ignore[arg-type,return-value]
    )

    with TestClient(app) as client:
        app.state.worker_tasks = (StoppedWorker(),)
        response = client.get("/health/ready")

        assert response.status_code == 503
        assert response.json()["checks"]["workers"] is False


def test_request_correlation_id_is_propagated_without_sensitive_headers(
    monkeypatch,
    tmp_path,
) -> None:
    request_logs: list[dict[str, object]] = []

    def record_info(_message: str, *, extra: dict[str, object]) -> None:
        request_logs.append(extra)

    monkeypatch.setattr(app_module.logger, "info", record_info)
    app = create_app(
        config=CloudControlConfig(
            database_url=f"sqlite:///{(tmp_path / 'correlation.sqlite3').as_posix()}"
        )
    )

    with TestClient(app) as client:
        response = client.get(
            "/health/live",
            headers={
                "X-Correlation-ID": "request-123",
                "Authorization": "Bearer never-log-this",
            },
        )

    assert response.headers["X-Correlation-ID"] == "request-123"
    assert request_logs == [
        {
            "correlation_id": "request-123",
            "method": "GET",
            "path": "/health/live",
            "status_code": 200,
        }
    ]
    assert "never-log-this" not in repr(request_logs)


def test_startup_database_failure_closes_database() -> None:
    closed = False

    class FailingRepository:
        def health_check(self) -> None:
            raise RuntimeError("database unavailable")

    class FailingDatabase:
        repository = FailingRepository()

        def close(self) -> None:
            nonlocal closed
            closed = True

    app = create_app(
        config=CloudControlConfig(database_url="sqlite:///:memory:"),
        database_factory=lambda _config: FailingDatabase(),  # type: ignore[arg-type,return-value]
    )

    with pytest.raises(RuntimeError, match="database unavailable"):
        with TestClient(app):
            pass
    assert closed is True


def test_persisted_queue_is_recovered_after_control_plane_restart(tmp_path) -> None:
    database_url = f"sqlite:///{(tmp_path / 'queue-recovery.sqlite3').as_posix()}"
    seed = CloudDatabase(database_url)
    now = utc_now()
    seed.repository.upsert_host("host-a", address=None, last_seen_at=now)
    seed.repository.replace_host_devices(
        "host-a",
        [
            PooledDevice(
                device_id="device-a",
                host_id="host-a",
                driver_type="wda",
                status="idle",
                synced_at=now,
            )
        ],
    )
    seed.repository.enqueue_task(
        ScheduledTask(
            id="persisted-task",
            goal="resume queued work",
            workflow_definition_id=None,
            constraints=TaskConstraints(),
            created_at=now,
        )
    )
    seed.close()

    app = create_app(
        config=CloudControlConfig(
            database_url=database_url,
            scheduler_interval_seconds=0.01,
            lease_reaper_interval_seconds=1,
        )
    )
    with TestClient(app):
        assert _wait_until(
            lambda: (
                app.state.cloud_services.repository.get_task("persisted-task").status
                == "assigned"
            )
        )

    verification = CloudDatabase(database_url)
    try:
        task = verification.repository.get_task("persisted-task")
        assert task is not None
        assert task.status == "assigned"
        assert task.attempt_count == 1
    finally:
        verification.close()


def test_expired_lease_is_recovered_after_control_plane_restart(tmp_path) -> None:
    database_url = f"sqlite:///{(tmp_path / 'lease-recovery.sqlite3').as_posix()}"
    seed = CloudDatabase(database_url)
    now = utc_now()
    seed.repository.upsert_host("host-a", address=None, last_seen_at=now)
    seed.repository.replace_host_devices(
        "host-a",
        [
            PooledDevice(
                device_id="device-a",
                host_id="host-a",
                driver_type="wda",
                status="idle",
                synced_at=now,
            )
        ],
    )
    seed.repository.enqueue_task(
        ScheduledTask(
            id="expired-task",
            goal="recover expired work",
            workflow_definition_id=None,
            constraints=TaskConstraints(),
            created_at=now,
        )
    )
    seed.repository.assign_task(
        task_id="expired-task",
        host_id="host-a",
        device_id="device-a",
        lease_id="expired-lease",
        lease_expires_at=now - timedelta(seconds=1),
        now=now - timedelta(minutes=1),
    )
    seed.close()

    app = create_app(
        config=CloudControlConfig(
            database_url=database_url,
            scheduler_interval_seconds=1,
            lease_reaper_interval_seconds=0.01,
            max_task_attempts=1,
        )
    )
    with TestClient(app):
        assert _wait_until(
            lambda: (
                app.state.cloud_services.repository.get_task("expired-task").status
                == "failed"
            )
        )

    verification = CloudDatabase(database_url)
    try:
        task = verification.repository.get_task("expired-task")
        assert task is not None
        assert task.status == "failed"
        assert task.failure_reason == "lease expired after 1 attempts"
        assert verification.repository.list_task_attempts("expired-task")[0].status == (
            "expired"
        )
    finally:
        verification.close()


def _wait_until(predicate, timeout_seconds: float = 1.0) -> bool:
    deadline = time.monotonic() + timeout_seconds
    while time.monotonic() < deadline:
        if predicate():
            return True
        time.sleep(0.01)
    return False


def test_production_app_rejects_missing_credentials() -> None:
    with pytest.raises(CloudConfigurationError, match="credential"):
        create_app(
            config=CloudControlConfig(
                environment="production",
                database_url="postgresql://db/cloud",
            )
        )


def test_cors_headers_are_absent_when_allow_list_is_empty() -> None:
    app = create_app(config=CloudControlConfig(database_url="sqlite:///:memory:"))
    with TestClient(app) as client:
        response = client.options(
            "/health/live",
            headers={
                "Origin": "http://console.example",
                "Access-Control-Request-Method": "GET",
            },
        )
    assert response.status_code >= 400
    assert "access-control-allow-origin" not in {
        key.lower() for key in response.headers
    }


def test_cors_headers_reflect_configured_origin_only() -> None:
    app = create_app(
        config=CloudControlConfig(
            database_url="sqlite:///:memory:",
            cors_allowed_origins=("http://console.example",),
        )
    )
    with TestClient(app) as client:
        allowed = client.options(
            "/health/live",
            headers={
                "Origin": "http://console.example",
                "Access-Control-Request-Method": "GET",
            },
        )
        blocked = client.options(
            "/health/live",
            headers={
                "Origin": "http://attacker.example",
                "Access-Control-Request-Method": "GET",
            },
        )

    assert allowed.status_code in {200, 204}
    assert allowed.headers["access-control-allow-origin"] == "http://console.example"
    # An origin that is not on the allow-list must not be echoed back.
    assert (
        blocked.headers.get("access-control-allow-origin") != "http://attacker.example"
    )


def test_load_control_config_parses_cors_allow_list() -> None:
    from cloud.control_config import load_control_config

    config = load_control_config(
        env={
            "CLOUD_ENVIRONMENT": "local",
            "CLOUD_DATABASE_URL": "sqlite:///:memory:",
            "CLOUD_CONSOLE_CORS_ORIGINS": (
                "http://console.example, https://console.example"
            ),
        }
    )
    assert config.cors_allowed_origins == (
        "http://console.example",
        "https://console.example",
    )


def test_load_control_config_defaults_to_empty_cors_allow_list() -> None:
    from cloud.control_config import load_control_config

    config = load_control_config(
        env={
            "CLOUD_ENVIRONMENT": "local",
            "CLOUD_DATABASE_URL": "sqlite:///:memory:",
        }
    )
    assert config.cors_allowed_origins == ()