//go:build integration package jobs_test import ( "context" "encoding/json" "sync" "testing" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tcpg "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/yan1h/agent-coding-workflow/internal/infra/db" "github.com/yan1h/agent-coding-workflow/internal/jobs" ) // setupRepo spins up a throwaway postgres container, runs migrations, and // returns a fresh PgRepository plus the underlying pool for tests that need to // poke at rows directly. Each test gets its own container so they may run in // parallel without bleeding state. func setupRepo(t *testing.T) (*jobs.PgRepository, *pgxpool.Pool) { t.Helper() ctx := context.Background() container, err := tcpg.Run(ctx, "postgres:16-alpine", tcpg.WithDatabase("acw"), tcpg.WithUsername("acw"), tcpg.WithPassword("acw"), ) require.NoError(t, err) t.Cleanup(func() { _ = container.Terminate(ctx) }) dsn, err := container.ConnectionString(ctx, "sslmode=disable") require.NoError(t, err) require.NoError(t, db.Migrate(ctx, dsn, "file://../../migrations")) pool, err := db.NewPool(ctx, dsn) require.NoError(t, err) t.Cleanup(pool.Close) return jobs.NewPostgresRepository(pool), pool } func TestEnqueue_PersistsJob(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) job, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{"topic":"x"}`), time.Now(), 5) require.NoError(t, err) assert.Equal(t, jobs.StatusPending, job.Status) assert.Equal(t, int32(0), job.Attempts) assert.Equal(t, int32(5), job.MaxAttempts) } func TestLease_PicksOldestPending(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) earlier := time.Now().Add(-10 * time.Second) later := time.Now() _, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{"a":1}`), later, 5) require.NoError(t, err) earliest, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{"a":2}`), earlier, 5) require.NoError(t, err) got, err := repo.Lease(ctx, "test-host#1") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, earliest.ID, got.ID) assert.Equal(t, jobs.StatusRunning, got.Status) assert.Equal(t, int32(1), got.Attempts) } func TestLease_ReturnsNilWhenEmpty(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) got, err := repo.Lease(ctx, "test-host#1") require.NoError(t, err) assert.Nil(t, got) } // TestLease_SkipLockedConcurrent verifies that with FOR UPDATE SKIP LOCKED in // the underlying query, only one of N concurrent leasers wins a single // pending row. func TestLease_SkipLockedConcurrent(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) _, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{"a":1}`), time.Now(), 5) require.NoError(t, err) var wg sync.WaitGroup results := make([]*jobs.Job, 5) leaseErrs := make([]error, 5) for i := 0; i < 5; i++ { wg.Add(1) go func(idx int) { defer wg.Done() j, err := repo.Lease(ctx, "test") results[idx] = j leaseErrs[idx] = err }(i) } wg.Wait() got := 0 for i := range results { require.NoError(t, leaseErrs[i]) if results[i] != nil { got++ } } assert.Equal(t, 1, got, "only one worker should win") } func TestComplete_ClearsLease(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) job, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) leased, err := repo.Lease(ctx, "h") require.NoError(t, err) require.NoError(t, repo.Complete(ctx, leased.ID, "h")) got, err := repo.Get(ctx, job.ID) require.NoError(t, err) assert.Equal(t, jobs.StatusCompleted, got.Status) assert.Nil(t, got.LeasedAt) assert.Nil(t, got.LeasedBy) } // TestComplete_LeaseLostIsNoOp verifies the fencing predicate: a write-back // from a worker that no longer holds the lease (e.g. the reaper re-leased the // job) matches 0 rows and returns ErrLeaseLost without clobbering the new run. func TestComplete_LeaseLostIsNoOp(t *testing.T) { t.Parallel() ctx := context.Background() repo, pool := setupRepo(t) _, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) // Worker A leases the job, then its handler stalls past the lease timeout. leased, err := repo.Lease(ctx, "workerA") require.NoError(t, err) // Reaper resets it to pending; worker B re-leases it (now leased_by=workerB). _, err = pool.Exec(ctx, `UPDATE jobs SET leased_at = now() - INTERVAL '10 minutes' WHERE id=$1`, leased.ID) require.NoError(t, err) ids, err := repo.ReapStuckRunningJobs(ctx, time.Now().Add(-5*time.Minute)) require.NoError(t, err) require.Len(t, ids, 1) reLeased, err := repo.Lease(ctx, "workerB") require.NoError(t, err) require.Equal(t, leased.ID, reLeased.ID) // Stale worker A's terminal write-backs must be no-ops (ErrLeaseLost) and // must not change the row that workerB now owns. assert.ErrorIs(t, repo.Complete(ctx, leased.ID, "workerA"), jobs.ErrLeaseLost) assert.ErrorIs(t, repo.FailWithRetry(ctx, leased.ID, "workerA", time.Now(), "stale"), jobs.ErrLeaseLost) assert.ErrorIs(t, repo.DeadLetter(ctx, leased.ID, "workerA", "stale"), jobs.ErrLeaseLost) got, err := repo.Get(ctx, leased.ID) require.NoError(t, err) assert.Equal(t, jobs.StatusRunning, got.Status, "row must still be owned by workerB's run") require.NotNil(t, got.LeasedBy) assert.Equal(t, "workerB", *got.LeasedBy) assert.Nil(t, got.LastError, "stale write must not have set an error") // Worker B (the true owner) completes successfully. require.NoError(t, repo.Complete(ctx, leased.ID, "workerB")) got, err = repo.Get(ctx, leased.ID) require.NoError(t, err) assert.Equal(t, jobs.StatusCompleted, got.Status) } func TestFailWithRetry_ReschedulesAndKeepsAttempts(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) _, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) leased, err := repo.Lease(ctx, "h") require.NoError(t, err) next := time.Now().Add(30 * time.Second) require.NoError(t, repo.FailWithRetry(ctx, leased.ID, "h", next, "boom")) got, err := repo.Get(ctx, leased.ID) require.NoError(t, err) assert.Equal(t, jobs.StatusPending, got.Status) assert.WithinDuration(t, next, got.ScheduledAt, time.Second) require.NotNil(t, got.LastError) assert.Equal(t, "boom", *got.LastError) assert.Equal(t, int32(1), got.Attempts) } func TestDeadLetter_FinalState(t *testing.T) { t.Parallel() ctx := context.Background() repo, _ := setupRepo(t) _, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) leased, err := repo.Lease(ctx, "h") require.NoError(t, err) require.NoError(t, repo.DeadLetter(ctx, leased.ID, "h", "max attempts")) got, err := repo.Get(ctx, leased.ID) require.NoError(t, err) assert.Equal(t, jobs.StatusDead, got.Status) require.NotNil(t, got.CompletedAt) } func TestReapStuck_ResetsLongLeased(t *testing.T) { t.Parallel() ctx := context.Background() repo, pool := setupRepo(t) _, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) leased, err := repo.Lease(ctx, "h") require.NoError(t, err) _, err = pool.Exec(ctx, `UPDATE jobs SET leased_at = now() - INTERVAL '10 minutes' WHERE id=$1`, leased.ID) require.NoError(t, err) ids, err := repo.ReapStuckRunningJobs(ctx, time.Now().Add(-5*time.Minute)) require.NoError(t, err) require.Len(t, ids, 1) assert.Equal(t, leased.ID, ids[0]) got, err := repo.Get(ctx, leased.ID) require.NoError(t, err) assert.Equal(t, jobs.StatusPending, got.Status) } func TestPurge_DeletesOldCompletedAndDead(t *testing.T) { t.Parallel() ctx := context.Background() repo, pool := setupRepo(t) mkOld := func(status jobs.JobStatus) uuid.UUID { j, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) _, err = pool.Exec(ctx, `UPDATE jobs SET status=$1, completed_at = now() - INTERVAL '10 days' WHERE id=$2`, string(status), j.ID) require.NoError(t, err) return j.ID } completedOld := mkOld(jobs.StatusCompleted) deadOld := mkOld(jobs.StatusDead) freshComp, err := repo.Enqueue(ctx, jobs.TypeWebhookDeliver, json.RawMessage(`{}`), time.Now(), 5) require.NoError(t, err) leasedFresh, err := repo.Lease(ctx, "h") require.NoError(t, err) require.Equal(t, freshComp.ID, leasedFresh.ID) require.NoError(t, repo.Complete(ctx, freshComp.ID, "h")) rows, err := repo.PurgeCompleted(ctx, time.Now().Add(-7*24*time.Hour)) require.NoError(t, err) assert.Equal(t, int64(2), rows) _, err = repo.Get(ctx, completedOld) assert.Error(t, err) _, err = repo.Get(ctx, deadOld) assert.Error(t, err) _, err = repo.Get(ctx, freshComp.ID) require.NoError(t, err) }