You've already forked agentic-coding-workflow
bugfix
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
package runners
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/yan1h/agent-coding-workflow/internal/infra/notify"
|
||||
)
|
||||
|
||||
// ReaperSession is the minimal session projection the reaper needs.
|
||||
type ReaperSession struct {
|
||||
ID uuid.UUID
|
||||
UserID uuid.UUID
|
||||
StartedAt time.Time
|
||||
LastActivityAt time.Time
|
||||
MaxWallClockSeconds *int32
|
||||
}
|
||||
|
||||
// AcpSessionReaperRepo is the narrow repo surface this runner needs.
|
||||
// internal/acp.Repository satisfies it via an adapter in app.go.
|
||||
type AcpSessionReaperRepo interface {
|
||||
// ListSessionsForReaper returns active sessions exceeding their wall-clock
|
||||
// cap (per-session or the global wallClockStartedBefore cutoff) or idle since
|
||||
// idleSince. A zero wallClockStartedBefore disables the global wall-clock cut.
|
||||
ListSessionsForReaper(ctx context.Context, wallClockStartedBefore, idleSince time.Time) ([]ReaperSession, error)
|
||||
// MarkSessionTerminatedReason records why a session was reaped (no-op if a
|
||||
// reason is already set, protecting against clobbering a budget-gate reason).
|
||||
MarkSessionTerminatedReason(ctx context.Context, id uuid.UUID, reason string) error
|
||||
}
|
||||
|
||||
// SessionKiller terminates a session subprocess. internal/acp.Supervisor.Kill
|
||||
// satisfies it directly.
|
||||
type SessionKiller interface {
|
||||
Kill(ctx context.Context, sid uuid.UUID, grace time.Duration)
|
||||
}
|
||||
|
||||
// AcpSessionReaperConfig controls the reaper's cutoffs.
|
||||
type AcpSessionReaperConfig struct {
|
||||
// WallClockTimeout is the global maximum session age (0 = rely only on the
|
||||
// per-session budget_max_wall_clock_seconds).
|
||||
WallClockTimeout time.Duration
|
||||
// IdleTimeout is the max time with no new activity before reaping.
|
||||
IdleTimeout time.Duration
|
||||
// KillGrace is passed to SessionKiller.Kill.
|
||||
KillGrace time.Duration
|
||||
}
|
||||
|
||||
// AcpSessionReaper terminates ACP sessions that exceed wall-clock or are idle.
|
||||
type AcpSessionReaper struct {
|
||||
repo AcpSessionReaperRepo
|
||||
killer SessionKiller
|
||||
disp *notify.Dispatcher
|
||||
cfg AcpSessionReaperConfig
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewAcpSessionReaper builds the reaper.
|
||||
func NewAcpSessionReaper(repo AcpSessionReaperRepo, killer SessionKiller, disp *notify.Dispatcher,
|
||||
cfg AcpSessionReaperConfig, log *slog.Logger) *AcpSessionReaper {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
if cfg.KillGrace <= 0 {
|
||||
cfg.KillGrace = 5 * time.Second
|
||||
}
|
||||
return &AcpSessionReaper{repo: repo, killer: killer, disp: disp, cfg: cfg, log: log}
|
||||
}
|
||||
|
||||
// Run reaps once. Periodic; idempotent (terminated_reason is only set once).
|
||||
func (r *AcpSessionReaper) Run(ctx context.Context) {
|
||||
now := time.Now()
|
||||
var wallBefore time.Time
|
||||
if r.cfg.WallClockTimeout > 0 {
|
||||
wallBefore = now.Add(-r.cfg.WallClockTimeout)
|
||||
}
|
||||
idleSince := now.Add(-r.cfg.IdleTimeout)
|
||||
if r.cfg.IdleTimeout <= 0 {
|
||||
// No idle timeout configured -> make the idle cutoff unreachable so only
|
||||
// wall-clock rules apply (a zero time would reap everything).
|
||||
idleSince = time.Time{}
|
||||
}
|
||||
|
||||
sessions, err := r.repo.ListSessionsForReaper(ctx, wallBefore, idleSince)
|
||||
if err != nil {
|
||||
r.log.Error("acp_session_reaper.list", "err", err.Error())
|
||||
return
|
||||
}
|
||||
for _, s := range sessions {
|
||||
reason := r.classify(now, s)
|
||||
if reason == "" {
|
||||
continue
|
||||
}
|
||||
if merr := r.repo.MarkSessionTerminatedReason(ctx, s.ID, reason); merr != nil {
|
||||
r.log.Error("acp_session_reaper.mark", "session_id", s.ID, "err", merr.Error())
|
||||
// continue: still attempt the kill so the resource is freed.
|
||||
}
|
||||
// Kill must not be fatal on error (idempotent; session may have already
|
||||
// exited between listing and now).
|
||||
r.killer.Kill(ctx, s.ID, r.cfg.KillGrace)
|
||||
r.log.Info("acp_session_reaper.killed", "session_id", s.ID, "reason", reason)
|
||||
r.dispatch(ctx, s, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// classify decides which reaper reason applies, preferring wall-clock over idle.
|
||||
func (r *AcpSessionReaper) classify(now time.Time, s ReaperSession) string {
|
||||
// Per-session wall-clock cap.
|
||||
if s.MaxWallClockSeconds != nil {
|
||||
if now.Sub(s.StartedAt) >= time.Duration(*s.MaxWallClockSeconds)*time.Second {
|
||||
return "reaper_wall_clock"
|
||||
}
|
||||
}
|
||||
// Global wall-clock cap.
|
||||
if r.cfg.WallClockTimeout > 0 && now.Sub(s.StartedAt) >= r.cfg.WallClockTimeout {
|
||||
return "reaper_wall_clock"
|
||||
}
|
||||
// Idle cap.
|
||||
if r.cfg.IdleTimeout > 0 {
|
||||
activity := s.LastActivityAt
|
||||
if activity.IsZero() {
|
||||
activity = s.StartedAt
|
||||
}
|
||||
if now.Sub(activity) >= r.cfg.IdleTimeout {
|
||||
return "reaper_idle"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r *AcpSessionReaper) dispatch(ctx context.Context, s ReaperSession, reason string) {
|
||||
if r.disp == nil {
|
||||
return
|
||||
}
|
||||
_ = r.disp.Dispatch(ctx, notify.Message{
|
||||
UserID: s.UserID,
|
||||
Topic: "acp.session_reaped",
|
||||
Severity: notify.SeverityWarning,
|
||||
Title: "ACP session terminated by reaper",
|
||||
Body: "An ACP session was terminated: " + reason + ".",
|
||||
Metadata: map[string]any{
|
||||
"session_id": s.ID.String(),
|
||||
"reason": reason,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package runners_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/yan1h/agent-coding-workflow/internal/jobs/runners"
|
||||
)
|
||||
|
||||
type fakeReaperRepo struct {
|
||||
sessions []runners.ReaperSession
|
||||
listErr error
|
||||
|
||||
mu sync.Mutex
|
||||
marked map[uuid.UUID]string
|
||||
markErr error
|
||||
}
|
||||
|
||||
func (f *fakeReaperRepo) ListSessionsForReaper(_ context.Context, _, _ time.Time) ([]runners.ReaperSession, error) {
|
||||
if f.listErr != nil {
|
||||
return nil, f.listErr
|
||||
}
|
||||
return f.sessions, nil
|
||||
}
|
||||
|
||||
func (f *fakeReaperRepo) MarkSessionTerminatedReason(_ context.Context, id uuid.UUID, reason string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.marked == nil {
|
||||
f.marked = map[uuid.UUID]string{}
|
||||
}
|
||||
f.marked[id] = reason
|
||||
return f.markErr
|
||||
}
|
||||
|
||||
type fakeKiller struct {
|
||||
mu sync.Mutex
|
||||
killed []uuid.UUID
|
||||
}
|
||||
|
||||
func (k *fakeKiller) Kill(_ context.Context, sid uuid.UUID, _ time.Duration) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.killed = append(k.killed, sid)
|
||||
}
|
||||
|
||||
func TestAcpSessionReaper_KillsWallClockAndIdle(t *testing.T) {
|
||||
now := time.Now()
|
||||
wallSession := runners.ReaperSession{
|
||||
ID: uuid.New(), UserID: uuid.New(),
|
||||
StartedAt: now.Add(-5 * time.Hour),
|
||||
LastActivityAt: now.Add(-1 * time.Minute), // active, but wall-clock exceeded
|
||||
}
|
||||
idleSession := runners.ReaperSession{
|
||||
ID: uuid.New(), UserID: uuid.New(),
|
||||
StartedAt: now.Add(-10 * time.Minute),
|
||||
LastActivityAt: now.Add(-1 * time.Hour), // idle
|
||||
}
|
||||
repo := &fakeReaperRepo{sessions: []runners.ReaperSession{wallSession, idleSession}}
|
||||
killer := &fakeKiller{}
|
||||
r := runners.NewAcpSessionReaper(repo, killer, nil, runners.AcpSessionReaperConfig{
|
||||
WallClockTimeout: 4 * time.Hour,
|
||||
IdleTimeout: 30 * time.Minute,
|
||||
}, discardLogger())
|
||||
|
||||
r.Run(context.Background())
|
||||
|
||||
assert.ElementsMatch(t, []uuid.UUID{wallSession.ID, idleSession.ID}, killer.killed)
|
||||
assert.Equal(t, "reaper_wall_clock", repo.marked[wallSession.ID])
|
||||
assert.Equal(t, "reaper_idle", repo.marked[idleSession.ID])
|
||||
}
|
||||
|
||||
func TestAcpSessionReaper_PerSessionWallClock(t *testing.T) {
|
||||
now := time.Now()
|
||||
cap30s := int32(30)
|
||||
s := runners.ReaperSession{
|
||||
ID: uuid.New(), UserID: uuid.New(),
|
||||
StartedAt: now.Add(-1 * time.Minute),
|
||||
LastActivityAt: now,
|
||||
MaxWallClockSeconds: &cap30s, // exceeded per-session cap, no global cap set
|
||||
}
|
||||
repo := &fakeReaperRepo{sessions: []runners.ReaperSession{s}}
|
||||
killer := &fakeKiller{}
|
||||
r := runners.NewAcpSessionReaper(repo, killer, nil, runners.AcpSessionReaperConfig{
|
||||
IdleTimeout: time.Hour, // generous idle so only wall-clock triggers
|
||||
}, discardLogger())
|
||||
|
||||
r.Run(context.Background())
|
||||
|
||||
require.Len(t, killer.killed, 1)
|
||||
assert.Equal(t, s.ID, killer.killed[0])
|
||||
assert.Equal(t, "reaper_wall_clock", repo.marked[s.ID])
|
||||
}
|
||||
|
||||
func TestAcpSessionReaper_SkipsWithinThresholds(t *testing.T) {
|
||||
now := time.Now()
|
||||
s := runners.ReaperSession{
|
||||
ID: uuid.New(), UserID: uuid.New(),
|
||||
StartedAt: now.Add(-5 * time.Minute),
|
||||
LastActivityAt: now.Add(-1 * time.Minute),
|
||||
}
|
||||
repo := &fakeReaperRepo{sessions: []runners.ReaperSession{s}}
|
||||
killer := &fakeKiller{}
|
||||
r := runners.NewAcpSessionReaper(repo, killer, nil, runners.AcpSessionReaperConfig{
|
||||
WallClockTimeout: 4 * time.Hour,
|
||||
IdleTimeout: 30 * time.Minute,
|
||||
}, discardLogger())
|
||||
|
||||
r.Run(context.Background())
|
||||
|
||||
assert.Empty(t, killer.killed)
|
||||
assert.Empty(t, repo.marked)
|
||||
}
|
||||
|
||||
func TestAcpSessionReaper_MarkErrorStillKills(t *testing.T) {
|
||||
now := time.Now()
|
||||
s := runners.ReaperSession{
|
||||
ID: uuid.New(), UserID: uuid.New(),
|
||||
StartedAt: now.Add(-5 * time.Hour),
|
||||
LastActivityAt: now,
|
||||
}
|
||||
repo := &fakeReaperRepo{sessions: []runners.ReaperSession{s}, markErr: errors.New("db down")}
|
||||
killer := &fakeKiller{}
|
||||
r := runners.NewAcpSessionReaper(repo, killer, nil, runners.AcpSessionReaperConfig{
|
||||
WallClockTimeout: 4 * time.Hour,
|
||||
IdleTimeout: 30 * time.Minute,
|
||||
}, discardLogger())
|
||||
|
||||
r.Run(context.Background())
|
||||
|
||||
require.Len(t, killer.killed, 1)
|
||||
}
|
||||
|
||||
func TestAcpSessionReaper_ListErrorIsNotFatal(t *testing.T) {
|
||||
repo := &fakeReaperRepo{listErr: errors.New("boom")}
|
||||
killer := &fakeKiller{}
|
||||
r := runners.NewAcpSessionReaper(repo, killer, nil, runners.AcpSessionReaperConfig{
|
||||
IdleTimeout: 30 * time.Minute,
|
||||
}, discardLogger())
|
||||
|
||||
r.Run(context.Background())
|
||||
assert.Empty(t, killer.killed)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package runners
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AcpTurnsPurgeRepo is the minimal repo surface this runner needs.
|
||||
// internal/acp.Repository satisfies it via an adapter in app.go.
|
||||
type AcpTurnsPurgeRepo interface {
|
||||
PurgeTurnsBefore(ctx context.Context, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// AcpTurnsPurge deletes acp_turns older than retention. Periodic; idempotent.
|
||||
type AcpTurnsPurge struct {
|
||||
repo AcpTurnsPurgeRepo
|
||||
retention time.Duration
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewAcpTurnsPurge builds the purger.
|
||||
func NewAcpTurnsPurge(repo AcpTurnsPurgeRepo, retention time.Duration, log *slog.Logger) *AcpTurnsPurge {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &AcpTurnsPurge{repo: repo, retention: retention, log: log}
|
||||
}
|
||||
|
||||
// Run purges once.
|
||||
func (r *AcpTurnsPurge) Run(ctx context.Context) {
|
||||
cutoff := time.Now().Add(-r.retention)
|
||||
rows, err := r.repo.PurgeTurnsBefore(ctx, cutoff)
|
||||
if err != nil {
|
||||
r.log.Error("acp_turns.purge", "err", err.Error())
|
||||
return
|
||||
}
|
||||
if rows > 0 {
|
||||
r.log.Info("acp_turns.purged", "count", rows, "before", cutoff.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package runners
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AuditRetentionRepo is the minimal surface this runner needs.
|
||||
// internal/audit.Reader satisfies it (PurgeBefore).
|
||||
type AuditRetentionRepo interface {
|
||||
PurgeBefore(ctx context.Context, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// AuditRetention deletes audit_logs older than retention. Periodic; idempotent.
|
||||
type AuditRetention struct {
|
||||
repo AuditRetentionRepo
|
||||
retention time.Duration
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewAuditRetention builds the audit-log retention purger.
|
||||
func NewAuditRetention(repo AuditRetentionRepo, retention time.Duration, log *slog.Logger) *AuditRetention {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &AuditRetention{repo: repo, retention: retention, log: log}
|
||||
}
|
||||
|
||||
// Run purges once. A non-positive retention disables the purge (safety: never
|
||||
// delete the whole table when misconfigured).
|
||||
func (r *AuditRetention) Run(ctx context.Context) {
|
||||
if r.retention <= 0 {
|
||||
return
|
||||
}
|
||||
cutoff := time.Now().Add(-r.retention)
|
||||
rows, err := r.repo.PurgeBefore(ctx, cutoff)
|
||||
if err != nil {
|
||||
r.log.Error("audit_retention.purge", "err", err.Error())
|
||||
return
|
||||
}
|
||||
if rows > 0 {
|
||||
r.log.Info("audit_retention.purged", "count", rows, "before", cutoff.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user