bugfix

internal/jobs/worker.go

@@ -20,6 +20,12 @@ type WorkerOptions struct {
 	Backoff      []time.Duration
 	Clock        clock.Clock
 	Log          *slog.Logger
+	// HandlerTimeout bounds a single handler invocation. It must be set slightly
+	// below the JobReaper's LeaseTimeout so a healthy long-running handler
+	// finishes (or is canceled) before the reaper re-leases the job to another
+	// worker and double-runs it. When <= 0 no per-job timeout is applied (the
+	// module always sets it from JobReaper.LeaseTimeout when reaping is enabled).
+	HandlerTimeout time.Duration
 	// OnDeadLetter, if non-nil, is invoked once after a job is moved to dead-letter
 	// (either because IsPermanent(err) or attempts exhausted). The hook runs in
 	// the worker goroutine; panics are recovered and logged but do not stop the
@@ -91,7 +97,9 @@ func (w *Worker) execute(ctx context.Context, job Job) {
 
 	h, ok := w.handlers[job.Type]
 	if !ok {
-		if derr := w.repo.DeadLetter(ctx, job.ID, fmt.Sprintf("unknown job type: %s", job.Type)); derr != nil {
+		if derr := w.repo.DeadLetter(ctx, job.ID, w.opts.ID, fmt.Sprintf("unknown job type: %s", job.Type)); w.leaseLost(derr, job, "dead_letter") {
+			return
+		} else if derr != nil {
 			w.opts.Log.Error("job.dead_letter_write_failed", "job_id", job.ID, "err", derr.Error())
 		}
 		w.opts.Log.Error("job.dead_letter", "job_id", job.ID, "job_type", job.Type, "reason", "unknown_type")
@@ -101,7 +109,9 @@ func (w *Worker) execute(ctx context.Context, job Job) {
 
 	err := w.runHandler(ctx, h, job)
 	if err == nil {
-		if cerr := w.repo.Complete(ctx, job.ID); cerr != nil {
+		if cerr := w.repo.Complete(ctx, job.ID, w.opts.ID); w.leaseLost(cerr, job, "complete") {
+			return
+		} else if cerr != nil {
 			w.opts.Log.Error("job.complete_write_failed", "job_id", job.ID, "err", cerr.Error())
 		}
 		w.opts.Log.Info("job.complete",
@@ -113,7 +123,9 @@ func (w *Worker) execute(ctx context.Context, job Job) {
 	// Permanent or attempts exhausted -> dead-letter
 	if IsPermanent(err) || job.Attempts >= job.MaxAttempts {
 		msg := truncate(err.Error(), 2000)
-		if derr := w.repo.DeadLetter(ctx, job.ID, msg); derr != nil {
+		if derr := w.repo.DeadLetter(ctx, job.ID, w.opts.ID, msg); w.leaseLost(derr, job, "dead_letter") {
+			return
+		} else if derr != nil {
 			w.opts.Log.Error("job.dead_letter_write_failed", "job_id", job.ID, "err", derr.Error())
 		}
 		w.opts.Log.Error("job.dead_letter",
@@ -126,7 +138,9 @@ func (w *Worker) execute(ctx context.Context, job Job) {
 	delay := backoffFor(w.opts.Backoff, int(job.Attempts))
 	next := w.opts.Clock.Now().Add(delay)
 	msg := truncate(err.Error(), 2000)
-	if ferr := w.repo.FailWithRetry(ctx, job.ID, next, msg); ferr != nil {
+	if ferr := w.repo.FailWithRetry(ctx, job.ID, w.opts.ID, next, msg); w.leaseLost(ferr, job, "fail_retry") {
+		return
+	} else if ferr != nil {
 		w.opts.Log.Error("job.fail_retry_write_failed", "job_id", job.ID, "err", ferr.Error())
 	}
 	w.opts.Log.Warn("job.fail_retry",
@@ -134,8 +148,30 @@ func (w *Worker) execute(ctx context.Context, job Job) {
 		"err", msg, "next_at", next.Format(time.RFC3339))
 }
 
-// runHandler wraps Handle with panic recovery, converting panics into retryable errors.
+// leaseLost reports whether writeErr is ErrLeaseLost. The fencing predicate on
+// the terminal/retry updates matched no row, meaning this worker's lease was
+// stolen (e.g. its handler outran LeaseTimeout and the reaper re-leased the
+// job). The write is correctly dropped: we only log and drop, and must NOT
+// treat it as a failure that schedules further work — the new run owns the job.
+func (w *Worker) leaseLost(writeErr error, job Job, phase string) bool {
+	if errors.Is(writeErr, ErrLeaseLost) {
+		w.opts.Log.Warn("job.lease_lost",
+			"job_id", job.ID, "job_type", job.Type, "attempt", job.Attempts, "phase", phase)
+		return true
+	}
+	return false
+}
+
+// runHandler wraps Handle with panic recovery (converting panics into retryable
+// errors) and bounds the handler to HandlerTimeout. Deriving a per-job timeout
+// below the reaper's LeaseTimeout ensures a healthy long-running handler is
+// canceled before the reaper re-leases the job, so it is never double-run.
 func (w *Worker) runHandler(ctx context.Context, h Handler, job Job) (err error) {
+	if w.opts.HandlerTimeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, w.opts.HandlerTimeout)
+		defer cancel()
+	}
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("handler panic: %v", r)