feat: expirar jobs stuck com status timeout

Adiciona mecanismo de expiração periódica para jobs que ficam presos
nos status received ou processing além de um threshold configurável.

- models: novo status StatusTimeout
- store: migration v2 com índice em jobs(status); método ExpireStuck
  que faz UPDATE atômico para os status alvo anteriores ao threshold
- expirer: novo pacote com goroutine periódica que chama ExpireStuck
  e loga WARN quando algum job é expirado
- config: campos JobTimeoutMinutes (default 120) e
  ExpirerIntervalMinutes (default 10), lidos via env vars
  JOB_TIMEOUT_MINUTES e EXPIRER_INTERVAL_MINUTES
- main: instancia e inicia o Expirer no startup
- dashboard: card Timeout com contagem; badge neutral na tabela de jobs
This commit is contained in:
Rodrigo Verdiani 2026-07-07 08:43:33 -03:00
parent 7e5f4cb155
commit 2eead05c34
9 changed files with 159 additions and 39 deletions

View File

@ -12,6 +12,7 @@ import (
"time"
"coda/internal/config"
"coda/internal/expirer"
"coda/internal/jobs"
"coda/internal/lidarr"
"coda/internal/logger"
@ -74,6 +75,13 @@ func main() {
}
pool.Start(ctx)
exp := expirer.New(
st.Jobs,
time.Duration(envCfg.JobTimeoutMinutes)*time.Minute,
time.Duration(envCfg.ExpirerIntervalMinutes)*time.Minute,
)
exp.Start(ctx)
r := chi.NewRouter()
r.Use(middleware.RequestID)
r.Use(middleware.Recoverer)

View File

@ -49,25 +49,27 @@ var editableKeys = map[Key]struct{}{
var timeRegex = regexp.MustCompile(`^\d{2}:\d{2}$`)
type Config struct {
Port string
Workers int
FFmpegBin string
FFprobeBin string
LogLevel string
DryRun bool
LidarrURL string
LidarrAPIKey string
WebhookSecret string
MusicPath string
DBPath string
Timezone string
WorkWindowEnabled bool
WorkWindowStart string
WorkWindowEnd string
NtfyEnabled bool
NtfyURL string
NtfyTopic string
NtfyToken string
Port string
Workers int
FFmpegBin string
FFprobeBin string
LogLevel string
DryRun bool
LidarrURL string
LidarrAPIKey string
WebhookSecret string
MusicPath string
DBPath string
Timezone string
WorkWindowEnabled bool
WorkWindowStart string
WorkWindowEnd string
NtfyEnabled bool
NtfyURL string
NtfyTopic string
NtfyToken string
JobTimeoutMinutes int
ExpirerIntervalMinutes int
}
func Load() (*Config, error) {
@ -83,25 +85,27 @@ func Build(overrides map[string]string) *Config {
func buildConfig(overrides map[string]string) *Config {
return &Config{
Port: mergeString(overrides, "PORT", "8080"),
Workers: mergeInt(overrides, "WORKERS", 4),
FFmpegBin: mergeString(overrides, "FFMPEG_BIN", "/usr/bin/ffmpeg"),
FFprobeBin: mergeString(overrides, "FFPROBE_BIN", "/usr/bin/ffprobe"),
LogLevel: mergeString(overrides, "LOG_LEVEL", "INFO"),
DryRun: mergeBool(overrides, "DRY_RUN", false),
LidarrURL: mergeString(overrides, "LIDARR_URL", ""),
LidarrAPIKey: mergeString(overrides, "LIDARR_API_KEY", ""),
WebhookSecret: mergeString(overrides, "WEBHOOK_SECRET", ""),
MusicPath: mergeString(overrides, "MUSIC_PATH", "/music"),
DBPath: mergeString(overrides, "DB_PATH", "/data/coda.db"),
Timezone: mergeString(overrides, "TIMEZONE", "UTC"),
WorkWindowEnabled: mergeBool(overrides, "WORK_WINDOW_ENABLED", false),
WorkWindowStart: mergeString(overrides, "WORK_WINDOW_START", "08:00"),
WorkWindowEnd: mergeString(overrides, "WORK_WINDOW_END", "18:00"),
NtfyEnabled: mergeBool(overrides, "NTFY_ENABLED", false),
NtfyURL: mergeString(overrides, "NTFY_URL", ""),
NtfyTopic: mergeString(overrides, "NTFY_TOPIC", ""),
NtfyToken: mergeString(overrides, "NTFY_TOKEN", ""),
Port: mergeString(overrides, "PORT", "8080"),
Workers: mergeInt(overrides, "WORKERS", 4),
FFmpegBin: mergeString(overrides, "FFMPEG_BIN", "/usr/bin/ffmpeg"),
FFprobeBin: mergeString(overrides, "FFPROBE_BIN", "/usr/bin/ffprobe"),
LogLevel: mergeString(overrides, "LOG_LEVEL", "INFO"),
DryRun: mergeBool(overrides, "DRY_RUN", false),
LidarrURL: mergeString(overrides, "LIDARR_URL", ""),
LidarrAPIKey: mergeString(overrides, "LIDARR_API_KEY", ""),
WebhookSecret: mergeString(overrides, "WEBHOOK_SECRET", ""),
MusicPath: mergeString(overrides, "MUSIC_PATH", "/music"),
DBPath: mergeString(overrides, "DB_PATH", "/data/coda.db"),
Timezone: mergeString(overrides, "TIMEZONE", "UTC"),
WorkWindowEnabled: mergeBool(overrides, "WORK_WINDOW_ENABLED", false),
WorkWindowStart: mergeString(overrides, "WORK_WINDOW_START", "08:00"),
WorkWindowEnd: mergeString(overrides, "WORK_WINDOW_END", "18:00"),
NtfyEnabled: mergeBool(overrides, "NTFY_ENABLED", false),
NtfyURL: mergeString(overrides, "NTFY_URL", ""),
NtfyTopic: mergeString(overrides, "NTFY_TOPIC", ""),
NtfyToken: mergeString(overrides, "NTFY_TOKEN", ""),
JobTimeoutMinutes: mergeInt(overrides, "JOB_TIMEOUT_MINUTES", 120),
ExpirerIntervalMinutes: mergeInt(overrides, "EXPIRER_INTERVAL_MINUTES", 10),
}
}

View File

@ -0,0 +1,57 @@
package expirer
import (
"context"
"log/slog"
"time"
"coda/internal/models"
)
type JobExpirer interface {
ExpireStuck(ctx context.Context, statuses []models.JobStatus, olderThan time.Duration) (int, error)
}
type Expirer struct {
jobs JobExpirer
timeout time.Duration
interval time.Duration
}
func New(jobs JobExpirer, timeout, interval time.Duration) *Expirer {
return &Expirer{
jobs: jobs,
timeout: timeout,
interval: interval,
}
}
func (e *Expirer) Start(ctx context.Context) {
go e.run(ctx)
}
func (e *Expirer) run(ctx context.Context) {
ticker := time.NewTicker(e.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
e.expire(ctx)
}
}
}
func (e *Expirer) expire(ctx context.Context) {
statuses := []models.JobStatus{models.StatusReceived, models.StatusProcessing}
n, err := e.jobs.ExpireStuck(ctx, statuses, e.timeout)
if err != nil {
slog.Error("expirer: failed to expire stuck jobs", "error", err)
return
}
if n > 0 {
slog.Warn("expirer: expired stuck jobs", "count", n, "timeout", e.timeout)
}
}

View File

@ -9,6 +9,7 @@ const (
StatusProcessing JobStatus = "processing"
StatusDone JobStatus = "done"
StatusError JobStatus = "error"
StatusTimeout JobStatus = "timeout"
)
type JobSource string

View File

@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"coda/internal/models"
@ -145,3 +146,35 @@ func (r *JobRepository) Finish(ctx context.Context, id string, status models.Job
}
return nil
}
func (r *JobRepository) ExpireStuck(ctx context.Context, statuses []models.JobStatus, olderThan time.Duration) (int, error) {
if len(statuses) == 0 {
return 0, nil
}
placeholders := make([]string, len(statuses))
args := make([]any, 0, len(statuses)+1)
for i, status := range statuses {
placeholders[i] = "?"
args = append(args, status)
}
args = append(args, time.Now().UTC().Add(-olderThan))
query := fmt.Sprintf(
`UPDATE jobs SET status = ?, finished_at = ?, error_msg = ? WHERE status IN (%s) AND created_at < ?`,
strings.Join(placeholders, ","),
)
finalArgs := []any{models.StatusTimeout, time.Now().UTC(), "job timed out"}
finalArgs = append(finalArgs, args...)
result, err := r.db.ExecContext(ctx, query, finalArgs...)
if err != nil {
return 0, fmt.Errorf("expire stuck jobs: %w", err)
}
rows, err := result.RowsAffected()
if err != nil {
return 0, fmt.Errorf("rows affected: %w", err)
}
return int(rows), nil
}

View File

@ -59,6 +59,12 @@ func migrate(ctx context.Context, db *sql.DB) error {
)`,
},
},
{
version: 2,
sql: []string{
`CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)`,
},
},
}
for _, m := range migrations {

View File

@ -23,6 +23,7 @@ type dashboardData struct {
ProcessingCount int
DoneCount int
ErrorCount int
TimeoutCount int
}
func (h *Handler) Dashboard(w http.ResponseWriter, r *http.Request) {
@ -45,6 +46,7 @@ func (h *Handler) Dashboard(w http.ResponseWriter, r *http.Request) {
dd.ProcessingCount = h.jobCount(ctx, models.StatusProcessing)
dd.DoneCount = h.jobCount(ctx, models.StatusDone)
dd.ErrorCount = h.jobCount(ctx, models.StatusError)
dd.TimeoutCount = h.jobCount(ctx, models.StatusTimeout)
h.render.Render(w, "dashboard.html", dd)
}
@ -64,6 +66,7 @@ func (h *Handler) Status(w http.ResponseWriter, r *http.Request) {
ProcessingCount: h.jobCount(ctx, models.StatusProcessing),
DoneCount: h.jobCount(ctx, models.StatusDone),
ErrorCount: h.jobCount(ctx, models.StatusError),
TimeoutCount: h.jobCount(ctx, models.StatusTimeout),
}
h.render.Render(w, "status.html", dd)

View File

@ -81,6 +81,8 @@ func statusBadge(level string) template.HTML {
return template.HTML(`<span class="badge badge-error">` + level + `</span>`)
case "received":
return template.HTML(`<span class="badge badge-info">` + level + `</span>`)
case "timeout":
return template.HTML(`<span class="badge badge-neutral">` + level + `</span>`)
default:
return template.HTML(level)
}

View File

@ -31,7 +31,7 @@
</div>
</div>
<div class="grid grid-cols-2 md:grid-cols-4 gap-4">
<div class="grid grid-cols-2 md:grid-cols-5 gap-4">
<div class="card bg-base-100 shadow">
<div class="card-body">
<h2 class="card-title">Received</h2>
@ -56,6 +56,12 @@
<p class="text-4xl font-bold text-error">{{ .ErrorCount }}</p>
</div>
</div>
<div class="card bg-base-100 shadow">
<div class="card-body">
<h2 class="card-title">Timeout</h2>
<p class="text-4xl font-bold text-neutral">{{ .TimeoutCount }}</p>
</div>
</div>
</div>
<div class="card bg-base-100 shadow">