feat(job): job progress tracking
This commit is contained in:
parent
2eead05c34
commit
e554149e8b
@ -20,16 +20,18 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Job struct {
|
type Job struct {
|
||||||
ID string
|
ID string
|
||||||
Artist string
|
Artist string
|
||||||
Album string
|
Album string
|
||||||
Path string
|
Path string
|
||||||
Status JobStatus
|
Status JobStatus
|
||||||
Source JobSource
|
Source JobSource
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
FinishedAt *time.Time
|
FinishedAt *time.Time
|
||||||
DurationMs int64
|
DurationMs int64
|
||||||
ErrorMsg string
|
ErrorMsg string
|
||||||
|
FilesTotal int
|
||||||
|
FilesProcessed int
|
||||||
}
|
}
|
||||||
|
|
||||||
type LogEntry struct {
|
type LogEntry struct {
|
||||||
|
|||||||
@ -43,12 +43,12 @@ func (r *JobRepository) Create(ctx context.Context, job *models.Job) error {
|
|||||||
|
|
||||||
func (r *JobRepository) Get(ctx context.Context, id string) (*models.Job, error) {
|
func (r *JobRepository) Get(ctx context.Context, id string) (*models.Job, error) {
|
||||||
row := r.db.QueryRowContext(ctx,
|
row := r.db.QueryRowContext(ctx,
|
||||||
`SELECT id, artist, album, path, status, source, created_at, finished_at, duration_ms, error_msg
|
`SELECT id, artist, album, path, status, source, created_at, finished_at, duration_ms, error_msg, files_total, files_processed
|
||||||
FROM jobs WHERE id = ?`, id)
|
FROM jobs WHERE id = ?`, id)
|
||||||
|
|
||||||
var job models.Job
|
var job models.Job
|
||||||
err := row.Scan(&job.ID, &job.Artist, &job.Album, &job.Path, &job.Status, &job.Source,
|
err := row.Scan(&job.ID, &job.Artist, &job.Album, &job.Path, &job.Status, &job.Source,
|
||||||
&job.CreatedAt, &job.FinishedAt, &job.DurationMs, &job.ErrorMsg)
|
&job.CreatedAt, &job.FinishedAt, &job.DurationMs, &job.ErrorMsg, &job.FilesTotal, &job.FilesProcessed)
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
return nil, ErrNotFound
|
return nil, ErrNotFound
|
||||||
}
|
}
|
||||||
@ -59,7 +59,7 @@ func (r *JobRepository) Get(ctx context.Context, id string) (*models.Job, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *JobRepository) List(ctx context.Context, params ListJobsParams) ([]models.Job, error) {
|
func (r *JobRepository) List(ctx context.Context, params ListJobsParams) ([]models.Job, error) {
|
||||||
query := `SELECT id, artist, album, path, status, source, created_at, finished_at, duration_ms, error_msg
|
query := `SELECT id, artist, album, path, status, source, created_at, finished_at, duration_ms, error_msg, files_total, files_processed
|
||||||
FROM jobs`
|
FROM jobs`
|
||||||
args := []any{}
|
args := []any{}
|
||||||
|
|
||||||
@ -89,7 +89,7 @@ func (r *JobRepository) List(ctx context.Context, params ListJobsParams) ([]mode
|
|||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var job models.Job
|
var job models.Job
|
||||||
if err := rows.Scan(&job.ID, &job.Artist, &job.Album, &job.Path, &job.Status, &job.Source,
|
if err := rows.Scan(&job.ID, &job.Artist, &job.Album, &job.Path, &job.Status, &job.Source,
|
||||||
&job.CreatedAt, &job.FinishedAt, &job.DurationMs, &job.ErrorMsg); err != nil {
|
&job.CreatedAt, &job.FinishedAt, &job.DurationMs, &job.ErrorMsg, &job.FilesTotal, &job.FilesProcessed); err != nil {
|
||||||
return nil, fmt.Errorf("scan job: %w", err)
|
return nil, fmt.Errorf("scan job: %w", err)
|
||||||
}
|
}
|
||||||
jobs = append(jobs, job)
|
jobs = append(jobs, job)
|
||||||
@ -114,6 +114,24 @@ func (r *JobRepository) Count(ctx context.Context, status *models.JobStatus) (in
|
|||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *JobRepository) UpdateProgress(ctx context.Context, id string, filesProcessed, filesTotal int) error {
|
||||||
|
result, err := r.db.ExecContext(ctx,
|
||||||
|
`UPDATE jobs SET files_processed = ?, files_total = ? WHERE id = ?`,
|
||||||
|
filesProcessed, filesTotal, id,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("update job progress: %w", err)
|
||||||
|
}
|
||||||
|
rows, err := result.RowsAffected()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("rows affected: %w", err)
|
||||||
|
}
|
||||||
|
if rows == 0 {
|
||||||
|
return ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *JobRepository) UpdateStatus(ctx context.Context, id string, status models.JobStatus) error {
|
func (r *JobRepository) UpdateStatus(ctx context.Context, id string, status models.JobStatus) error {
|
||||||
result, err := r.db.ExecContext(ctx, `UPDATE jobs SET status = ? WHERE id = ?`, status, id)
|
result, err := r.db.ExecContext(ctx, `UPDATE jobs SET status = ? WHERE id = ?`, status, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -65,6 +65,13 @@ func migrate(ctx context.Context, db *sql.DB) error {
|
|||||||
`CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)`,
|
`CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)`,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
version: 3,
|
||||||
|
sql: []string{
|
||||||
|
`ALTER TABLE jobs ADD COLUMN files_total INTEGER NOT NULL DEFAULT 0`,
|
||||||
|
`ALTER TABLE jobs ADD COLUMN files_processed INTEGER NOT NULL DEFAULT 0`,
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, m := range migrations {
|
for _, m := range migrations {
|
||||||
|
|||||||
@ -128,6 +128,50 @@ func TestJobUpdateStatusNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJobUpdateProgress(t *testing.T) {
|
||||||
|
st := openTestDB(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
job := &models.Job{Artist: "A", Album: "B", Path: "/p", Status: models.StatusProcessing, Source: models.SourceManual}
|
||||||
|
if err := st.Jobs.Create(ctx, job); err != nil {
|
||||||
|
t.Fatalf("Create: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := st.Jobs.UpdateProgress(ctx, job.ID, 0, 5); err != nil {
|
||||||
|
t.Fatalf("UpdateProgress (set total): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, _ := st.Jobs.Get(ctx, job.ID)
|
||||||
|
if got.FilesTotal != 5 {
|
||||||
|
t.Errorf("FilesTotal = %d, want 5", got.FilesTotal)
|
||||||
|
}
|
||||||
|
if got.FilesProcessed != 0 {
|
||||||
|
t.Errorf("FilesProcessed = %d, want 0", got.FilesProcessed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := st.Jobs.UpdateProgress(ctx, job.ID, 3, 5); err != nil {
|
||||||
|
t.Fatalf("UpdateProgress (mid): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, _ = st.Jobs.Get(ctx, job.ID)
|
||||||
|
if got.FilesProcessed != 3 {
|
||||||
|
t.Errorf("FilesProcessed = %d, want 3", got.FilesProcessed)
|
||||||
|
}
|
||||||
|
if got.FilesTotal != 5 {
|
||||||
|
t.Errorf("FilesTotal = %d, want 5", got.FilesTotal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobUpdateProgressNotFound(t *testing.T) {
|
||||||
|
st := openTestDB(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
err := st.Jobs.UpdateProgress(ctx, "nonexistent", 1, 5)
|
||||||
|
if err != ErrNotFound {
|
||||||
|
t.Errorf("expected ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJobFinish(t *testing.T) {
|
func TestJobFinish(t *testing.T) {
|
||||||
st := openTestDB(t)
|
st := openTestDB(t)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|||||||
@ -81,11 +81,12 @@ func (h *Handler) jobCount(ctx context.Context, status models.JobStatus) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type jobsData struct {
|
type jobsData struct {
|
||||||
Jobs []models.Job
|
Jobs []models.Job
|
||||||
Page int
|
Page int
|
||||||
TotalCount int
|
TotalCount int
|
||||||
PageSize int
|
PageSize int
|
||||||
Status string
|
Status string
|
||||||
|
HasProcessing bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) Jobs(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) Jobs(w http.ResponseWriter, r *http.Request) {
|
||||||
@ -113,6 +114,43 @@ func (h *Handler) Jobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
total, _ := h.store.Jobs.Count(r.Context(), statusFilter)
|
total, _ := h.store.Jobs.Count(r.Context(), statusFilter)
|
||||||
|
|
||||||
|
jd := jobsData{
|
||||||
|
Jobs: jobs,
|
||||||
|
Page: page,
|
||||||
|
PageSize: pageSize,
|
||||||
|
TotalCount: total,
|
||||||
|
Status: r.URL.Query().Get("status"),
|
||||||
|
HasProcessing: h.jobCount(r.Context(), models.StatusProcessing) > 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.render.Render(w, "jobs.html", jd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) JobsRows(w http.ResponseWriter, r *http.Request) {
|
||||||
|
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||||
|
if page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
pageSize := 20
|
||||||
|
|
||||||
|
var statusFilter *models.JobStatus
|
||||||
|
if s := r.URL.Query().Get("status"); s != "" {
|
||||||
|
st := models.JobStatus(s)
|
||||||
|
statusFilter = &st
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs, err := h.store.Jobs.List(r.Context(), store.ListJobsParams{
|
||||||
|
Status: statusFilter,
|
||||||
|
Limit: pageSize,
|
||||||
|
Offset: (page - 1) * pageSize,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "failed to list jobs", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
total, _ := h.store.Jobs.Count(r.Context(), statusFilter)
|
||||||
|
|
||||||
jd := jobsData{
|
jd := jobsData{
|
||||||
Jobs: jobs,
|
Jobs: jobs,
|
||||||
Page: page,
|
Page: page,
|
||||||
@ -121,5 +159,5 @@ func (h *Handler) Jobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
Status: r.URL.Query().Get("status"),
|
Status: r.URL.Query().Get("status"),
|
||||||
}
|
}
|
||||||
|
|
||||||
h.render.Render(w, "jobs.html", jd)
|
h.render.Render(w, "partials/jobs_rows.html", jd)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -50,6 +50,7 @@ func Router(d Deps) chi.Router {
|
|||||||
r.Post("/settings", h.SettingsSave)
|
r.Post("/settings", h.SettingsSave)
|
||||||
|
|
||||||
r.Get("/partials/status", h.Status)
|
r.Get("/partials/status", h.Status)
|
||||||
|
r.Get("/partials/jobs", h.JobsRows)
|
||||||
|
|
||||||
r.Get("/api/library/browse", h.LibraryBrowse)
|
r.Get("/api/library/browse", h.LibraryBrowse)
|
||||||
r.Get("/api/library/scan", h.LibraryScan)
|
r.Get("/api/library/scan", h.LibraryScan)
|
||||||
|
|||||||
@ -302,3 +302,59 @@ func TestLibraryConvertInvalidPath(t *testing.T) {
|
|||||||
t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJobsRowsPartialProgressBar(t *testing.T) {
|
||||||
|
musicPath := t.TempDir()
|
||||||
|
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||||
|
st, err := store.New(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("store.New: %v", err)
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
|
||||||
|
svc, err := config.NewService(context.Background(), &mockStore{data: map[string]string{
|
||||||
|
"MUSIC_PATH": musicPath,
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewService: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
q := queue.New(100)
|
||||||
|
jobSvc := jobs.New(st, q)
|
||||||
|
sc := scanner.New(&config.Config{MusicPath: musicPath})
|
||||||
|
|
||||||
|
router := Router(Deps{
|
||||||
|
Cfg: svc,
|
||||||
|
Store: st,
|
||||||
|
Queue: q,
|
||||||
|
Scanner: sc,
|
||||||
|
Enqueuer: jobSvc,
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
job := &models.Job{
|
||||||
|
Artist: "Test Artist",
|
||||||
|
Album: "Test Album",
|
||||||
|
Path: "/music/test",
|
||||||
|
Status: models.StatusProcessing,
|
||||||
|
Source: models.SourceManual,
|
||||||
|
}
|
||||||
|
if err := st.Jobs.Create(ctx, job); err != nil {
|
||||||
|
t.Fatalf("Create: %v", err)
|
||||||
|
}
|
||||||
|
if err := st.Jobs.UpdateProgress(ctx, job.ID, 2, 5); err != nil {
|
||||||
|
t.Fatalf("UpdateProgress: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := get(t, router, "/partials/jobs")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
body := rec.Body.String()
|
||||||
|
if !strings.Contains(body, "<progress") {
|
||||||
|
t.Errorf("expected <progress> element in response, body:\n%s", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "2/5") {
|
||||||
|
t.Errorf("expected 2/5 progress text in response, body:\n%s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -141,8 +141,12 @@ func (p *Pool) process(ctx context.Context, id string) {
|
|||||||
|
|
||||||
logger.Info("starting job", "path", job.Path, "flac_count", len(flacFiles))
|
logger.Info("starting job", "path", job.Path, "flac_count", len(flacFiles))
|
||||||
|
|
||||||
|
if err := p.store.Jobs.UpdateProgress(ctx, id, 0, len(flacFiles)); err != nil {
|
||||||
|
logger.Warn("failed to set files total", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
var mp3Paths []string
|
var mp3Paths []string
|
||||||
for _, flac := range flacFiles {
|
for i, flac := range flacFiles {
|
||||||
logger.Info("converting file", "file", filepath.Base(flac))
|
logger.Info("converting file", "file", filepath.Base(flac))
|
||||||
mp3, err := p.transcoder.ConvertFile(ctx, flac)
|
mp3, err := p.transcoder.ConvertFile(ctx, flac)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -150,6 +154,9 @@ func (p *Pool) process(ctx context.Context, id string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
mp3Paths = append(mp3Paths, mp3)
|
mp3Paths = append(mp3Paths, mp3)
|
||||||
|
if err := p.store.Jobs.UpdateProgress(ctx, id, i+1, len(flacFiles)); err != nil {
|
||||||
|
logger.Warn("failed to update progress", "error", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := p.validator.Validate(ctx, mp3Paths); err != nil {
|
if err := p.validator.Validate(ctx, mp3Paths); err != nil {
|
||||||
|
|||||||
@ -424,3 +424,50 @@ func TestPoolValidationFails(t *testing.T) {
|
|||||||
t.Error("flac should be preserved when validation fails")
|
t.Error("flac should be preserved when validation fails")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWorkerRecordsProgress(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
albumDir := filepath.Join(dir, "Test Artist", "Progress Album")
|
||||||
|
if err := os.MkdirAll(albumDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{"01 track.flac", "02 track.flac", "03 track.flac"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(albumDir, name), []byte("fake flac"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
st, _, jobSvc, pool, _ := setupPool(t, map[string]string{
|
||||||
|
"DRY_RUN": "true",
|
||||||
|
"WORK_WINDOW_ENABLED": "false",
|
||||||
|
"MUSIC_PATH": dir,
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
id, err := jobSvc.Enqueue(ctx, jobs.EnqueueInput{
|
||||||
|
Artist: "Test Artist",
|
||||||
|
Album: "Progress Album",
|
||||||
|
Path: albumDir,
|
||||||
|
Source: models.SourceManual,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Enqueue: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
pool.process(ctx, id)
|
||||||
|
|
||||||
|
job, err := st.Jobs.Get(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get: %v", err)
|
||||||
|
}
|
||||||
|
if job.FilesTotal != 3 {
|
||||||
|
t.Errorf("FilesTotal = %d, want 3", job.FilesTotal)
|
||||||
|
}
|
||||||
|
if job.FilesProcessed != 3 {
|
||||||
|
t.Errorf("FilesProcessed = %d, want 3", job.FilesProcessed)
|
||||||
|
}
|
||||||
|
if job.Status != models.StatusDone {
|
||||||
|
t.Errorf("Status = %q, want done", job.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -22,17 +22,13 @@
|
|||||||
<th>Source</th>
|
<th>Source</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody id="jobs-tbody"
|
||||||
{{ range .Jobs }}
|
{{ if .HasProcessing }}
|
||||||
<tr>
|
hx-get="/partials/jobs?page={{ .Page }}{{ if .Status }}&status={{ .Status }}{{ end }}"
|
||||||
<td class="text-xs">{{ .CreatedAt.Format "2006-01-02 15:04" }}</td>
|
hx-trigger="every 3s"
|
||||||
<td>{{ .Artist }}</td>
|
hx-swap="innerHTML"
|
||||||
<td>{{ .Album }}</td>
|
{{ end }}>
|
||||||
<td>{{ if .DurationMs }}{{ formatDuration .DurationMs }}{{ else }}—{{ end }}</td>
|
{{ template "partials/jobs_rows.html" . }}
|
||||||
<td>{{ statusBadge (printf "%s" .Status) }}</td>
|
|
||||||
<td><span class="badge">{{ .Source }}</span></td>
|
|
||||||
</tr>
|
|
||||||
{{ end }}
|
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
20
web/templates/partials/jobs_rows.html
Normal file
20
web/templates/partials/jobs_rows.html
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
{{ define "partials/jobs_rows.html" }}
|
||||||
|
{{ range .Jobs }}
|
||||||
|
<tr>
|
||||||
|
<td class="text-xs">{{ .CreatedAt.Format "2006-01-02 15:04" }}</td>
|
||||||
|
<td>{{ .Artist }}</td>
|
||||||
|
<td>{{ .Album }}</td>
|
||||||
|
<td>{{ if .DurationMs }}{{ formatDuration .DurationMs }}{{ else }}—{{ end }}</td>
|
||||||
|
<td>
|
||||||
|
{{ statusBadge (printf "%s" .Status) }}
|
||||||
|
{{ if and (eq (printf "%s" .Status) "processing") (gt .FilesTotal 0) }}
|
||||||
|
<div class="flex items-center gap-1 mt-1">
|
||||||
|
<progress class="progress progress-warning w-20" value="{{ .FilesProcessed }}" max="{{ .FilesTotal }}"></progress>
|
||||||
|
<span class="text-xs tabular-nums">{{ .FilesProcessed }}/{{ .FilesTotal }}</span>
|
||||||
|
</div>
|
||||||
|
{{ end }}
|
||||||
|
</td>
|
||||||
|
<td><span class="badge">{{ .Source }}</span></td>
|
||||||
|
</tr>
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
Loading…
x
Reference in New Issue
Block a user