refactor: enhance manga import job handling with error tracking and new endpoint #37
@ -1,11 +1,7 @@
|
|||||||
package com.magamochi.content.controller;
|
package com.magamochi.content.controller;
|
||||||
|
|
||||||
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
||||||
import com.magamochi.content.model.dto.FileImportRequestDTO;
|
import com.magamochi.content.model.dto.*;
|
||||||
import com.magamochi.content.model.dto.MangaContentDTO;
|
|
||||||
import com.magamochi.content.model.dto.MangaContentImagesDTO;
|
|
||||||
import com.magamochi.content.model.dto.PresignedImportRequestDTO;
|
|
||||||
import com.magamochi.content.model.dto.PresignedImportResponseDTO;
|
|
||||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||||
import com.magamochi.content.service.ContentDownloadService;
|
import com.magamochi.content.service.ContentDownloadService;
|
||||||
import com.magamochi.content.service.ContentImportService;
|
import com.magamochi.content.service.ContentImportService;
|
||||||
@ -107,4 +103,14 @@ public class ContentController {
|
|||||||
@RequestBody PresignedImportRequestDTO request) {
|
@RequestBody PresignedImportRequestDTO request) {
|
||||||
return DefaultResponseDTO.ok(contentImportService.requestPresignedImport(request));
|
return DefaultResponseDTO.ok(contentImportService.requestPresignedImport(request));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Get a list of manga import jobs",
|
||||||
|
description = "Returns a list of manga import jobs.",
|
||||||
|
tags = {"Content"},
|
||||||
|
operationId = "getMangaImportJobs")
|
||||||
|
@GetMapping(value = "/import/jobs")
|
||||||
|
public DefaultResponseDTO<List<MangaImportJobDTO>> requestPresignedImport() {
|
||||||
|
return DefaultResponseDTO.ok(contentImportService.getImportJobs());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,31 @@
|
|||||||
|
package com.magamochi.content.model.dto;
|
||||||
|
|
||||||
|
import com.magamochi.content.model.entity.MangaImportJob;
|
||||||
|
import com.magamochi.content.model.enumeration.ImportJobStatus;
|
||||||
|
import java.time.Instant;
|
||||||
|
|
||||||
|
public record MangaImportJobDTO(
|
||||||
|
Long id,
|
||||||
|
ImportJobStatus status,
|
||||||
|
Long malId,
|
||||||
|
Long aniListId,
|
||||||
|
String filename,
|
||||||
|
String s3Key,
|
||||||
|
Instant createdAt,
|
||||||
|
Instant updatedAt,
|
||||||
|
String errorMessage,
|
||||||
|
String errorStackTrace) {
|
||||||
|
public static MangaImportJobDTO from(MangaImportJob mangaImportJob) {
|
||||||
|
return new MangaImportJobDTO(
|
||||||
|
mangaImportJob.getId(),
|
||||||
|
mangaImportJob.getStatus(),
|
||||||
|
mangaImportJob.getMalId(),
|
||||||
|
mangaImportJob.getAniListId(),
|
||||||
|
mangaImportJob.getOriginalFilename(),
|
||||||
|
mangaImportJob.getS3FileKey(),
|
||||||
|
mangaImportJob.getCreatedAt(),
|
||||||
|
mangaImportJob.getUpdatedAt(),
|
||||||
|
mangaImportJob.getErrorMessage(),
|
||||||
|
mangaImportJob.getErrorStacktrace());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -30,6 +30,10 @@ public class MangaImportJob {
|
|||||||
@Enumerated(EnumType.STRING)
|
@Enumerated(EnumType.STRING)
|
||||||
private ImportJobStatus status;
|
private ImportJobStatus status;
|
||||||
|
|
||||||
|
private String errorMessage;
|
||||||
|
|
||||||
|
private String errorStacktrace;
|
||||||
|
|
||||||
@CreationTimestamp private Instant createdAt;
|
@CreationTimestamp private Instant createdAt;
|
||||||
|
|
||||||
@UpdateTimestamp private Instant updatedAt;
|
@UpdateTimestamp private Instant updatedAt;
|
||||||
|
|||||||
@ -2,8 +2,11 @@ package com.magamochi.content.queue.consumer;
|
|||||||
|
|
||||||
import static java.util.Objects.nonNull;
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
|
import com.magamochi.content.model.enumeration.ImportJobStatus;
|
||||||
import com.magamochi.content.queue.command.FileImportCommand;
|
import com.magamochi.content.queue.command.FileImportCommand;
|
||||||
import com.magamochi.content.service.ContentImportService;
|
import com.magamochi.content.service.ContentImportService;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||||
@ -24,14 +27,16 @@ public class FileImportConsumer {
|
|||||||
|
|
||||||
if (nonNull(command.mangaImportJobId())) {
|
if (nonNull(command.mangaImportJobId())) {
|
||||||
contentImportService.updateJobStatus(
|
contentImportService.updateJobStatus(
|
||||||
command.mangaImportJobId(),
|
command.mangaImportJobId(), ImportJobStatus.SUCCESS, null, null);
|
||||||
com.magamochi.content.model.enumeration.ImportJobStatus.SUCCESS);
|
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
if (nonNull(command.mangaImportJobId())) {
|
if (nonNull(command.mangaImportJobId())) {
|
||||||
|
var sw = new StringWriter();
|
||||||
|
var pw = new PrintWriter(sw);
|
||||||
|
e.printStackTrace(pw);
|
||||||
|
|
||||||
contentImportService.updateJobStatus(
|
contentImportService.updateJobStatus(
|
||||||
command.mangaImportJobId(),
|
command.mangaImportJobId(), ImportJobStatus.FAILED, e.getMessage(), sw.toString());
|
||||||
com.magamochi.content.model.enumeration.ImportJobStatus.FAILED);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
throw e;
|
throw e;
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import com.magamochi.catalog.service.MangaContentProviderService;
|
|||||||
import com.magamochi.catalog.service.MangaResolutionService;
|
import com.magamochi.catalog.service.MangaResolutionService;
|
||||||
import com.magamochi.common.exception.UnprocessableException;
|
import com.magamochi.common.exception.UnprocessableException;
|
||||||
import com.magamochi.common.model.enumeration.ContentType;
|
import com.magamochi.common.model.enumeration.ContentType;
|
||||||
|
import com.magamochi.content.model.dto.MangaImportJobDTO;
|
||||||
import com.magamochi.content.model.dto.PresignedImportRequestDTO;
|
import com.magamochi.content.model.dto.PresignedImportRequestDTO;
|
||||||
import com.magamochi.content.model.dto.PresignedImportResponseDTO;
|
import com.magamochi.content.model.dto.PresignedImportResponseDTO;
|
||||||
import com.magamochi.content.model.entity.MangaContent;
|
import com.magamochi.content.model.entity.MangaContent;
|
||||||
@ -119,12 +120,15 @@ public class ContentImportService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
public void updateJobStatus(Long jobId, ImportJobStatus status) {
|
public void updateJobStatus(
|
||||||
|
Long jobId, ImportJobStatus status, String errorMessage, String errorStacktrace) {
|
||||||
mangaImportJobRepository
|
mangaImportJobRepository
|
||||||
.findById(jobId)
|
.findById(jobId)
|
||||||
.ifPresent(
|
.ifPresent(
|
||||||
job -> {
|
job -> {
|
||||||
job.setStatus(status);
|
job.setStatus(status);
|
||||||
|
job.setErrorMessage(errorMessage);
|
||||||
|
job.setErrorStacktrace(errorStacktrace);
|
||||||
mangaImportJobRepository.save(job);
|
mangaImportJobRepository.save(job);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@ -202,4 +206,8 @@ public class ContentImportService {
|
|||||||
.build());
|
.build());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<MangaImportJobDTO> getImportJobs() {
|
||||||
|
return mangaImportJobRepository.findAll().stream().map(MangaImportJobDTO::from).toList();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -8,6 +8,8 @@ import com.magamochi.content.queue.command.FileImportCommand;
|
|||||||
import com.magamochi.content.queue.producer.FileImportProducer;
|
import com.magamochi.content.queue.producer.FileImportProducer;
|
||||||
import com.magamochi.image.service.S3Service;
|
import com.magamochi.image.service.S3Service;
|
||||||
import com.magamochi.ingestion.service.ContentProviderService;
|
import com.magamochi.ingestion.service.ContentProviderService;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
@ -47,8 +49,14 @@ public class PendingImportScannerTask {
|
|||||||
fileImportProducer.sendFileImportCommand(
|
fileImportProducer.sendFileImportCommand(
|
||||||
new FileImportCommand(mangaContentProvider.getId(), job.getS3FileKey(), job.getId()));
|
new FileImportCommand(mangaContentProvider.getId(), job.getS3FileKey(), job.getId()));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
var sw = new StringWriter();
|
||||||
|
var pw = new PrintWriter(sw);
|
||||||
|
e.printStackTrace(pw);
|
||||||
|
|
||||||
log.error("Failed to enqueue job {}", job.getId(), e);
|
log.error("Failed to enqueue job {}", job.getId(), e);
|
||||||
job.setStatus(ImportJobStatus.FAILED);
|
job.setStatus(ImportJobStatus.FAILED);
|
||||||
|
job.setErrorMessage(e.getMessage());
|
||||||
|
job.setErrorStacktrace(sw.toString());
|
||||||
mangaImportJobRepository.save(job);
|
mangaImportJobRepository.save(job);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -11,9 +11,13 @@ public record ContentProviderListDTO(@NotNull List<ContentProviderDTO> providers
|
|||||||
contentProviders.stream().map(ContentProviderDTO::from).toList());
|
contentProviders.stream().map(ContentProviderDTO::from).toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
public record ContentProviderDTO(long id, @NotBlank String name) {
|
public record ContentProviderDTO(long id, @NotBlank String name, String url, boolean active) {
|
||||||
public static ContentProviderDTO from(ContentProvider contentProvider) {
|
public static ContentProviderDTO from(ContentProvider contentProvider) {
|
||||||
return new ContentProviderDTO(contentProvider.getId(), contentProvider.getName());
|
return new ContentProviderDTO(
|
||||||
|
contentProvider.getId(),
|
||||||
|
contentProvider.getName(),
|
||||||
|
contentProvider.getUrl(),
|
||||||
|
contentProvider.isActive());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -22,6 +22,8 @@ public class ContentProvider {
|
|||||||
|
|
||||||
private String name;
|
private String name;
|
||||||
|
|
||||||
|
private String url;
|
||||||
|
|
||||||
private boolean active;
|
private boolean active;
|
||||||
|
|
||||||
private Boolean supportsContentFetch;
|
private Boolean supportsContentFetch;
|
||||||
|
|||||||
@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE manga_import_job
|
||||||
|
ADD COLUMN error_message VARCHAR,
|
||||||
|
ADD COLUMN error_stacktrace VARCHAR;
|
||||||
Loading…
x
Reference in New Issue
Block a user