Compare commits
No commits in common. "92dce8eb00be030af8d440fa9fbbd1dcfa3fe4e8" and "0bb2e0cacc12b7325ba30e80b8fdd98fd1a74541" have entirely different histories.
92dce8eb00
...
0bb2e0cacc
@ -4,8 +4,6 @@ import com.magamochi.common.model.dto.DefaultResponseDTO;
|
|||||||
import com.magamochi.content.model.dto.FileImportRequestDTO;
|
import com.magamochi.content.model.dto.FileImportRequestDTO;
|
||||||
import com.magamochi.content.model.dto.MangaContentDTO;
|
import com.magamochi.content.model.dto.MangaContentDTO;
|
||||||
import com.magamochi.content.model.dto.MangaContentImagesDTO;
|
import com.magamochi.content.model.dto.MangaContentImagesDTO;
|
||||||
import com.magamochi.content.model.dto.PresignedImportRequestDTO;
|
|
||||||
import com.magamochi.content.model.dto.PresignedImportResponseDTO;
|
|
||||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||||
import com.magamochi.content.service.ContentDownloadService;
|
import com.magamochi.content.service.ContentDownloadService;
|
||||||
import com.magamochi.content.service.ContentImportService;
|
import com.magamochi.content.service.ContentImportService;
|
||||||
@ -95,16 +93,4 @@ public class ContentController {
|
|||||||
|
|
||||||
return DefaultResponseDTO.ok().build();
|
return DefaultResponseDTO.ok().build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Operation(
|
|
||||||
summary = "Request presigned URL for import",
|
|
||||||
description =
|
|
||||||
"Generates a presigned URL to upload a file directly to S3 and registers a pending import job.",
|
|
||||||
tags = {"Content"},
|
|
||||||
operationId = "requestPresignedImport")
|
|
||||||
@PostMapping(value = "/import/presigned")
|
|
||||||
public DefaultResponseDTO<PresignedImportResponseDTO> requestPresignedImport(
|
|
||||||
@RequestBody PresignedImportRequestDTO request) {
|
|
||||||
return DefaultResponseDTO.ok(contentImportService.requestPresignedImport(request));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,8 +0,0 @@
|
|||||||
package com.magamochi.content.model.dto;
|
|
||||||
|
|
||||||
import jakarta.validation.constraints.NotBlank;
|
|
||||||
import lombok.Builder;
|
|
||||||
|
|
||||||
@Builder
|
|
||||||
public record PresignedImportRequestDTO(
|
|
||||||
Long malId, Long aniListId, @NotBlank String originalFilename) {}
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
package com.magamochi.content.model.dto;
|
|
||||||
|
|
||||||
import lombok.Builder;
|
|
||||||
|
|
||||||
@Builder
|
|
||||||
public record PresignedImportResponseDTO(Long jobId, String presignedUrl, String fileKey) {}
|
|
||||||
@ -1,36 +0,0 @@
|
|||||||
package com.magamochi.content.model.entity;
|
|
||||||
|
|
||||||
import com.magamochi.content.model.enumeration.ImportJobStatus;
|
|
||||||
import jakarta.persistence.*;
|
|
||||||
import java.time.Instant;
|
|
||||||
import lombok.*;
|
|
||||||
import org.hibernate.annotations.CreationTimestamp;
|
|
||||||
import org.hibernate.annotations.UpdateTimestamp;
|
|
||||||
|
|
||||||
@Getter
|
|
||||||
@Setter
|
|
||||||
@Builder
|
|
||||||
@NoArgsConstructor
|
|
||||||
@AllArgsConstructor
|
|
||||||
@Entity
|
|
||||||
@Table(name = "manga_import_job")
|
|
||||||
public class MangaImportJob {
|
|
||||||
@Id
|
|
||||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
|
||||||
private Long id;
|
|
||||||
|
|
||||||
private Long malId;
|
|
||||||
|
|
||||||
private Long aniListId;
|
|
||||||
|
|
||||||
private String originalFilename;
|
|
||||||
|
|
||||||
private String s3FileKey;
|
|
||||||
|
|
||||||
@Enumerated(EnumType.STRING)
|
|
||||||
private ImportJobStatus status;
|
|
||||||
|
|
||||||
@CreationTimestamp private Instant createdAt;
|
|
||||||
|
|
||||||
@UpdateTimestamp private Instant updatedAt;
|
|
||||||
}
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
package com.magamochi.content.model.enumeration;
|
|
||||||
|
|
||||||
public enum ImportJobStatus {
|
|
||||||
PENDING,
|
|
||||||
PROCESSING,
|
|
||||||
SUCCESS,
|
|
||||||
FAILED
|
|
||||||
}
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
package com.magamochi.content.model.repository;
|
|
||||||
|
|
||||||
import com.magamochi.content.model.entity.MangaImportJob;
|
|
||||||
import com.magamochi.content.model.enumeration.ImportJobStatus;
|
|
||||||
import java.util.List;
|
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
|
||||||
import org.springframework.stereotype.Repository;
|
|
||||||
|
|
||||||
@Repository
|
|
||||||
public interface MangaImportJobRepository extends JpaRepository<MangaImportJob, Long> {
|
|
||||||
List<MangaImportJob> findByStatus(ImportJobStatus status);
|
|
||||||
}
|
|
||||||
@ -1,4 +1,3 @@
|
|||||||
package com.magamochi.content.queue.command;
|
package com.magamochi.content.queue.command;
|
||||||
|
|
||||||
public record FileImportCommand(
|
public record FileImportCommand(long mangaContentProviderId, String filename) {}
|
||||||
long mangaContentProviderId, String filename, Long mangaImportJobId) {}
|
|
||||||
|
|||||||
@ -1,7 +1,5 @@
|
|||||||
package com.magamochi.content.queue.consumer;
|
package com.magamochi.content.queue.consumer;
|
||||||
|
|
||||||
import static java.util.Objects.nonNull;
|
|
||||||
|
|
||||||
import com.magamochi.content.queue.command.FileImportCommand;
|
import com.magamochi.content.queue.command.FileImportCommand;
|
||||||
import com.magamochi.content.service.ContentImportService;
|
import com.magamochi.content.service.ContentImportService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
@ -18,23 +16,6 @@ public class FileImportConsumer {
|
|||||||
@RabbitListener(queues = "${queues.file-import}")
|
@RabbitListener(queues = "${queues.file-import}")
|
||||||
public void receiveFileImportCommand(FileImportCommand command) {
|
public void receiveFileImportCommand(FileImportCommand command) {
|
||||||
log.info("Received file import command: {}", command);
|
log.info("Received file import command: {}", command);
|
||||||
try {
|
contentImportService.importFile(command.mangaContentProviderId(), command.filename());
|
||||||
contentImportService.importFile(
|
|
||||||
command.mangaContentProviderId(), command.filename(), command.mangaImportJobId());
|
|
||||||
|
|
||||||
if (nonNull(command.mangaImportJobId())) {
|
|
||||||
contentImportService.updateJobStatus(
|
|
||||||
command.mangaImportJobId(),
|
|
||||||
com.magamochi.content.model.enumeration.ImportJobStatus.SUCCESS);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
if (nonNull(command.mangaImportJobId())) {
|
|
||||||
contentImportService.updateJobStatus(
|
|
||||||
command.mangaImportJobId(),
|
|
||||||
com.magamochi.content.model.enumeration.ImportJobStatus.FAILED);
|
|
||||||
}
|
|
||||||
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,20 +1,14 @@
|
|||||||
package com.magamochi.content.service;
|
package com.magamochi.content.service;
|
||||||
|
|
||||||
import static java.util.Objects.isNull;
|
import static java.util.Objects.isNull;
|
||||||
import static java.util.Objects.nonNull;
|
|
||||||
import static org.apache.commons.lang3.StringUtils.isBlank;
|
import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||||
|
|
||||||
import com.magamochi.catalog.service.MangaContentProviderService;
|
import com.magamochi.catalog.service.MangaContentProviderService;
|
||||||
import com.magamochi.catalog.service.MangaResolutionService;
|
import com.magamochi.catalog.service.MangaResolutionService;
|
||||||
import com.magamochi.common.exception.UnprocessableException;
|
import com.magamochi.common.exception.UnprocessableException;
|
||||||
import com.magamochi.common.model.enumeration.ContentType;
|
import com.magamochi.common.model.enumeration.ContentType;
|
||||||
import com.magamochi.content.model.dto.PresignedImportRequestDTO;
|
|
||||||
import com.magamochi.content.model.dto.PresignedImportResponseDTO;
|
|
||||||
import com.magamochi.content.model.entity.MangaContentImage;
|
import com.magamochi.content.model.entity.MangaContentImage;
|
||||||
import com.magamochi.content.model.entity.MangaImportJob;
|
|
||||||
import com.magamochi.content.model.enumeration.ImportJobStatus;
|
|
||||||
import com.magamochi.content.model.repository.MangaContentImageRepository;
|
import com.magamochi.content.model.repository.MangaContentImageRepository;
|
||||||
import com.magamochi.content.model.repository.MangaImportJobRepository;
|
|
||||||
import com.magamochi.content.queue.command.FileImportCommand;
|
import com.magamochi.content.queue.command.FileImportCommand;
|
||||||
import com.magamochi.content.queue.producer.FileImportProducer;
|
import com.magamochi.content.queue.producer.FileImportProducer;
|
||||||
import com.magamochi.image.service.ImageFetchService;
|
import com.magamochi.image.service.ImageFetchService;
|
||||||
@ -31,7 +25,6 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import net.greypanther.natsort.CaseInsensitiveSimpleNaturalComparator;
|
import net.greypanther.natsort.CaseInsensitiveSimpleNaturalComparator;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Propagation;
|
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
@ -48,7 +41,6 @@ public class ContentImportService {
|
|||||||
|
|
||||||
private final FileImportProducer fileImportProducer;
|
private final FileImportProducer fileImportProducer;
|
||||||
private final MangaContentImageRepository mangaContentImageRepository;
|
private final MangaContentImageRepository mangaContentImageRepository;
|
||||||
private final MangaImportJobRepository mangaImportJobRepository;
|
|
||||||
private final ImageService imageService;
|
private final ImageService imageService;
|
||||||
|
|
||||||
public void importFiles(String malId, String aniListId, @NotNull List<MultipartFile> files) {
|
public void importFiles(String malId, String aniListId, @NotNull List<MultipartFile> files) {
|
||||||
@ -81,86 +73,43 @@ public class ContentImportService {
|
|||||||
log.info("Temp file uploaded to S3: {}", filename);
|
log.info("Temp file uploaded to S3: {}", filename);
|
||||||
|
|
||||||
fileImportProducer.sendFileImportCommand(
|
fileImportProducer.sendFileImportCommand(
|
||||||
new FileImportCommand(mangaContentProvider.getId(), filename, null));
|
new FileImportCommand(mangaContentProvider.getId(), filename));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new UnprocessableException("Failed to upload file to S3");
|
throw new UnprocessableException("Failed to upload file to S3");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public PresignedImportResponseDTO requestPresignedImport(PresignedImportRequestDTO request) {
|
|
||||||
if (isNull(request.malId()) && isNull(request.aniListId())) {
|
|
||||||
throw new UnprocessableException("Either MyAnimeList or AniList IDs are required.");
|
|
||||||
}
|
|
||||||
|
|
||||||
var uuid = UUID.randomUUID().toString();
|
|
||||||
var fileKey = "temp/import/" + uuid + "-" + request.originalFilename();
|
|
||||||
|
|
||||||
var job =
|
|
||||||
mangaImportJobRepository.save(
|
|
||||||
MangaImportJob.builder()
|
|
||||||
.malId(request.malId())
|
|
||||||
.aniListId(request.aniListId())
|
|
||||||
.originalFilename(request.originalFilename())
|
|
||||||
.s3FileKey(fileKey)
|
|
||||||
.status(ImportJobStatus.PENDING)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
var presignedUrl = s3Service.generatePresignedUploadUrl(fileKey);
|
|
||||||
|
|
||||||
return PresignedImportResponseDTO.builder()
|
|
||||||
.jobId(job.getId())
|
|
||||||
.fileKey(fileKey)
|
|
||||||
.presignedUrl(presignedUrl)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
|
||||||
public void updateJobStatus(Long jobId, ImportJobStatus status) {
|
|
||||||
mangaImportJobRepository
|
|
||||||
.findById(jobId)
|
|
||||||
.ifPresent(
|
|
||||||
job -> {
|
|
||||||
job.setStatus(status);
|
|
||||||
mangaImportJobRepository.save(job);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public void importFile(Long mangaContentProviderId, String filename, Long mangaImportJobId) {
|
public void importFile(Long mangaContentProviderId, String filename) {
|
||||||
var contentName = removeImportPrefix(removeFileExtension(filename));
|
|
||||||
|
|
||||||
if (nonNull(mangaImportJobId)) {
|
|
||||||
var jobOpt = mangaImportJobRepository.findById(mangaImportJobId);
|
|
||||||
if (jobOpt.isPresent()) {
|
|
||||||
contentName = removeFileExtension(jobOpt.get().getOriginalFilename());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var mangaContent =
|
var mangaContent =
|
||||||
contentIngestService.ingest(mangaContentProviderId, contentName, null, "en-US");
|
contentIngestService.ingest(
|
||||||
|
mangaContentProviderId,
|
||||||
|
removeImportPrefix(removeFileExtension(filename)),
|
||||||
|
null,
|
||||||
|
"en-US");
|
||||||
|
|
||||||
try (var is = s3Service.getFileStream(filename);
|
try (var is = s3Service.getFileStream(filename);
|
||||||
var zis = new ZipInputStream(is)) {
|
var zis = new ZipInputStream(is)) {
|
||||||
|
|
||||||
Map<String, byte[]> entryMap =
|
Map<String, byte[]> entryMap =
|
||||||
new TreeMap<>(CaseInsensitiveSimpleNaturalComparator.getInstance());
|
new TreeMap<>(
|
||||||
|
CaseInsensitiveSimpleNaturalComparator
|
||||||
|
.getInstance()); // TreeMap keeps keys sorted naturally
|
||||||
|
|
||||||
ZipEntry entry;
|
ZipEntry entry;
|
||||||
while ((entry = zis.getNextEntry()) != null) {
|
while ((entry = zis.getNextEntry()) != null) {
|
||||||
if (entry.isDirectory()) {
|
if (entry.isDirectory()) continue;
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var os = new ByteArrayOutputStream();
|
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||||
zis.transferTo(os);
|
zis.transferTo(os);
|
||||||
entryMap.put(entry.getName(), os.toByteArray());
|
entryMap.put(entry.getName(), os.toByteArray());
|
||||||
zis.closeEntry();
|
zis.closeEntry();
|
||||||
}
|
}
|
||||||
|
|
||||||
var position = 0;
|
int position = 0;
|
||||||
for (var sortedEntry : entryMap.entrySet()) {
|
for (Map.Entry<String, byte[]> sortedEntry : entryMap.entrySet()) {
|
||||||
var bytes = sortedEntry.getValue();
|
byte[] bytes = sortedEntry.getValue();
|
||||||
|
|
||||||
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
|
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
|
||||||
var image = imageService.find(imageId);
|
var image = imageService.find(imageId);
|
||||||
@ -185,7 +134,7 @@ public class ContentImportService {
|
|||||||
return filename;
|
return filename;
|
||||||
}
|
}
|
||||||
|
|
||||||
var lastDotIndex = filename.lastIndexOf('.');
|
int lastDotIndex = filename.lastIndexOf('.');
|
||||||
|
|
||||||
// No dot, or dot is the first character (like .gitignore)
|
// No dot, or dot is the first character (like .gitignore)
|
||||||
if (lastDotIndex <= 0) {
|
if (lastDotIndex <= 0) {
|
||||||
|
|||||||
@ -1,56 +0,0 @@
|
|||||||
package com.magamochi.content.task;
|
|
||||||
|
|
||||||
import com.magamochi.catalog.service.MangaContentProviderService;
|
|
||||||
import com.magamochi.catalog.service.MangaResolutionService;
|
|
||||||
import com.magamochi.content.model.enumeration.ImportJobStatus;
|
|
||||||
import com.magamochi.content.model.repository.MangaImportJobRepository;
|
|
||||||
import com.magamochi.content.queue.command.FileImportCommand;
|
|
||||||
import com.magamochi.content.queue.producer.FileImportProducer;
|
|
||||||
import com.magamochi.image.service.S3Service;
|
|
||||||
import com.magamochi.ingestion.service.ContentProviderService;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.extern.log4j.Log4j2;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
@Log4j2
|
|
||||||
@Component
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class PendingImportScannerTask {
|
|
||||||
private final MangaImportJobRepository mangaImportJobRepository;
|
|
||||||
private final S3Service s3Service;
|
|
||||||
private final MangaResolutionService mangaResolutionService;
|
|
||||||
private final ContentProviderService contentProviderService;
|
|
||||||
private final MangaContentProviderService mangaContentProviderService;
|
|
||||||
private final FileImportProducer fileImportProducer;
|
|
||||||
|
|
||||||
@Scheduled(fixedDelayString = "${tasks.pending-import-scanner.delay:30000}")
|
|
||||||
public void scanPendingImports() {
|
|
||||||
var pendingJobs = mangaImportJobRepository.findByStatus(ImportJobStatus.PENDING);
|
|
||||||
|
|
||||||
for (var job : pendingJobs) {
|
|
||||||
if (!s3Service.objectExists(job.getS3FileKey())) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("Found file for job {} in S3: {}", job.getId(), job.getS3FileKey());
|
|
||||||
|
|
||||||
try {
|
|
||||||
var manga = mangaResolutionService.findOrCreateManga(job.getAniListId(), job.getMalId());
|
|
||||||
|
|
||||||
var contentProvider = contentProviderService.findManualImportContentProvider();
|
|
||||||
var mangaContentProvider = mangaContentProviderService.findOrCreate(manga, contentProvider);
|
|
||||||
|
|
||||||
job.setStatus(ImportJobStatus.PROCESSING);
|
|
||||||
mangaImportJobRepository.save(job);
|
|
||||||
|
|
||||||
fileImportProducer.sendFileImportCommand(
|
|
||||||
new FileImportCommand(mangaContentProvider.getId(), job.getS3FileKey(), job.getId()));
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Failed to enqueue job {}", job.getId(), e);
|
|
||||||
job.setStatus(ImportJobStatus.FAILED);
|
|
||||||
mangaImportJobRepository.save(job);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -9,7 +9,6 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
|||||||
import software.amazon.awssdk.regions.Region;
|
import software.amazon.awssdk.regions.Region;
|
||||||
import software.amazon.awssdk.services.s3.S3Client;
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
import software.amazon.awssdk.services.s3.S3Configuration;
|
import software.amazon.awssdk.services.s3.S3Configuration;
|
||||||
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
|
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
public class S3ClientConfig {
|
public class S3ClientConfig {
|
||||||
@ -35,18 +34,4 @@ public class S3ClientConfig {
|
|||||||
.serviceConfiguration(configuration)
|
.serviceConfiguration(configuration)
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
|
||||||
public S3Presigner s3Presigner() {
|
|
||||||
var credentials = AwsBasicCredentials.create(accessKey, secretKey);
|
|
||||||
|
|
||||||
var configuration = S3Configuration.builder().pathStyleAccessEnabled(true).build();
|
|
||||||
|
|
||||||
return S3Presigner.builder()
|
|
||||||
.endpointOverride(URI.create(endpoint))
|
|
||||||
.credentialsProvider(StaticCredentialsProvider.create(credentials))
|
|
||||||
.region(Region.US_EAST_1)
|
|
||||||
.serviceConfiguration(configuration)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,23 +3,18 @@ package com.magamochi.image.service;
|
|||||||
import static java.util.Objects.nonNull;
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
import software.amazon.awssdk.core.sync.RequestBody;
|
import software.amazon.awssdk.core.sync.RequestBody;
|
||||||
import software.amazon.awssdk.services.s3.S3Client;
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
import software.amazon.awssdk.services.s3.model.*;
|
import software.amazon.awssdk.services.s3.model.*;
|
||||||
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
|
|
||||||
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
|
|
||||||
|
|
||||||
@Log4j2
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class S3Service {
|
public class S3Service {
|
||||||
@ -31,7 +26,6 @@ public class S3Service {
|
|||||||
private String baseUrl;
|
private String baseUrl;
|
||||||
|
|
||||||
private final S3Client s3Client;
|
private final S3Client s3Client;
|
||||||
private final S3Presigner s3Presigner;
|
|
||||||
|
|
||||||
public String uploadFile(byte[] data, String contentType, String filename) {
|
public String uploadFile(byte[] data, String contentType, String filename) {
|
||||||
var request =
|
var request =
|
||||||
@ -70,14 +64,14 @@ public class S3Service {
|
|||||||
}
|
}
|
||||||
|
|
||||||
final int BATCH_SIZE = 500;
|
final int BATCH_SIZE = 500;
|
||||||
var allObjects =
|
List<ObjectIdentifier> allObjects =
|
||||||
objectKeys.stream().map(key -> ObjectIdentifier.builder().key(key).build()).toList();
|
objectKeys.stream().map(key -> ObjectIdentifier.builder().key(key).build()).toList();
|
||||||
|
|
||||||
for (var i = 0; i < allObjects.size(); i += BATCH_SIZE) {
|
for (int i = 0; i < allObjects.size(); i += BATCH_SIZE) {
|
||||||
var end = Math.min(i + BATCH_SIZE, allObjects.size());
|
var end = Math.min(i + BATCH_SIZE, allObjects.size());
|
||||||
var batch = allObjects.subList(i, end);
|
List<ObjectIdentifier> batch = allObjects.subList(i, end);
|
||||||
|
|
||||||
var deleteRequest =
|
DeleteObjectsRequest deleteRequest =
|
||||||
DeleteObjectsRequest.builder()
|
DeleteObjectsRequest.builder()
|
||||||
.bucket(bucket)
|
.bucket(bucket)
|
||||||
.delete(Delete.builder().objects(batch).build())
|
.delete(Delete.builder().objects(batch).build())
|
||||||
@ -90,17 +84,23 @@ public class S3Service {
|
|||||||
response
|
response
|
||||||
.errors()
|
.errors()
|
||||||
.forEach(
|
.forEach(
|
||||||
error -> log.error("Error deleting key: {} -> {}", error.key(), error.message()));
|
error ->
|
||||||
|
System.err.println(
|
||||||
continue;
|
"Error deleting key: " + error.key() + " -> " + error.message()));
|
||||||
|
} else {
|
||||||
|
System.out.println(
|
||||||
|
"Deleted "
|
||||||
|
+ batch.size()
|
||||||
|
+ " objects successfully (batch "
|
||||||
|
+ (i / BATCH_SIZE + 1)
|
||||||
|
+ ")");
|
||||||
}
|
}
|
||||||
|
|
||||||
log.info("Deleted {} objects successfully (batch {})", batch.size(), i / BATCH_SIZE + 1);
|
|
||||||
} catch (S3Exception e) {
|
} catch (S3Exception e) {
|
||||||
log.error(
|
System.err.println(
|
||||||
"Failed to delete batch starting at index {}: {}",
|
"Failed to delete batch starting at index "
|
||||||
i,
|
+ i
|
||||||
e.awsErrorDetails().errorMessage());
|
+ ": "
|
||||||
|
+ e.awsErrorDetails().errorMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -110,33 +110,4 @@ public class S3Service {
|
|||||||
|
|
||||||
return s3Client.getObject(request);
|
return s3Client.getObject(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String generatePresignedUploadUrl(String key) {
|
|
||||||
try {
|
|
||||||
var putObjectRequest = PutObjectRequest.builder().bucket(bucket).key(key).build();
|
|
||||||
|
|
||||||
var presignRequest =
|
|
||||||
PutObjectPresignRequest.builder()
|
|
||||||
.signatureDuration(Duration.ofMinutes(60))
|
|
||||||
.putObjectRequest(putObjectRequest)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
return s3Presigner.presignPutObject(presignRequest).url().toString();
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Error generating presigned url for key: {}", key, e);
|
|
||||||
throw new RuntimeException("Failed to generate presigned url", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean objectExists(String key) {
|
|
||||||
try {
|
|
||||||
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build());
|
|
||||||
return true;
|
|
||||||
} catch (NoSuchKeyException e) {
|
|
||||||
return false;
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Error checking object existence for key: {}", key, e);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,11 +0,0 @@
|
|||||||
CREATE TABLE manga_import_job
|
|
||||||
(
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
mal_id BIGINT,
|
|
||||||
ani_list_id BIGINT,
|
|
||||||
original_filename VARCHAR(1000) NOT NULL,
|
|
||||||
s3_file_key VARCHAR(1000) NOT NULL,
|
|
||||||
status VARCHAR(50) NOT NULL,
|
|
||||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
Loading…
x
Reference in New Issue
Block a user