diff --git a/src/main/java/com/magamochi/catalog/model/repository/MangaContentProviderRepository.java b/src/main/java/com/magamochi/catalog/model/repository/MangaContentProviderRepository.java index 7632849..10f21b0 100644 --- a/src/main/java/com/magamochi/catalog/model/repository/MangaContentProviderRepository.java +++ b/src/main/java/com/magamochi/catalog/model/repository/MangaContentProviderRepository.java @@ -2,9 +2,13 @@ package com.magamochi.catalog.model.repository; import com.magamochi.catalog.model.entity.MangaContentProvider; import jakarta.validation.constraints.NotBlank; +import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; public interface MangaContentProviderRepository extends JpaRepository { boolean existsByMangaTitleIgnoreCaseAndContentProvider_Id( @NotBlank String mangaTitle, long contentProviderId); + + Optional findByManga_IdAndContentProvider_Id( + long mangaId, long contentProviderId); } diff --git a/src/main/java/com/magamochi/catalog/service/MangaContentProviderService.java b/src/main/java/com/magamochi/catalog/service/MangaContentProviderService.java index 85aa949..3d90ef4 100644 --- a/src/main/java/com/magamochi/catalog/service/MangaContentProviderService.java +++ b/src/main/java/com/magamochi/catalog/service/MangaContentProviderService.java @@ -1,8 +1,10 @@ package com.magamochi.catalog.service; +import com.magamochi.catalog.model.entity.Manga; import com.magamochi.catalog.model.entity.MangaContentProvider; import com.magamochi.catalog.model.repository.MangaContentProviderRepository; import com.magamochi.common.exception.NotFoundException; +import com.magamochi.ingestion.model.entity.ContentProvider; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -19,4 +21,17 @@ public class MangaContentProviderService { new NotFoundException( "MangaContentProvider not found - ID: " + mangaContentProviderId)); } + + public MangaContentProvider findOrCreate(Manga manga, ContentProvider contentProvider) { + return mangaContentProviderRepository + .findByManga_IdAndContentProvider_Id(manga.getId(), contentProvider.getId()) + .orElseGet( + () -> + mangaContentProviderRepository.save( + MangaContentProvider.builder() + .manga(manga) + .mangaTitle(manga.getTitle()) + .contentProvider(contentProvider) + .build())); + } } diff --git a/src/main/java/com/magamochi/catalog/service/MangaResolutionService.java b/src/main/java/com/magamochi/catalog/service/MangaResolutionService.java index 51a6fcb..c27ad1b 100644 --- a/src/main/java/com/magamochi/catalog/service/MangaResolutionService.java +++ b/src/main/java/com/magamochi/catalog/service/MangaResolutionService.java @@ -98,6 +98,10 @@ public class MangaResolutionService { return Optional.of(new ProviderResult(bestTitle, malId)); } + public Manga findOrCreateManga(Long aniListId, Long malId) { + return findOrCreateManga(null, aniListId, malId); + } + private Manga findOrCreateManga(String canonicalTitle, Long aniListId, Long malId) { if (nonNull(aniListId)) { var existingByAniList = mangaRepository.findByAniListId(aniListId); @@ -113,20 +117,24 @@ public class MangaResolutionService { } } - return mangaRepository - .findByTitleIgnoreCase(canonicalTitle) - .orElseGet( - () -> { - var newManga = - Manga.builder().title(canonicalTitle).malId(malId).aniListId(aniListId).build(); + if (nonNull(canonicalTitle)) { + var existingByTitle = mangaRepository.findByTitleIgnoreCase(canonicalTitle); + if (existingByTitle.isPresent()) { + return existingByTitle.get(); + } + } - var savedManga = mangaRepository.save(newManga); + return createAndNotifyManga(canonicalTitle, aniListId, malId); + } - mangaUpdateProducer.sendMangaUpdateCommand( - new MangaUpdateCommand(savedManga.getId())); + private Manga createAndNotifyManga(String title, Long aniListId, Long malId) { + var manga = + mangaRepository.save( + Manga.builder().title(title).aniListId(aniListId).malId(malId).build()); - return savedManga; - }); + mangaUpdateProducer.sendMangaUpdateCommand(new MangaUpdateCommand(manga.getId())); + + return manga; } private record ProviderResult(String title, Long externalId) {} diff --git a/src/main/java/com/magamochi/catalog/service/MyAnimeListService.java b/src/main/java/com/magamochi/catalog/service/MyAnimeListService.java index 530745f..6396d0b 100644 --- a/src/main/java/com/magamochi/catalog/service/MyAnimeListService.java +++ b/src/main/java/com/magamochi/catalog/service/MyAnimeListService.java @@ -77,7 +77,7 @@ public class MyAnimeListService { } private MangaStatus mapStatus(String malStatus) { - return switch (malStatus) { + return switch (malStatus.toLowerCase()) { case "finished" -> MangaStatus.COMPLETED; case "publishing" -> MangaStatus.ONGOING; case "on hiatus" -> MangaStatus.HIATUS; diff --git a/src/main/java/com/magamochi/ingestion/providers/ContentProviders.java b/src/main/java/com/magamochi/common/ContentProviders.java similarity index 75% rename from src/main/java/com/magamochi/ingestion/providers/ContentProviders.java rename to src/main/java/com/magamochi/common/ContentProviders.java index ad860e6..eb7f9ac 100644 --- a/src/main/java/com/magamochi/ingestion/providers/ContentProviders.java +++ b/src/main/java/com/magamochi/common/ContentProviders.java @@ -1,8 +1,9 @@ -package com.magamochi.ingestion.providers; +package com.magamochi.common; public class ContentProviders { public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog"; public static final String MANGA_LIVRE_TO = "Manga Livre.to"; public static final String PINK_ROSA_SCAN = "Pink Rosa Scan"; public static final String MANGA_DEX = "MangaDex"; + public static final String MANUAL_IMPORT = "Manual Import"; } diff --git a/src/main/java/com/magamochi/common/config/RabbitConfig.java b/src/main/java/com/magamochi/common/config/RabbitConfig.java index 905ff59..f15bca0 100644 --- a/src/main/java/com/magamochi/common/config/RabbitConfig.java +++ b/src/main/java/com/magamochi/common/config/RabbitConfig.java @@ -37,6 +37,9 @@ public class RabbitConfig { @Value("${queues.image-fetch}") private String imageFetchQueue; + @Value("${queues.file-import}") + private String fileImportQueue; + @Value("${topics.image-updates}") private String imageUpdatesTopic; @@ -68,6 +71,11 @@ public class RabbitConfig { return new Queue(mangaCoverUpdateQueue, false); } + @Bean + public Queue fileImportQueue() { + return new Queue(fileImportQueue, false); + } + @Bean public Binding bindingMangaCoverUpdateQueue( Queue mangaCoverUpdateQueue, TopicExchange imageUpdatesExchange) { diff --git a/src/main/java/com/magamochi/content/controller/ContentController.java b/src/main/java/com/magamochi/content/controller/ContentController.java index de4088f..d671553 100644 --- a/src/main/java/com/magamochi/content/controller/ContentController.java +++ b/src/main/java/com/magamochi/content/controller/ContentController.java @@ -1,10 +1,12 @@ package com.magamochi.content.controller; import com.magamochi.common.model.dto.DefaultResponseDTO; +import com.magamochi.content.model.dto.FileImportRequestDTO; import com.magamochi.content.model.dto.MangaContentDTO; import com.magamochi.content.model.dto.MangaContentImagesDTO; import com.magamochi.content.model.enumeration.ContentArchiveFileType; import com.magamochi.content.service.ContentDownloadService; +import com.magamochi.content.service.ContentImportService; import com.magamochi.content.service.ContentService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; @@ -25,6 +27,7 @@ import org.springframework.web.bind.annotation.*; public class ContentController { private final ContentService contentService; private final ContentDownloadService contentDownloadService; + private final ContentImportService contentImportService; @Operation( summary = "Get the content for a specific manga/content provider combination", @@ -76,4 +79,18 @@ public class ContentController { .header("Content-Disposition", "attachment; filename=\"" + response.filename() + "\"") .body(response.content()); } + + @Operation( + summary = "Import multiple files", + description = "Accepts multiple content files via multipart/form-data and processes them.", + tags = {"Content"}, + operationId = "importContentFiles") + @PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public DefaultResponseDTO importContentFiles( + @ModelAttribute FileImportRequestDTO requestDTO) { + contentImportService.importFiles( + requestDTO.malId(), requestDTO.aniListId(), requestDTO.files()); + + return DefaultResponseDTO.ok().build(); + } } diff --git a/src/main/java/com/magamochi/content/model/dto/FileImportRequestDTO.java b/src/main/java/com/magamochi/content/model/dto/FileImportRequestDTO.java new file mode 100644 index 0000000..95eac4e --- /dev/null +++ b/src/main/java/com/magamochi/content/model/dto/FileImportRequestDTO.java @@ -0,0 +1,6 @@ +package com.magamochi.content.model.dto; + +import java.util.List; +import org.springframework.web.multipart.MultipartFile; + +public record FileImportRequestDTO(String malId, String aniListId, List files) {} diff --git a/src/main/java/com/magamochi/content/queue/command/FileImportCommand.java b/src/main/java/com/magamochi/content/queue/command/FileImportCommand.java new file mode 100644 index 0000000..5243531 --- /dev/null +++ b/src/main/java/com/magamochi/content/queue/command/FileImportCommand.java @@ -0,0 +1,3 @@ +package com.magamochi.content.queue.command; + +public record FileImportCommand(long mangaContentProviderId, String filename) {} diff --git a/src/main/java/com/magamochi/content/queue/consumer/FileImportConsumer.java b/src/main/java/com/magamochi/content/queue/consumer/FileImportConsumer.java new file mode 100644 index 0000000..285765b --- /dev/null +++ b/src/main/java/com/magamochi/content/queue/consumer/FileImportConsumer.java @@ -0,0 +1,21 @@ +package com.magamochi.content.queue.consumer; + +import com.magamochi.content.queue.command.FileImportCommand; +import com.magamochi.content.service.ContentImportService; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class FileImportConsumer { + private final ContentImportService contentImportService; + + @RabbitListener(queues = "${queues.file-import}") + public void receiveFileImportCommand(FileImportCommand command) { + log.info("Received file import command: {}", command); + contentImportService.importFile(command.mangaContentProviderId(), command.filename()); + } +} diff --git a/src/main/java/com/magamochi/content/queue/producer/FileImportProducer.java b/src/main/java/com/magamochi/content/queue/producer/FileImportProducer.java new file mode 100644 index 0000000..1b7f9c7 --- /dev/null +++ b/src/main/java/com/magamochi/content/queue/producer/FileImportProducer.java @@ -0,0 +1,23 @@ +package com.magamochi.content.queue.producer; + +import com.magamochi.content.queue.command.FileImportCommand; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class FileImportProducer { + private final RabbitTemplate rabbitTemplate; + + @Value("${queues.file-import}") + private String fileImportQueue; + + public void sendFileImportCommand(FileImportCommand command) { + rabbitTemplate.convertAndSend(fileImportQueue, command); + log.info("Sent file import command: {}", command); + } +} diff --git a/src/main/java/com/magamochi/content/service/ContentImportService.java b/src/main/java/com/magamochi/content/service/ContentImportService.java new file mode 100644 index 0000000..0877bb0 --- /dev/null +++ b/src/main/java/com/magamochi/content/service/ContentImportService.java @@ -0,0 +1,151 @@ +package com.magamochi.content.service; + +import static java.util.Objects.isNull; +import static org.apache.commons.lang3.StringUtils.isBlank; + +import com.magamochi.catalog.service.MangaContentProviderService; +import com.magamochi.catalog.service.MangaResolutionService; +import com.magamochi.common.exception.UnprocessableException; +import com.magamochi.common.model.enumeration.ContentType; +import com.magamochi.content.model.entity.MangaContentImage; +import com.magamochi.content.model.repository.MangaContentImageRepository; +import com.magamochi.content.queue.command.FileImportCommand; +import com.magamochi.content.queue.producer.FileImportProducer; +import com.magamochi.image.service.ImageFetchService; +import com.magamochi.image.service.ImageService; +import com.magamochi.image.service.S3Service; +import com.magamochi.ingestion.service.ContentProviderService; +import jakarta.validation.constraints.NotNull; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class ContentImportService { + private final ContentProviderService contentProviderService; + private final MangaResolutionService mangaResolutionService; + private final MangaContentProviderService mangaContentProviderService; + private final ContentIngestService contentIngestService; + private final ImageFetchService imageFetchService; + private final S3Service s3Service; + + private final FileImportProducer fileImportProducer; + private final MangaContentImageRepository mangaContentImageRepository; + private final ImageService imageService; + + public void importFiles(String malId, String aniListId, @NotNull List files) { + if (isBlank(malId) && isBlank(aniListId)) { + throw new UnprocessableException("Either MyAnimeList or AniList IDs are required."); + } + + if (files.isEmpty()) { + return; + } + + var manga = + mangaResolutionService.findOrCreateManga( + isBlank(aniListId) ? null : Long.parseLong(aniListId), + isBlank(malId) ? null : Long.parseLong(malId)); + var contentProvider = contentProviderService.findManualImportContentProvider(); + + var mangaContentProvider = mangaContentProviderService.findOrCreate(manga, contentProvider); + + var sortedFiles = files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList(); + + sortedFiles.forEach( + file -> { + try { + var filename = + s3Service.uploadFile( + file.getBytes(), + file.getContentType(), + "temp/import/" + file.getOriginalFilename()); + log.info("Temp file uploaded to S3: {}", filename); + + fileImportProducer.sendFileImportCommand( + new FileImportCommand(mangaContentProvider.getId(), filename)); + } catch (IOException e) { + throw new UnprocessableException("Failed to upload file to S3"); + } + }); + } + + @Transactional + public void importFile(Long mangaContentProviderId, String filename) { + var mangaContent = + contentIngestService.ingest( + mangaContentProviderId, + removeImportPrefix(removeFileExtension(filename)), + null, + "en-US"); + + try { + var is = s3Service.getFileStream(filename); + var zis = new ZipInputStream(is); + + ZipEntry entry; + var position = 0; + + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + + var os = new ByteArrayOutputStream(); + zis.transferTo(os); + var bytes = os.toByteArray(); + + var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE); + var image = imageService.find(imageId); + + mangaContentImageRepository.save( + MangaContentImage.builder() + .image(image) + .mangaContent(mangaContent) + .position(position) + .build()); + + zis.closeEntry(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + mangaContent.setDownloaded(true); + s3Service.deleteObjects(Set.of(filename)); + } + + private String removeFileExtension(String filename) { + if (isBlank(filename)) { + return filename; + } + + int lastDotIndex = filename.lastIndexOf('.'); + + // No dot, or dot is the first character (like .gitignore) + if (lastDotIndex <= 0) { + return filename; + } + + return filename.substring(0, lastDotIndex); + } + + private String removeImportPrefix(String path) { + if (isNull(path)) { + return null; + } + + return path.replace("temp/import/", ""); + } +} diff --git a/src/main/java/com/magamochi/content/service/ContentIngestService.java b/src/main/java/com/magamochi/content/service/ContentIngestService.java index 3d2f13b..3855a21 100644 --- a/src/main/java/com/magamochi/content/service/ContentIngestService.java +++ b/src/main/java/com/magamochi/content/service/ContentIngestService.java @@ -1,6 +1,7 @@ package com.magamochi.content.service; import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; import com.magamochi.catalog.service.LanguageService; import com.magamochi.catalog.service.MangaContentProviderService; @@ -34,22 +35,24 @@ public class ContentIngestService { private final ImageFetchProducer imageFetchProducer; private final ImageService imageService; - public void ingest( + public MangaContent ingest( long mangaContentProviderId, @NotBlank String title, - @NotBlank String url, + String url, @NotBlank String languageCode) { log.info("Ingesting Manga Content ({}) for provider {}", title, mangaContentProviderId); var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId); - if (mangaContentRepository.existsByMangaContentProvider_IdAndUrlIgnoreCase( - mangaContentProvider.getId(), url)) { - log.info( - "Manga Content ({}) for provider {} already exists. Skipped.", - title, - mangaContentProviderId); - return; + if (nonNull(url)) { + if (mangaContentRepository.existsByMangaContentProvider_IdAndUrlIgnoreCase( + mangaContentProvider.getId(), url)) { + log.info( + "Manga Content ({}) for provider {} already exists. Skipped.", + title, + mangaContentProviderId); + return null; + } } var language = languageService.find(languageCode); @@ -68,6 +71,8 @@ public class ContentIngestService { title, mangaContentProviderId, mangaContent.getId()); + + return mangaContent; } @Transactional diff --git a/src/main/java/com/magamochi/controller/MangaImportController.java b/src/main/java/com/magamochi/controller/MangaImportController.java index 06c9e19..cde4efb 100644 --- a/src/main/java/com/magamochi/controller/MangaImportController.java +++ b/src/main/java/com/magamochi/controller/MangaImportController.java @@ -6,15 +6,8 @@ import com.magamochi.model.dto.ImportRequestDTO; // import com.magamochi.service.MangaImportService; import com.magamochi.service.ProviderManualMangaImportService; import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import java.util.List; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; @RestController @RequestMapping("/manga/import") @@ -34,29 +27,4 @@ public class MangaImportController { return DefaultResponseDTO.ok( providerManualMangaImportService.importFromProvider(providerId, requestDTO)); } - - @Operation( - summary = "Upload multiple files", - description = "Accepts multiple files via multipart/form-data and processes them.", - tags = {"Manga Import"}, - operationId = "importMultipleFiles") - @PostMapping( - value = "/upload", - consumes = {"multipart/form-data"}) - public DefaultResponseDTO uploadMultipleFiles( - @RequestPart("malId") @NotBlank String malId, - @Parameter( - description = "List of files to upload", - required = true, - content = - @Content( - mediaType = "multipart/form-data", - schema = @Schema(type = "array", format = "binary"))) - @RequestPart("files") - @NotNull - List files) { - // mangaImportService.importMangaFiles(malId, files); - - return DefaultResponseDTO.ok().build(); - } } diff --git a/src/main/java/com/magamochi/image/service/ImageFetchService.java b/src/main/java/com/magamochi/image/service/ImageFetchService.java index 0b7c7eb..d522cff 100644 --- a/src/main/java/com/magamochi/image/service/ImageFetchService.java +++ b/src/main/java/com/magamochi/image/service/ImageFetchService.java @@ -37,28 +37,36 @@ public class ImageFetchService { var imageBytes = response.body(); - var fileContentType = resolveContentType(response, imageBytes); - - var fileHash = computeHash(imageBytes); - - return imageManagerService.upload( - imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash); + return uploadImage(imageBytes, response, contentType); } catch (Exception e) { log.error("Failed to fetch image from URL: {}", imageUrl, e); return null; } } - private String resolveContentType(HttpResponse response, byte[] fileBytes) { - var headerType = - response - .headers() - .firstValue("Content-Type") - .map(val -> val.split(";")[0].trim().toLowerCase()) - .orElse(null); + public UUID uploadImage( + byte[] imageBytes, HttpResponse httpResponse, ContentType contentType) + throws NoSuchAlgorithmException { + var fileContentType = resolveContentType(httpResponse, imageBytes); - if (nonNull(headerType) && headerType.startsWith("image/")) { - return headerType; + var fileHash = computeHash(imageBytes); + + return imageManagerService.upload( + imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash); + } + + private String resolveContentType(HttpResponse response, byte[] fileBytes) { + if (nonNull(response)) { + var headerType = + response + .headers() + .firstValue("Content-Type") + .map(val -> val.split(";")[0].trim().toLowerCase()) + .orElse(null); + + if (nonNull(headerType) && headerType.startsWith("image/")) { + return headerType; + } } return tika.detect(fileBytes); diff --git a/src/main/java/com/magamochi/ingestion/model/repository/ContentProviderRepository.java b/src/main/java/com/magamochi/ingestion/model/repository/ContentProviderRepository.java index 8f1e612..8232dec 100644 --- a/src/main/java/com/magamochi/ingestion/model/repository/ContentProviderRepository.java +++ b/src/main/java/com/magamochi/ingestion/model/repository/ContentProviderRepository.java @@ -1,6 +1,9 @@ package com.magamochi.ingestion.model.repository; import com.magamochi.ingestion.model.entity.ContentProvider; +import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; -public interface ContentProviderRepository extends JpaRepository {} +public interface ContentProviderRepository extends JpaRepository { + Optional findByNameIgnoreCase(String name); +} diff --git a/src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java index bdc7765..9257b16 100644 --- a/src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java @@ -5,11 +5,11 @@ import static java.util.Objects.isNull; import com.google.common.util.concurrent.RateLimiter; import com.magamochi.catalog.model.entity.MangaContentProvider; import com.magamochi.client.MangaDexClient; +import com.magamochi.common.ContentProviders; import com.magamochi.common.exception.UnprocessableException; import com.magamochi.ingestion.model.dto.ContentImageInfoDTO; import com.magamochi.ingestion.model.dto.ContentInfoDTO; import com.magamochi.ingestion.providers.ContentProvider; -import com.magamochi.ingestion.providers.ContentProviders; import com.magamochi.ingestion.providers.ManualImportContentProvider; import java.util.*; import java.util.stream.IntStream; diff --git a/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java index 3c566de..f149214 100644 --- a/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java @@ -1,11 +1,11 @@ package com.magamochi.ingestion.providers.impl; import com.magamochi.catalog.model.entity.MangaContentProvider; +import com.magamochi.common.ContentProviders; import com.magamochi.ingestion.model.dto.ContentImageInfoDTO; import com.magamochi.ingestion.model.dto.ContentInfoDTO; import com.magamochi.ingestion.model.dto.MangaInfoDTO; import com.magamochi.ingestion.providers.ContentProvider; -import com.magamochi.ingestion.providers.ContentProviders; import com.magamochi.ingestion.providers.PagedContentProvider; import java.io.IOException; import java.util.*; diff --git a/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java index 3ec9263..057a43f 100644 --- a/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java @@ -3,11 +3,11 @@ package com.magamochi.ingestion.providers.impl; import static java.util.Objects.nonNull; import com.magamochi.catalog.model.entity.MangaContentProvider; +import com.magamochi.common.ContentProviders; import com.magamochi.ingestion.model.dto.ContentImageInfoDTO; import com.magamochi.ingestion.model.dto.ContentInfoDTO; import com.magamochi.ingestion.model.dto.MangaInfoDTO; import com.magamochi.ingestion.providers.ContentProvider; -import com.magamochi.ingestion.providers.ContentProviders; import com.magamochi.ingestion.providers.PagedContentProvider; import com.magamochi.ingestion.service.FlareService; import java.util.*; diff --git a/src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java index 2e4ef56..4383634 100644 --- a/src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java @@ -4,11 +4,11 @@ import static java.util.Objects.isNull; import static org.apache.commons.lang3.StringUtils.isBlank; import com.magamochi.catalog.model.entity.MangaContentProvider; +import com.magamochi.common.ContentProviders; import com.magamochi.ingestion.model.dto.ContentImageInfoDTO; import com.magamochi.ingestion.model.dto.ContentInfoDTO; import com.magamochi.ingestion.model.dto.MangaInfoDTO; import com.magamochi.ingestion.providers.ContentProvider; -import com.magamochi.ingestion.providers.ContentProviders; import com.magamochi.ingestion.providers.PagedContentProvider; import com.magamochi.ingestion.service.FlareService; import java.util.*; diff --git a/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java b/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java index 026a31c..ab923df 100644 --- a/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java +++ b/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java @@ -2,6 +2,7 @@ package com.magamochi.ingestion.service; import static java.util.Objects.nonNull; +import com.magamochi.common.ContentProviders; import com.magamochi.common.exception.NotFoundException; import com.magamochi.ingestion.model.dto.ContentProviderListDTO; import com.magamochi.ingestion.model.entity.ContentProvider; @@ -32,4 +33,10 @@ public class ContentProviderService { new NotFoundException( "Content Provider not found (ID: " + contentProviderId + ").")); } + + public ContentProvider findManualImportContentProvider() { + return contentProviderRepository + .findByNameIgnoreCase(ContentProviders.MANUAL_IMPORT) + .orElseThrow(() -> new NotFoundException("Manual Import Content Provider not found")); + } } diff --git a/src/main/java/com/magamochi/service/MangaImportService.java b/src/main/java/com/magamochi/service/MangaImportService.java deleted file mode 100644 index 510f9c2..0000000 --- a/src/main/java/com/magamochi/service/MangaImportService.java +++ /dev/null @@ -1,438 +0,0 @@ -// package com.magamochi.service; -// -// import static java.util.Objects.isNull; -// import static java.util.Objects.nonNull; -// -// import com.google.common.util.concurrent.RateLimiter; -// import com.magamochi.catalog.model.entity.Genre; -// import com.magamochi.catalog.model.repository.GenreRepository; -// import com.magamochi.catalog.client.AniListClient; -// import com.magamochi.catalog.client.JikanClient; -// import com.magamochi.common.exception.NotFoundException; -// import com.magamochi.ingestion.model.entity.ContentProvider; -// import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; -// import com.magamochi.model.entity.*; -// import com.magamochi.model.repository.*; -// import com.magamochi.catalog.util.DoubleUtil; -// import java.io.*; -// import java.net.URI; -// import java.net.URISyntaxException; -// import java.net.URL; -// import java.time.OffsetDateTime; -// import java.time.ZoneOffset; -// import java.util.ArrayList; -// import java.util.Comparator; -// import java.util.List; -// import java.util.stream.IntStream; -// import java.util.zip.ZipEntry; -// import java.util.zip.ZipInputStream; -// import lombok.RequiredArgsConstructor; -// import lombok.extern.log4j.Log4j2; -// import org.apache.commons.lang3.StringUtils; -// import org.springframework.stereotype.Service; -// import org.springframework.web.multipart.MultipartFile; -// -// @Log4j2 -//// @Service -// @RequiredArgsConstructor -// public class MangaImportService { -// private final ProviderService providerService; -// private final MangaCreationService mangaCreationService; -// private final ImageService imageService; -// private final LanguageService languageService; -// -// private final GenreRepository genreRepository; -// private final MangaGenreRepository mangaGenreRepository; -// private final MangaContentProviderRepository mangaContentProviderRepository; -// private final AuthorRepository authorRepository; -// private final MangaAuthorRepository mangaAuthorRepository; -// private final MangaChapterRepository mangaChapterRepository; -// private final MangaRepository mangaRepository; -// -// private final JikanClient jikanClient; -// private final AniListClient aniListClient; -// private final MangaChapterImageRepository mangaChapterImageRepository; -// private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository; -// -// private final RateLimiter jikanRateLimiter; -// -// public void importMangaFiles(String malId, List files) { -// log.info("Importing manga files for MAL ID {}", malId); -// var provider = providerService.getOrCreateProvider("Manual Import", false); -// -// jikanRateLimiter.acquire(); -// var mangaData = jikanClient.getMangaById(Long.parseLong(malId)); -// -// var mangaProvider = getOrCreateMangaProvider(mangaData.data().title(), provider); -// -// var sortedFiles = -// files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList(); -// -// IntStream.rangeClosed(1, sortedFiles.size()) -// .forEach( -// fileIndex -> { -// var file = sortedFiles.get(fileIndex - 1); -// log.info( -// "Importing file {}/{}: {}, for Mangá {}", -// fileIndex, -// sortedFiles.size(), -// file.getOriginalFilename(), -// mangaProvider.getManga().getTitle()); -// -// var chapter = -// persistMangaChapter( -// mangaProvider, -// new ContentProviderMangaChapterResponseDTO( -// removeFileExtension(file.getOriginalFilename()), -// "manual_" + file.getOriginalFilename(), -// file.getOriginalFilename(), -// "en-US")); -// -// List allChapterImages = new ArrayList<>(); -// try (InputStream is = file.getInputStream(); -// ZipInputStream zis = new ZipInputStream(is)) { -// ZipEntry entry; -// var position = 0; -// -// while ((entry = zis.getNextEntry()) != null) { -// if (entry.isDirectory()) { -// continue; -// } -// -// var os = new ByteArrayOutputStream(); -// zis.transferTo(os); -// var bytes = os.toByteArray(); -// -// var image = -// imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapter.getId()); -// -// var chapterImage = -// MangaChapterImage.builder() -// .position(position++) -// .image(image) -// .mangaChapter(chapter) -// .build(); -// -// allChapterImages.add(chapterImage); -// zis.closeEntry(); -// } -// -// log.info("Chapter images added for chapter {}", chapter.getTitle()); -// } catch (IOException e) { -// throw new RuntimeException(e); -// } -// -// mangaChapterImageRepository.saveAll(allChapterImages); -// chapter.setDownloaded(true); -// mangaChapterRepository.save(chapter); -// }); -// -// log.info("Import manga files for MAL ID {} completed.", malId); -// } -// -// public void updateMangaData(Long mangaId) { -// var manga = -// mangaRepository -// .findById(mangaId) -// .orElseThrow(() -> new NotFoundException("Manga not found for ID: " + mangaId)); -// -// updateMangaData(manga); -// } -// -// public void updateMangaData(Manga manga) { -// log.info("Updating manga {}", manga.getTitle()); -// -// if (nonNull(manga.getMalId())) { -// try { -// updateFromJikan(manga); -// return; -// } catch (Exception e) { -// log.warn( -// "Error updating manga data from Jikan for manga {}. Trying AniList... Error: {}", -// manga.getTitle(), -// e.getMessage()); -// } -// } -// -// if (nonNull(manga.getAniListId())) { -// try { -// updateFromAniList(manga); -// return; -// } catch (Exception e) { -// log.warn( -// "Error updating manga data from AniList for manga {}. Error: {}", -// manga.getTitle(), -// e.getMessage()); -// } -// } -// -// log.warn( -// "Could not update manga data for {}. No provider data available/found.", -// manga.getTitle()); -// } -// -// private void updateFromJikan(Manga manga) throws IOException, URISyntaxException { -// jikanRateLimiter.acquire(); -// var mangaData = jikanClient.getMangaById(manga.getMalId()); -// -// manga.setSynopsis(mangaData.data().synopsis()); -// manga.setStatus(mangaData.data().status()); -// manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2)); -// manga.setPublishedFrom(mangaData.data().published().from()); -// manga.setPublishedTo(mangaData.data().published().to()); -// manga.setChapterCount(mangaData.data().chapters()); -// -// var authors = -// mangaData.data().authors().stream() -// .map( -// authorData -> -// authorRepository -// .findByMalId(authorData.mal_id()) -// .orElseGet( -// () -> -// authorRepository.save( -// Author.builder() -// .malId(authorData.mal_id()) -// .name(authorData.name()) -// .build()))) -// .toList(); -// -// updateMangaAuthors(manga, authors); -// -// var genres = -// mangaData.data().genres().stream() -// .map( -// genreData -> -// genreRepository -// .findByMalId(genreData.mal_id()) -// .orElseGet( -// () -> -// genreRepository.save( -// Genre.builder() -// .malId(genreData.mal_id()) -// .name(genreData.name()) -// .build()))) -// .toList(); -// -// updateMangaGenres(manga, genres); -// -// if (isNull(manga.getCoverImage())) { -// downloadCoverImage(manga, mangaData.data().images().jpg().large_image_url()); -// } -// -// var mangaEntity = mangaRepository.save(manga); -// var alternativeTitles = -// mangaData.data().title_synonyms().stream() -// .map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build()) -// .toList(); -// mangaAlternativeTitlesRepository.saveAll(alternativeTitles); -// } -// -// private void updateFromAniList(Manga manga) throws IOException, URISyntaxException { -// var query = -// """ -// query ($id: Int) { -// Media (id: $id, type: MANGA) { -// startDate { year month day } -// endDate { year month day } -// description -// status -// averageScore -// chapters -// coverImage { large } -// genres -// staff { -// edges { -// role -// node { -// name { -// full -// } -// } -// } -// } -// } -// } -// """; -// var request = -// new AniListClient.GraphQLRequest( -// query, new AniListClient.GraphQLRequest.Variables(manga.getAniListId())); -// var media = aniListClient.getManga(request).data().Media(); -// -// manga.setSynopsis(media.description()); -// manga.setStatus(mapAniListStatus(media.status())); -// manga.setScore(DoubleUtil.round((double) media.averageScore() / 10, 2)); // 0-100 -> 0-10 -// manga.setPublishedFrom(convertFuzzyDate(media.startDate())); -// manga.setPublishedTo(convertFuzzyDate(media.endDate())); -// manga.setChapterCount(media.chapters()); -// -// var authors = -// media.staff().edges().stream() -// .filter(edge -> isAuthorRole(edge.role())) -// .map(edge -> edge.node().name().full()) -// .distinct() -// .map( -// name -> -// authorRepository -// .findByName(name) -// .orElseGet( -// () -> authorRepository.save(Author.builder().name(name).build()))) -// .toList(); -// -// updateMangaAuthors(manga, authors); -// -// var genres = -// media.genres().stream() -// .map( -// name -> -// genreRepository -// .findByName(name) -// .orElseGet(() -> -// genreRepository.save(Genre.builder().name(name).build()))) -// .toList(); -// -// updateMangaGenres(manga, genres); -// -// if (isNull(manga.getCoverImage())) { -// downloadCoverImage(manga, media.coverImage().large()); -// } -// -// mangaRepository.save(manga); -// } -// -// private boolean isAuthorRole(String role) { -// return role.equalsIgnoreCase("Story & Art") -// || role.equalsIgnoreCase("Story") -// || role.equalsIgnoreCase("Art"); -// } -// -// private String mapAniListStatus(String status) { -// return switch (status) { -// case "RELEASING" -> "Publishing"; -// case "FINISHED" -> "Finished"; -// case "NOT_YET_RELEASED" -> "Not yet published"; -// default -> "Unknown"; -// }; -// } -// -// private OffsetDateTime convertFuzzyDate(AniListClient.MangaResponse.Manga.FuzzyDate date) { -// if (isNull(date) || isNull(date.year())) { -// return null; -// } -// return OffsetDateTime.of( -// date.year(), -// isNull(date.month()) ? 1 : date.month(), -// isNull(date.day()) ? 1 : date.day(), -// 0, -// 0, -// 0, -// 0, -// ZoneOffset.UTC); -// } -// -// private void updateMangaAuthors(Manga manga, List authors) { -// var mangaAuthors = -// authors.stream() -// .map( -// author -> -// mangaAuthorRepository -// .findByMangaAndAuthor(manga, author) -// .orElseGet( -// () -> -// mangaAuthorRepository.save( -// MangaAuthor.builder().manga(manga).author(author).build()))) -// .toList(); -// manga.setMangaAuthors(mangaAuthors); -// } -// -// private void updateMangaGenres(Manga manga, List genres) { -// var mangaGenres = -// genres.stream() -// .map( -// genre -> -// mangaGenreRepository -// .findByMangaAndGenre(manga, genre) -// .orElseGet( -// () -> -// mangaGenreRepository.save( -// MangaGenre.builder().manga(manga).genre(genre).build()))) -// .toList(); -// manga.setMangaGenres(mangaGenres); -// } -// -// private void downloadCoverImage(Manga manga, String imageUrl) -// throws IOException, URISyntaxException { -// var inputStream = -// new BufferedInputStream(new URL(new URI(imageUrl).toASCIIString()).openStream()); -// -// var bytes = inputStream.readAllBytes(); -// -// inputStream.close(); -// var image = imageService.uploadImage(bytes, "image/jpeg", "cover"); -// -// manga.setCoverImage(image); -// } -// -// public MangaChapter persistMangaChapter( -// MangaContentProvider mangaContentProvider, ContentProviderMangaChapterResponseDTO chapter) { -// var mangaChapter = -// mangaChapterRepository -// .findByMangaContentProviderAndUrlIgnoreCase(mangaContentProvider, -// chapter.url()) -// .orElseGet(MangaChapter::new); -// -// mangaChapter.setMangaContentProvider(mangaContentProvider); -// mangaChapter.setTitle(chapter.title()); -// mangaChapter.setUrl(chapter.url()); -// -// var language = languageService.getOrThrow(chapter.languageCode()); -// mangaChapter.setLanguage(language); -// -// if (nonNull(chapter.chapter())) { -// try { -// mangaChapter.setChapterNumber(Integer.parseInt(chapter.chapter())); -// } catch (NumberFormatException e) { -// log.warn( -// "Could not parse chapter number {} from manga {}", -// chapter.chapter(), -// mangaContentProvider.getManga().getTitle()); -// } -// } -// -// return mangaChapterRepository.save(mangaChapter); -// } -// -// private MangaContentProvider getOrCreateMangaProvider( -// String title, ContentProvider contentProvider) { -// return mangaContentProviderRepository -// .findByMangaTitleIgnoreCaseAndContentProvider(title, contentProvider) -// .orElseGet( -// () -> { -// jikanRateLimiter.acquire(); -// var manga = mangaCreationService.getOrCreateManga(title, "manual", contentProvider); -// -// return mangaContentProviderRepository.save( -// MangaContentProvider.builder() -// .manga(manga) -// .mangaTitle(manga.getTitle()) -// .contentProvider(contentProvider) -// .url("manual") -// .build()); -// }); -// } -// -// private String removeFileExtension(String filename) { -// if (StringUtils.isBlank(filename)) { -// return filename; -// } -// -// int lastDotIndex = filename.lastIndexOf('.'); -// -// // No dot, or dot is the first character (like .gitignore) -// if (lastDotIndex <= 0) { -// return filename; -// } -// -// return filename.substring(0, lastDotIndex); -// } -// } diff --git a/src/main/java/com/magamochi/service/OldMangaService.java b/src/main/java/com/magamochi/service/OldMangaService.java index c3e3934..2bdbfd4 100644 --- a/src/main/java/com/magamochi/service/OldMangaService.java +++ b/src/main/java/com/magamochi/service/OldMangaService.java @@ -5,7 +5,6 @@ import com.magamochi.common.exception.NotFoundException; import com.magamochi.content.model.entity.MangaContent; import com.magamochi.model.dto.*; import com.magamochi.queue.MangaChapterDownloadProducer; -import com.magamochi.user.service.UserService; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -15,8 +14,6 @@ import org.springframework.stereotype.Service; @Service @RequiredArgsConstructor public class OldMangaService { - private final UserService userService; - private final MangaContentProviderRepository mangaContentProviderRepository; private final MangaChapterDownloadProducer mangaChapterDownloadProducer; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 0f0c4e4..e79b09b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -68,6 +68,7 @@ queues: provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:mangamochi.provider.page.ingest} image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch} manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update} + file-import: ${FILE_IMPORT_QUEUE:mangamochi.file.import} routing-key: image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update} diff --git a/src/main/resources/db/migration/V0004__CONTENT_PROVIDERS.sql b/src/main/resources/db/migration/V0004__CONTENT_PROVIDERS.sql new file mode 100644 index 0000000..8d5e48b --- /dev/null +++ b/src/main/resources/db/migration/V0004__CONTENT_PROVIDERS.sql @@ -0,0 +1,6 @@ +ALTER TABLE content_providers + ALTER COLUMN url DROP NOT NULL; + +INSERT INTO content_providers(name, url, active, supports_content_fetch, manual_import) +VALUES ('MangaDex', NULL, TRUE, TRUE, TRUE), + ('Manual Import', NULL, TRUE, FALSE, FALSE); \ No newline at end of file diff --git a/src/main/resources/db/migration/V0005__MANGA_CONTENT.sql b/src/main/resources/db/migration/V0005__MANGA_CONTENT.sql new file mode 100644 index 0000000..52aea08 --- /dev/null +++ b/src/main/resources/db/migration/V0005__MANGA_CONTENT.sql @@ -0,0 +1,2 @@ +ALTER TABLE manga_contents + ALTER COLUMN url DROP NOT NULL; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0006__MANGA_CONTENT.sql b/src/main/resources/db/migration/V0006__MANGA_CONTENT.sql new file mode 100644 index 0000000..8eeaa25 --- /dev/null +++ b/src/main/resources/db/migration/V0006__MANGA_CONTENT.sql @@ -0,0 +1,3 @@ +ALTER TABLE manga_content_provider + ALTER COLUMN url DROP NOT NULL, + ALTER COLUMN manga_title DROP NOT NULL; \ No newline at end of file