diff --git a/src/main/java/com/magamochi/common/config/RabbitConfig.java b/src/main/java/com/magamochi/common/config/RabbitConfig.java index bdbc28c..6b70e2a 100644 --- a/src/main/java/com/magamochi/common/config/RabbitConfig.java +++ b/src/main/java/com/magamochi/common/config/RabbitConfig.java @@ -47,6 +47,9 @@ public class RabbitConfig { @Value("${routing-key.image-update}") private String imageUpdateRoutingKey; + @Value("${queues.manga-content-download}") + private String mangaContentDownloadQueue; + @Bean public TopicExchange imageUpdatesExchange() { return new TopicExchange(imageUpdatesTopic); @@ -192,27 +195,24 @@ public class RabbitConfig { return QueueBuilder.nonDurable(providerPageIngestQueue + ".dlq").build(); } - // TODO: remove unused queues - - @Value("${rabbit-mq.queues.manga-chapter-download}") - private String mangaChapterDownloadQueue; - - @Value("${rabbit-mq.queues.manga-follow-update-chapter}") - private String mangaFollowUpdateChapterQueue; - @Bean - public Queue mangaChapterDownloadQueue() { - return QueueBuilder.nonDurable(mangaChapterDownloadQueue) + public Queue mangaContentDownloadQueue() { + return QueueBuilder.nonDurable(mangaContentDownloadQueue) .deadLetterExchange("") - .deadLetterRoutingKey(mangaChapterDownloadQueue + ".dlq") + .deadLetterRoutingKey(mangaContentDownloadQueue + ".dlq") .build(); } @Bean - public Queue mangaChapterDownloadDlq() { - return QueueBuilder.nonDurable(mangaChapterDownloadQueue + ".dlq").build(); + public Queue mangaContentDownloadDlq() { + return QueueBuilder.nonDurable(mangaContentDownloadQueue + ".dlq").build(); } + // TODO: remove unused queues + + @Value("${rabbit-mq.queues.manga-follow-update-chapter}") + private String mangaFollowUpdateChapterQueue; + @Bean public Queue mangaFollowUpdateChapterQueue() { return QueueBuilder.nonDurable(mangaFollowUpdateChapterQueue) diff --git a/src/main/java/com/magamochi/controller/MangaController.java b/src/main/java/com/magamochi/controller/MangaController.java deleted file mode 100644 index ceab0cf..0000000 --- a/src/main/java/com/magamochi/controller/MangaController.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.magamochi.controller; - -import com.magamochi.common.model.dto.DefaultResponseDTO; -import com.magamochi.service.OldMangaService; -import io.swagger.v3.oas.annotations.Operation; -import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/mangas") -@RequiredArgsConstructor -public class MangaController { - private final OldMangaService oldMangaService; - - @Operation( - summary = "Fetch all chapters", - description = "Fetch all not yet downloaded chapters from the provider", - tags = {"Manga Chapter"}, - operationId = "fetchAllChapters") - @PostMapping(value = "/{mangaProviderId}/fetch-all-chapters") - public DefaultResponseDTO fetchAllChapters(@PathVariable Long mangaProviderId) { - oldMangaService.fetchAllNotDownloadedChapters(mangaProviderId); - - return DefaultResponseDTO.ok().build(); - } -} diff --git a/src/main/java/com/magamochi/ingestion/controller/IngestionController.java b/src/main/java/com/magamochi/ingestion/controller/IngestionController.java index f8304bf..a14ba15 100644 --- a/src/main/java/com/magamochi/ingestion/controller/IngestionController.java +++ b/src/main/java/com/magamochi/ingestion/controller/IngestionController.java @@ -66,6 +66,18 @@ public class IngestionController { return DefaultResponseDTO.ok().build(); } + @Operation( + summary = "Fetch all content's images", + description = "Fetch all not yet downloaded content's images from the provider", + tags = {"Ingestion"}, + operationId = "fetchAllContentImages") + @PostMapping(value = "/manga-content-providers/{mangaContentProviderId}/fetch-all-chapters") + public DefaultResponseDTO fetchAllContentImages(@PathVariable Long mangaContentProviderId) { + ingestionService.fetchAllContentImages(mangaContentProviderId); + + return DefaultResponseDTO.ok().build(); + } + @Operation( summary = "Fetch content from a content provider", description = "Fetch the content (images) from the content provider", diff --git a/src/main/java/com/magamochi/ingestion/queue/command/MangaContentDownloadCommand.java b/src/main/java/com/magamochi/ingestion/queue/command/MangaContentDownloadCommand.java new file mode 100644 index 0000000..f107a32 --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/command/MangaContentDownloadCommand.java @@ -0,0 +1,3 @@ +package com.magamochi.ingestion.queue.command; + +public record MangaContentDownloadCommand(Long contentId) {} diff --git a/src/main/java/com/magamochi/ingestion/queue/consumer/MangaContentDownloadConsumer.java b/src/main/java/com/magamochi/ingestion/queue/consumer/MangaContentDownloadConsumer.java new file mode 100644 index 0000000..47e7115 --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/consumer/MangaContentDownloadConsumer.java @@ -0,0 +1,21 @@ +package com.magamochi.ingestion.queue.consumer; + +import com.magamochi.ingestion.queue.command.MangaContentDownloadCommand; +import com.magamochi.ingestion.service.IngestionService; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class MangaContentDownloadConsumer { + private final IngestionService ingestionService; + + @RabbitListener(queues = "${queues.manga-content-download}") + public void receiveMangaContentDownloadCommand(MangaContentDownloadCommand command) { + log.info("Received manga content download command: {}", command); + ingestionService.fetchContent(command.contentId()); + } +} diff --git a/src/main/java/com/magamochi/ingestion/queue/producer/MangaContentDownloadProducer.java b/src/main/java/com/magamochi/ingestion/queue/producer/MangaContentDownloadProducer.java new file mode 100644 index 0000000..61ecc7e --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/producer/MangaContentDownloadProducer.java @@ -0,0 +1,23 @@ +package com.magamochi.ingestion.queue.producer; + +import com.magamochi.ingestion.queue.command.MangaContentDownloadCommand; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class MangaContentDownloadProducer { + private final RabbitTemplate rabbitTemplate; + + @Value("${queues.manga-content-download}") + private String mangaContentDownloadQueue; + + public void sendMangaContentDownloadCommand(MangaContentDownloadCommand command) { + rabbitTemplate.convertAndSend(mangaContentDownloadQueue, command); + log.info("Sent manga content download command: {}", command); + } +} diff --git a/src/main/java/com/magamochi/ingestion/service/IngestionService.java b/src/main/java/com/magamochi/ingestion/service/IngestionService.java index 5621153..87767f9 100644 --- a/src/main/java/com/magamochi/ingestion/service/IngestionService.java +++ b/src/main/java/com/magamochi/ingestion/service/IngestionService.java @@ -4,16 +4,16 @@ import com.magamochi.catalog.service.MangaContentProviderService; import com.magamochi.common.queue.command.MangaContentImageIngestCommand; import com.magamochi.common.queue.command.MangaContentIngestCommand; import com.magamochi.common.queue.command.MangaIngestCommand; +import com.magamochi.content.model.entity.MangaContent; import com.magamochi.content.service.ContentService; import com.magamochi.ingestion.model.dto.ProviderMangaMetadataDTO; import com.magamochi.ingestion.providers.ContentProviderFactory; import com.magamochi.ingestion.providers.ManualImportContentProviderFactory; import com.magamochi.ingestion.providers.PagedContentProviderFactory; +import com.magamochi.ingestion.queue.command.MangaContentDownloadCommand; import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand; -import com.magamochi.ingestion.queue.producer.MangaContentImageIngestProducer; -import com.magamochi.ingestion.queue.producer.MangaContentIngestProducer; -import com.magamochi.ingestion.queue.producer.MangaIngestProducer; -import com.magamochi.ingestion.queue.producer.ProviderPageIngestProducer; +import com.magamochi.ingestion.queue.producer.*; +import java.util.stream.Collectors; import java.util.stream.IntStream; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -33,6 +33,7 @@ public class IngestionService { private final MangaIngestProducer mangaIngestProducer; private final MangaContentIngestProducer mangaContentIngestProducer; private final MangaContentImageIngestProducer mangaContentImageIngestProducer; + private final MangaContentDownloadProducer mangaContentDownloadProducer; public void fetchContentProviderMangas(long contentProviderId) { var contentProvider = contentProviderService.find(contentProviderId); @@ -112,4 +113,19 @@ public class IngestionService { manualImportContentProviderFactory.getManualImportContentProvider(providerName); return contentProvider.getMangaMetadata(url); } + + public void fetchAllContentImages(Long mangaContentProviderId) { + var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId); + + var contentIds = + mangaContentProvider.getMangaContents().stream() + .filter(mangaContent -> !mangaContent.getDownloaded()) + .map(MangaContent::getId) + .collect(Collectors.toSet()); + + contentIds.forEach( + contentId -> + mangaContentDownloadProducer.sendMangaContentDownloadCommand( + new MangaContentDownloadCommand(contentId))); + } } diff --git a/src/main/java/com/magamochi/model/dto/MangaChapterDownloadCommand.java b/src/main/java/com/magamochi/model/dto/MangaChapterDownloadCommand.java deleted file mode 100644 index 13d8b3a..0000000 --- a/src/main/java/com/magamochi/model/dto/MangaChapterDownloadCommand.java +++ /dev/null @@ -1,3 +0,0 @@ -package com.magamochi.model.dto; - -public record MangaChapterDownloadCommand(Long chapterId) {} diff --git a/src/main/java/com/magamochi/queue/MangaChapterDownloadProducer.java b/src/main/java/com/magamochi/queue/MangaChapterDownloadProducer.java deleted file mode 100644 index 740eaf4..0000000 --- a/src/main/java/com/magamochi/queue/MangaChapterDownloadProducer.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.magamochi.queue; - -import com.magamochi.model.dto.MangaChapterDownloadCommand; -import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -@Log4j2 -@Service -@RequiredArgsConstructor -public class MangaChapterDownloadProducer { - private final RabbitTemplate rabbitTemplate; - - @Value("${rabbit-mq.queues.manga-chapter-download}") - private String mangaChapterDownloadQueue; - - public void sendMangaChapterDownloadCommand(MangaChapterDownloadCommand command) { - rabbitTemplate.convertAndSend(mangaChapterDownloadQueue, command); - log.info("Sent manga chapter download command: {}", command); - } -} diff --git a/src/main/java/com/magamochi/service/OldMangaService.java b/src/main/java/com/magamochi/service/OldMangaService.java index 2bdbfd4..9d0e607 100644 --- a/src/main/java/com/magamochi/service/OldMangaService.java +++ b/src/main/java/com/magamochi/service/OldMangaService.java @@ -1,11 +1,5 @@ package com.magamochi.service; -import com.magamochi.catalog.model.repository.MangaContentProviderRepository; -import com.magamochi.common.exception.NotFoundException; -import com.magamochi.content.model.entity.MangaContent; -import com.magamochi.model.dto.*; -import com.magamochi.queue.MangaChapterDownloadProducer; -import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; @@ -14,29 +8,6 @@ import org.springframework.stereotype.Service; @Service @RequiredArgsConstructor public class OldMangaService { - private final MangaContentProviderRepository mangaContentProviderRepository; - - private final MangaChapterDownloadProducer mangaChapterDownloadProducer; - - public void fetchAllNotDownloadedChapters(Long mangaProviderId) { - var mangaProvider = - mangaContentProviderRepository - .findById(mangaProviderId) - .orElseThrow( - () -> new NotFoundException("Manga Provider not found for ID: " + mangaProviderId)); - - var chapterIds = - mangaProvider.getMangaContents().stream() - .filter(mangaChapter -> !mangaChapter.getDownloaded()) - .map(MangaContent::getId) - .collect(Collectors.toSet()); - - chapterIds.forEach( - chapterId -> - mangaChapterDownloadProducer.sendMangaChapterDownloadCommand( - new MangaChapterDownloadCommand(chapterId))); - } - // public void fetchFollowedMangaChapters(Long mangaProviderId) { // var mangaProvider = // mangaContentProviderRepository diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index bc65589..3b2e3a9 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -75,13 +75,13 @@ queues: image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch} manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update} file-import: ${FILE_IMPORT_QUEUE:mangamochi.file.import} + manga-content-download: ${MANGA_CONTENT_DOWNLOAD_QUEUE:mangamochi.manga.content.download} routing-key: image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update} rabbit-mq: queues: - manga-chapter-download: ${MANGA_CHAPTER_DOWNLOAD_QUEUE:mangaChapterDownloadQueue} manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangaFollowUpdateChapterQueue} image-service: