mangamochi/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement manga content download functionality with new endpoints and queue integration

Browse Source
This commit is contained in:
Rodrigo Verdiani 2026-03-31 10:25:19 -03:00
parent 53cbde24d9
commit 8a1157b5da
11 changed files with 93 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/main/java/com/magamochi/common/config/RabbitConfig.java
View File

@ -47,6 +47,9 @@ public class RabbitConfig {
@Value("${routing-key.image-update}")
private String imageUpdateRoutingKey;
@Value("${queues.manga-content-download}")
private String mangaContentDownloadQueue;
@Bean
public TopicExchange imageUpdatesExchange() {
return new TopicExchange(imageUpdatesTopic);
@ -192,27 +195,24 @@ public class RabbitConfig {
return QueueBuilder.nonDurable(providerPageIngestQueue + ".dlq").build();
}
// TODO: remove unused queues
@Value("${rabbit-mq.queues.manga-chapter-download}")
private String mangaChapterDownloadQueue;
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
private String mangaFollowUpdateChapterQueue;
@Bean
public Queue mangaChapterDownloadQueue() {
return QueueBuilder.nonDurable(mangaChapterDownloadQueue)
public Queue mangaContentDownloadQueue() {
return QueueBuilder.nonDurable(mangaContentDownloadQueue)
.deadLetterExchange("")
.deadLetterRoutingKey(mangaChapterDownloadQueue + ".dlq")
.deadLetterRoutingKey(mangaContentDownloadQueue + ".dlq")
.build();
}
@Bean
public Queue mangaChapterDownloadDlq() {
return QueueBuilder.nonDurable(mangaChapterDownloadQueue + ".dlq").build();
public Queue mangaContentDownloadDlq() {
return QueueBuilder.nonDurable(mangaContentDownloadQueue + ".dlq").build();
}
// TODO: remove unused queues
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
private String mangaFollowUpdateChapterQueue;
@Bean
public Queue mangaFollowUpdateChapterQueue() {
return QueueBuilder.nonDurable(mangaFollowUpdateChapterQueue)

26
src/main/java/com/magamochi/controller/MangaController.java
View File

@ -1,26 +0,0 @@
package com.magamochi.controller;
import com.magamochi.common.model.dto.DefaultResponseDTO;
import com.magamochi.service.OldMangaService;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/mangas")
@RequiredArgsConstructor
public class MangaController {
private final OldMangaService oldMangaService;
@Operation(
summary = "Fetch all chapters",
description = "Fetch all not yet downloaded chapters from the provider",
tags = {"Manga Chapter"},
operationId = "fetchAllChapters")
@PostMapping(value = "/{mangaProviderId}/fetch-all-chapters")
public DefaultResponseDTO<Void> fetchAllChapters(@PathVariable Long mangaProviderId) {
oldMangaService.fetchAllNotDownloadedChapters(mangaProviderId);
return DefaultResponseDTO.ok().build();
}
}

12
src/main/java/com/magamochi/ingestion/controller/IngestionController.java
View File

@ -66,6 +66,18 @@ public class IngestionController {
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Fetch all content's images",
description = "Fetch all not yet downloaded content's images from the provider",
tags = {"Ingestion"},
operationId = "fetchAllContentImages")
@PostMapping(value = "/manga-content-providers/{mangaContentProviderId}/fetch-all-chapters")
public DefaultResponseDTO<Void> fetchAllContentImages(@PathVariable Long mangaContentProviderId) {
ingestionService.fetchAllContentImages(mangaContentProviderId);
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Fetch content from a content provider",
description = "Fetch the content (images) from the content provider",

3
src/main/java/com/magamochi/ingestion/queue/command/MangaContentDownloadCommand.java Normal file
View File

@ -0,0 +1,3 @@
package com.magamochi.ingestion.queue.command;
public record MangaContentDownloadCommand(Long contentId) {}

21
src/main/java/com/magamochi/ingestion/queue/consumer/MangaContentDownloadConsumer.java Normal file
View File

@ -0,0 +1,21 @@
package com.magamochi.ingestion.queue.consumer;
import com.magamochi.ingestion.queue.command.MangaContentDownloadCommand;
import com.magamochi.ingestion.service.IngestionService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class MangaContentDownloadConsumer {
private final IngestionService ingestionService;
@RabbitListener(queues = "${queues.manga-content-download}")
public void receiveMangaContentDownloadCommand(MangaContentDownloadCommand command) {
log.info("Received manga content download command: {}", command);
ingestionService.fetchContent(command.contentId());
}
}

23
src/main/java/com/magamochi/ingestion/queue/producer/MangaContentDownloadProducer.java Normal file
View File

@ -0,0 +1,23 @@
package com.magamochi.ingestion.queue.producer;
import com.magamochi.ingestion.queue.command.MangaContentDownloadCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class MangaContentDownloadProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${queues.manga-content-download}")
private String mangaContentDownloadQueue;
public void sendMangaContentDownloadCommand(MangaContentDownloadCommand command) {
rabbitTemplate.convertAndSend(mangaContentDownloadQueue, command);
log.info("Sent manga content download command: {}", command);
}
}

24
src/main/java/com/magamochi/ingestion/service/IngestionService.java
View File

@ -4,16 +4,16 @@ import com.magamochi.catalog.service.MangaContentProviderService;
import com.magamochi.common.queue.command.MangaContentImageIngestCommand;
import com.magamochi.common.queue.command.MangaContentIngestCommand;
import com.magamochi.common.queue.command.MangaIngestCommand;
import com.magamochi.content.model.entity.MangaContent;
import com.magamochi.content.service.ContentService;
import com.magamochi.ingestion.model.dto.ProviderMangaMetadataDTO;
import com.magamochi.ingestion.providers.ContentProviderFactory;
import com.magamochi.ingestion.providers.ManualImportContentProviderFactory;
import com.magamochi.ingestion.providers.PagedContentProviderFactory;
import com.magamochi.ingestion.queue.command.MangaContentDownloadCommand;
import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
import com.magamochi.ingestion.queue.producer.MangaContentImageIngestProducer;
import com.magamochi.ingestion.queue.producer.MangaContentIngestProducer;
import com.magamochi.ingestion.queue.producer.MangaIngestProducer;
import com.magamochi.ingestion.queue.producer.ProviderPageIngestProducer;
import com.magamochi.ingestion.queue.producer.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@ -33,6 +33,7 @@ public class IngestionService {
private final MangaIngestProducer mangaIngestProducer;
private final MangaContentIngestProducer mangaContentIngestProducer;
private final MangaContentImageIngestProducer mangaContentImageIngestProducer;
private final MangaContentDownloadProducer mangaContentDownloadProducer;
public void fetchContentProviderMangas(long contentProviderId) {
var contentProvider = contentProviderService.find(contentProviderId);
@ -112,4 +113,19 @@ public class IngestionService {
manualImportContentProviderFactory.getManualImportContentProvider(providerName);
return contentProvider.getMangaMetadata(url);
}
public void fetchAllContentImages(Long mangaContentProviderId) {
var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId);
var contentIds =
mangaContentProvider.getMangaContents().stream()
.filter(mangaContent -> !mangaContent.getDownloaded())
.map(MangaContent::getId)
.collect(Collectors.toSet());
contentIds.forEach(
contentId ->
mangaContentDownloadProducer.sendMangaContentDownloadCommand(
new MangaContentDownloadCommand(contentId)));
}
}

3
src/main/java/com/magamochi/model/dto/MangaChapterDownloadCommand.java
View File

@ -1,3 +0,0 @@
package com.magamochi.model.dto;
public record MangaChapterDownloadCommand(Long chapterId) {}

23
src/main/java/com/magamochi/queue/MangaChapterDownloadProducer.java
View File

@ -1,23 +0,0 @@
package com.magamochi.queue;
import com.magamochi.model.dto.MangaChapterDownloadCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class MangaChapterDownloadProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${rabbit-mq.queues.manga-chapter-download}")
private String mangaChapterDownloadQueue;
public void sendMangaChapterDownloadCommand(MangaChapterDownloadCommand command) {
rabbitTemplate.convertAndSend(mangaChapterDownloadQueue, command);
log.info("Sent manga chapter download command: {}", command);
}
}

29
src/main/java/com/magamochi/service/OldMangaService.java
View File

@ -1,11 +1,5 @@
package com.magamochi.service;
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.content.model.entity.MangaContent;
import com.magamochi.model.dto.*;
import com.magamochi.queue.MangaChapterDownloadProducer;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
@ -14,29 +8,6 @@ import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class OldMangaService {
private final MangaContentProviderRepository mangaContentProviderRepository;
private final MangaChapterDownloadProducer mangaChapterDownloadProducer;
public void fetchAllNotDownloadedChapters(Long mangaProviderId) {
var mangaProvider =
mangaContentProviderRepository
.findById(mangaProviderId)
.orElseThrow(
() -> new NotFoundException("Manga Provider not found for ID: " + mangaProviderId));
var chapterIds =
mangaProvider.getMangaContents().stream()
.filter(mangaChapter -> !mangaChapter.getDownloaded())
.map(MangaContent::getId)
.collect(Collectors.toSet());
chapterIds.forEach(
chapterId ->
mangaChapterDownloadProducer.sendMangaChapterDownloadCommand(
new MangaChapterDownloadCommand(chapterId)));
}
// public void fetchFollowedMangaChapters(Long mangaProviderId) {
// var mangaProvider =
// mangaContentProviderRepository

2
src/main/resources/application.yml
View File

@ -75,13 +75,13 @@ queues:
image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch}
manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update}
file-import: ${FILE_IMPORT_QUEUE:mangamochi.file.import}
manga-content-download: ${MANGA_CONTENT_DOWNLOAD_QUEUE:mangamochi.manga.content.download}
routing-key:
image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update}
rabbit-mq:
queues:
manga-chapter-download: ${MANGA_CHAPTER_DOWNLOAD_QUEUE:mangaChapterDownloadQueue}
manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangaFollowUpdateChapterQueue}
image-service: