From 5f66490daee67b668ed557802b8c44ac62781571 Mon Sep 17 00:00:00 2001 From: Rodrigo Verdiani Date: Tue, 17 Mar 2026 20:37:25 -0300 Subject: [PATCH] refactor: content provider ingestion --- .../{ => common}/config/RabbitConfig.java | 20 +++- .../queue/command/MangaIngestCommand.java | 6 ++ .../controller/ManagementController.java | 28 +----- .../{ => ingestion}/client/FlareClient.java | 2 +- .../controller/IngestionController.java | 33 ++++++- .../model/dto/ContentProviderListDTO.java | 2 +- .../ingestion/model/dto/MangaInfoDTO.java | 6 ++ .../providers/ContentProvider.java | 2 +- .../providers/ContentProviderFactory.java | 2 +- .../providers/ContentProviders.java | 7 +- .../ManualImportContentProvider.java | 2 +- .../ManualImportContentProviderFactory.java | 2 +- .../providers/PagedContentProvider.java | 10 ++ .../PagedContentProviderFactory.java | 2 +- .../providers/impl/MangaDexProvider.java | 8 +- .../impl/MangaLivreBlogProvider.java | 18 ++-- .../providers/impl/MangaLivreProvider.java | 19 ++-- .../providers/impl/PinkRosaScanProvider.java | 16 +-- .../command/ProviderPageIngestCommand.java | 3 + .../consumer/ProviderPageIngestConsumer.java | 21 ++++ .../queue/producer/MangaIngestProducer.java | 23 +++++ .../producer/ProviderPageIngestProducer.java | 23 +++++ .../service/ContentProviderService.java | 10 ++ .../ingestion/service/FlareService.java | 2 +- .../service/FlareSessionManager.java | 3 +- .../service}/FlareSessionRegistry.java | 2 +- .../ingestion/service/IngestionService.java | 51 ++++++++++ .../task/FlareSessionCleanupTask.java | 4 +- .../task/FlareStartupCleanupTask.java | 2 +- .../task/IngestFromContentProvidersTask.java | 31 ++++++ .../ContentProviderMangaInfoResponseDTO.java | 7 -- .../magamochi/model/dto/MangaMessageDTO.java | 3 +- .../service/MangaChapterService.java | 2 +- .../magamochi/service/MangaListService.java | 2 +- .../com/magamochi/service/MangaService.java | 2 +- .../ProviderManualMangaImportService.java | 2 +- .../providers/PagedContentProvider.java | 10 -- .../service/providers/impl/BatoProvider.java | 99 ------------------- .../magamochi/task/UpdateMangaDataTask.java | 30 ------ .../magamochi/task/UpdateMangaListTask.java | 67 ------------- src/main/resources/application.yml | 4 + .../db/migration/V0001__CREATE_TABLES.sql | 1 + .../db/migration/V0002__CONTENT_PROVIDERS.sql | 4 + 43 files changed, 295 insertions(+), 298 deletions(-) rename src/main/java/com/magamochi/{ => common}/config/RabbitConfig.java (79%) create mode 100644 src/main/java/com/magamochi/common/queue/command/MangaIngestCommand.java rename src/main/java/com/magamochi/{ => ingestion}/client/FlareClient.java (98%) create mode 100644 src/main/java/com/magamochi/ingestion/model/dto/MangaInfoDTO.java rename src/main/java/com/magamochi/{service => ingestion}/providers/ContentProvider.java (89%) rename src/main/java/com/magamochi/{service => ingestion}/providers/ContentProviderFactory.java (93%) rename src/main/java/com/magamochi/{service => ingestion}/providers/ContentProviders.java (76%) rename src/main/java/com/magamochi/{service => ingestion}/providers/ManualImportContentProvider.java (67%) rename src/main/java/com/magamochi/{service => ingestion}/providers/ManualImportContentProviderFactory.java (93%) create mode 100644 src/main/java/com/magamochi/ingestion/providers/PagedContentProvider.java rename src/main/java/com/magamochi/{service => ingestion}/providers/PagedContentProviderFactory.java (93%) rename src/main/java/com/magamochi/{service => ingestion}/providers/impl/MangaDexProvider.java (95%) rename src/main/java/com/magamochi/{service => ingestion}/providers/impl/MangaLivreBlogProvider.java (92%) rename src/main/java/com/magamochi/{service => ingestion}/providers/impl/MangaLivreProvider.java (89%) rename src/main/java/com/magamochi/{service => ingestion}/providers/impl/PinkRosaScanProvider.java (90%) create mode 100644 src/main/java/com/magamochi/ingestion/queue/command/ProviderPageIngestCommand.java create mode 100644 src/main/java/com/magamochi/ingestion/queue/consumer/ProviderPageIngestConsumer.java create mode 100644 src/main/java/com/magamochi/ingestion/queue/producer/MangaIngestProducer.java create mode 100644 src/main/java/com/magamochi/ingestion/queue/producer/ProviderPageIngestProducer.java rename src/main/java/com/magamochi/{registry => ingestion/service}/FlareSessionRegistry.java (93%) create mode 100644 src/main/java/com/magamochi/ingestion/service/IngestionService.java create mode 100644 src/main/java/com/magamochi/ingestion/task/IngestFromContentProvidersTask.java delete mode 100644 src/main/java/com/magamochi/model/dto/ContentProviderMangaInfoResponseDTO.java delete mode 100644 src/main/java/com/magamochi/service/providers/PagedContentProvider.java delete mode 100644 src/main/java/com/magamochi/service/providers/impl/BatoProvider.java delete mode 100644 src/main/java/com/magamochi/task/UpdateMangaDataTask.java delete mode 100644 src/main/java/com/magamochi/task/UpdateMangaListTask.java create mode 100644 src/main/resources/db/migration/V0002__CONTENT_PROVIDERS.sql diff --git a/src/main/java/com/magamochi/config/RabbitConfig.java b/src/main/java/com/magamochi/common/config/RabbitConfig.java similarity index 79% rename from src/main/java/com/magamochi/config/RabbitConfig.java rename to src/main/java/com/magamochi/common/config/RabbitConfig.java index 9f920f7..f9c991e 100644 --- a/src/main/java/com/magamochi/config/RabbitConfig.java +++ b/src/main/java/com/magamochi/common/config/RabbitConfig.java @@ -1,4 +1,4 @@ -package com.magamochi.config; +package com.magamochi.common.config; import org.springframework.amqp.core.Queue; import org.springframework.amqp.rabbit.connection.ConnectionFactory; @@ -10,6 +10,24 @@ import org.springframework.context.annotation.Configuration; @Configuration public class RabbitConfig { + @Value("${queues.manga-ingest}") + private String mangaIngestQueue; + + @Value("${queues.provider-page-ingest}") + private String providerPageIngestQueue; + + @Bean + public Queue mangaIngestQueue() { + return new Queue(mangaIngestQueue, false); + } + + @Bean + public Queue providerPageIngestQueue() { + return new Queue(providerPageIngestQueue, false); + } + + // TODO: remove unused queues + @Value("${rabbit-mq.queues.manga-data-update}") private String mangaDataUpdateQueue; diff --git a/src/main/java/com/magamochi/common/queue/command/MangaIngestCommand.java b/src/main/java/com/magamochi/common/queue/command/MangaIngestCommand.java new file mode 100644 index 0000000..e29223c --- /dev/null +++ b/src/main/java/com/magamochi/common/queue/command/MangaIngestCommand.java @@ -0,0 +1,6 @@ +package com.magamochi.common.queue.command; + +import jakarta.validation.constraints.NotBlank; + +public record MangaIngestCommand( + long providerId, @NotBlank String mangaTitle, @NotBlank String url) {} diff --git a/src/main/java/com/magamochi/controller/ManagementController.java b/src/main/java/com/magamochi/controller/ManagementController.java index c35ed20..4d8b0c6 100644 --- a/src/main/java/com/magamochi/controller/ManagementController.java +++ b/src/main/java/com/magamochi/controller/ManagementController.java @@ -2,12 +2,12 @@ package com.magamochi.controller; import com.magamochi.client.NtfyClient; import com.magamochi.common.dto.DefaultResponseDTO; +import com.magamochi.ingestion.task.IngestFromContentProvidersTask; import com.magamochi.model.dto.UpdateMangaDataCommand; import com.magamochi.model.repository.UserRepository; import com.magamochi.queue.UpdateMangaDataProducer; import com.magamochi.task.ImageCleanupTask; import com.magamochi.task.MangaFollowUpdateTask; -import com.magamochi.task.UpdateMangaListTask; import io.swagger.v3.oas.annotations.Operation; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.*; @@ -16,7 +16,7 @@ import org.springframework.web.bind.annotation.*; @RequestMapping("/management") @RequiredArgsConstructor public class ManagementController { - private final UpdateMangaListTask updateMangaListTask; + private final IngestFromContentProvidersTask ingestFromContentProvidersTask; private final ImageCleanupTask imageCleanupTask; private final MangaFollowUpdateTask mangaFollowUpdateTask; private final UserRepository userRepository; @@ -35,30 +35,6 @@ public class ManagementController { return DefaultResponseDTO.ok().build(); } - @Operation( - summary = "Queue update manga list", - description = "Queue the retrieval of the manga lists from the content providers", - tags = {"Management"}, - operationId = "updateMangaList") - @PostMapping("update-manga-list") - public DefaultResponseDTO updateMangaList() { - updateMangaListTask.updateMangaList(); - - return DefaultResponseDTO.ok().build(); - } - - @Operation( - summary = "Queue update provider manga list", - description = "Queue the retrieval of the manga list for a specific provider", - tags = {"Management"}, - operationId = "updateProviderMangaList") - @PostMapping("update-provider-manga-list") - public DefaultResponseDTO updateProviderMangaList(@RequestParam Long providerId) { - updateMangaListTask.updateProviderMangaList(providerId); - - return DefaultResponseDTO.ok().build(); - } - @Operation( summary = "Cleanup unused S3 images", description = "Triggers the cleanup of untracked S3 images", diff --git a/src/main/java/com/magamochi/client/FlareClient.java b/src/main/java/com/magamochi/ingestion/client/FlareClient.java similarity index 98% rename from src/main/java/com/magamochi/client/FlareClient.java rename to src/main/java/com/magamochi/ingestion/client/FlareClient.java index 6716757..42e52de 100644 --- a/src/main/java/com/magamochi/client/FlareClient.java +++ b/src/main/java/com/magamochi/ingestion/client/FlareClient.java @@ -1,4 +1,4 @@ -package com.magamochi.client; +package com.magamochi.ingestion.client; import io.github.resilience4j.retry.annotation.Retry; import java.util.List; diff --git a/src/main/java/com/magamochi/ingestion/controller/IngestionController.java b/src/main/java/com/magamochi/ingestion/controller/IngestionController.java index 581b76d..c4539e4 100644 --- a/src/main/java/com/magamochi/ingestion/controller/IngestionController.java +++ b/src/main/java/com/magamochi/ingestion/controller/IngestionController.java @@ -3,18 +3,17 @@ package com.magamochi.ingestion.controller; import com.magamochi.common.dto.DefaultResponseDTO; import com.magamochi.ingestion.model.dto.ContentProviderListDTO; import com.magamochi.ingestion.service.ContentProviderService; +import com.magamochi.ingestion.service.IngestionService; import io.swagger.v3.oas.annotations.Operation; import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; @RestController @RequestMapping("/ingestion") @RequiredArgsConstructor public class IngestionController { private final ContentProviderService contentProviderService; + private final IngestionService ingestionService; @Operation( summary = "Get a list of content providers", @@ -26,4 +25,30 @@ public class IngestionController { @RequestParam(name = "manualImport", required = false) Boolean manualImport) { return DefaultResponseDTO.ok(contentProviderService.getProviders(manualImport)); } + + @Operation( + summary = "Fetch mangas from a content provider", + description = + "Triggers the ingestion process for a specific content provider, fetching manga data and queuing it for processing.", + tags = {"Ingestion"}, + operationId = "fetchContentProviderMangas") + @PostMapping("/providers/{providerId}/fetch") + public DefaultResponseDTO fetchContentProviderMangas(@PathVariable Long providerId) { + ingestionService.fetchContentProviderMangas(providerId); + + return DefaultResponseDTO.ok().build(); + } + + @Operation( + summary = "Fetch mangas from all content providers", + description = + "Triggers the ingestion process for all content providers, fetching manga data and queuing them for processing.", + tags = {"Ingestion"}, + operationId = "fetchAllContentProviderMangas") + @PostMapping("/providers/fetch") + public DefaultResponseDTO fetchAllContentProviderMangas() { + ingestionService.fetchAllContentProviderMangas(); + + return DefaultResponseDTO.ok().build(); + } } diff --git a/src/main/java/com/magamochi/ingestion/model/dto/ContentProviderListDTO.java b/src/main/java/com/magamochi/ingestion/model/dto/ContentProviderListDTO.java index 1aa266b..b78d549 100644 --- a/src/main/java/com/magamochi/ingestion/model/dto/ContentProviderListDTO.java +++ b/src/main/java/com/magamochi/ingestion/model/dto/ContentProviderListDTO.java @@ -11,7 +11,7 @@ public record ContentProviderListDTO(@NotNull List providers contentProviders.stream().map(ContentProviderDTO::from).toList()); } - record ContentProviderDTO(long id, @NotBlank String name) { + public record ContentProviderDTO(long id, @NotBlank String name) { public static ContentProviderDTO from(ContentProvider contentProvider) { return new ContentProviderDTO(contentProvider.getId(), contentProvider.getName()); } diff --git a/src/main/java/com/magamochi/ingestion/model/dto/MangaInfoDTO.java b/src/main/java/com/magamochi/ingestion/model/dto/MangaInfoDTO.java new file mode 100644 index 0000000..192fb3e --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/model/dto/MangaInfoDTO.java @@ -0,0 +1,6 @@ +package com.magamochi.ingestion.model.dto; + +import com.magamochi.model.enumeration.MangaStatus; +import jakarta.validation.constraints.NotBlank; + +public record MangaInfoDTO(@NotBlank String title, @NotBlank String url, MangaStatus status) {} diff --git a/src/main/java/com/magamochi/service/providers/ContentProvider.java b/src/main/java/com/magamochi/ingestion/providers/ContentProvider.java similarity index 89% rename from src/main/java/com/magamochi/service/providers/ContentProvider.java rename to src/main/java/com/magamochi/ingestion/providers/ContentProvider.java index fb43df0..17eb162 100644 --- a/src/main/java/com/magamochi/service/providers/ContentProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/ContentProvider.java @@ -1,4 +1,4 @@ -package com.magamochi.service.providers; +package com.magamochi.ingestion.providers; import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; import com.magamochi.model.entity.MangaContentProvider; diff --git a/src/main/java/com/magamochi/service/providers/ContentProviderFactory.java b/src/main/java/com/magamochi/ingestion/providers/ContentProviderFactory.java similarity index 93% rename from src/main/java/com/magamochi/service/providers/ContentProviderFactory.java rename to src/main/java/com/magamochi/ingestion/providers/ContentProviderFactory.java index 25dc06f..71fd653 100644 --- a/src/main/java/com/magamochi/service/providers/ContentProviderFactory.java +++ b/src/main/java/com/magamochi/ingestion/providers/ContentProviderFactory.java @@ -1,4 +1,4 @@ -package com.magamochi.service.providers; +package com.magamochi.ingestion.providers; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/com/magamochi/service/providers/ContentProviders.java b/src/main/java/com/magamochi/ingestion/providers/ContentProviders.java similarity index 76% rename from src/main/java/com/magamochi/service/providers/ContentProviders.java rename to src/main/java/com/magamochi/ingestion/providers/ContentProviders.java index d68908b..ad860e6 100644 --- a/src/main/java/com/magamochi/service/providers/ContentProviders.java +++ b/src/main/java/com/magamochi/ingestion/providers/ContentProviders.java @@ -1,9 +1,8 @@ -package com.magamochi.service.providers; +package com.magamochi.ingestion.providers; public class ContentProviders { - public static final String MANGA_LIVRE_TO = "Manga Livre.to"; public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog"; - public static final String MANGA_DEX = "MangaDex"; + public static final String MANGA_LIVRE_TO = "Manga Livre.to"; public static final String PINK_ROSA_SCAN = "Pink Rosa Scan"; - public static final String BATO = "Bato"; + public static final String MANGA_DEX = "MangaDex"; } diff --git a/src/main/java/com/magamochi/service/providers/ManualImportContentProvider.java b/src/main/java/com/magamochi/ingestion/providers/ManualImportContentProvider.java similarity index 67% rename from src/main/java/com/magamochi/service/providers/ManualImportContentProvider.java rename to src/main/java/com/magamochi/ingestion/providers/ManualImportContentProvider.java index 807f98c..1a36387 100644 --- a/src/main/java/com/magamochi/service/providers/ManualImportContentProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/ManualImportContentProvider.java @@ -1,4 +1,4 @@ -package com.magamochi.service.providers; +package com.magamochi.ingestion.providers; public interface ManualImportContentProvider { String getMangaTitle(String value); diff --git a/src/main/java/com/magamochi/service/providers/ManualImportContentProviderFactory.java b/src/main/java/com/magamochi/ingestion/providers/ManualImportContentProviderFactory.java similarity index 93% rename from src/main/java/com/magamochi/service/providers/ManualImportContentProviderFactory.java rename to src/main/java/com/magamochi/ingestion/providers/ManualImportContentProviderFactory.java index a60f858..009c19a 100644 --- a/src/main/java/com/magamochi/service/providers/ManualImportContentProviderFactory.java +++ b/src/main/java/com/magamochi/ingestion/providers/ManualImportContentProviderFactory.java @@ -1,4 +1,4 @@ -package com.magamochi.service.providers; +package com.magamochi.ingestion.providers; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/com/magamochi/ingestion/providers/PagedContentProvider.java b/src/main/java/com/magamochi/ingestion/providers/PagedContentProvider.java new file mode 100644 index 0000000..593724c --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/providers/PagedContentProvider.java @@ -0,0 +1,10 @@ +package com.magamochi.ingestion.providers; + +import com.magamochi.ingestion.model.dto.MangaInfoDTO; +import java.util.List; + +public interface PagedContentProvider { + int getTotalPages(); + + List getMangasFromPage(int page); +} diff --git a/src/main/java/com/magamochi/service/providers/PagedContentProviderFactory.java b/src/main/java/com/magamochi/ingestion/providers/PagedContentProviderFactory.java similarity index 93% rename from src/main/java/com/magamochi/service/providers/PagedContentProviderFactory.java rename to src/main/java/com/magamochi/ingestion/providers/PagedContentProviderFactory.java index 4909c6e..bdddc6f 100644 --- a/src/main/java/com/magamochi/service/providers/PagedContentProviderFactory.java +++ b/src/main/java/com/magamochi/ingestion/providers/PagedContentProviderFactory.java @@ -1,4 +1,4 @@ -package com.magamochi.service.providers; +package com.magamochi.ingestion.providers; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/com/magamochi/service/providers/impl/MangaDexProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java similarity index 95% rename from src/main/java/com/magamochi/service/providers/impl/MangaDexProvider.java rename to src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java index 42ba712..5a46e81 100644 --- a/src/main/java/com/magamochi/service/providers/impl/MangaDexProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java @@ -1,15 +1,15 @@ -package com.magamochi.service.providers.impl; +package com.magamochi.ingestion.providers.impl; import static java.util.Objects.isNull; import com.google.common.util.concurrent.RateLimiter; import com.magamochi.client.MangaDexClient; import com.magamochi.common.exception.UnprocessableException; +import com.magamochi.ingestion.providers.ContentProvider; +import com.magamochi.ingestion.providers.ContentProviders; +import com.magamochi.ingestion.providers.ManualImportContentProvider; import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; import com.magamochi.model.entity.MangaContentProvider; -import com.magamochi.service.providers.ContentProvider; -import com.magamochi.service.providers.ContentProviders; -import com.magamochi.service.providers.ManualImportContentProvider; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; diff --git a/src/main/java/com/magamochi/service/providers/impl/MangaLivreBlogProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java similarity index 92% rename from src/main/java/com/magamochi/service/providers/impl/MangaLivreBlogProvider.java rename to src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java index abc9ae4..f97bdd3 100644 --- a/src/main/java/com/magamochi/service/providers/impl/MangaLivreBlogProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java @@ -1,12 +1,12 @@ -package com.magamochi.service.providers.impl; +package com.magamochi.ingestion.providers.impl; +import com.magamochi.ingestion.model.dto.MangaInfoDTO; +import com.magamochi.ingestion.providers.ContentProvider; +import com.magamochi.ingestion.providers.ContentProviders; +import com.magamochi.ingestion.providers.PagedContentProvider; import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; -import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.model.entity.MangaContentProvider; import com.magamochi.model.enumeration.MangaStatus; -import com.magamochi.service.providers.ContentProvider; -import com.magamochi.service.providers.ContentProviders; -import com.magamochi.service.providers.PagedContentProvider; import java.io.IOException; import java.util.*; import java.util.regex.Pattern; @@ -101,7 +101,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv } @Override - public List getMangasFromPage(Integer page) { + public List getMangasFromPage(int page) { log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_BLOG, page); try { @@ -134,7 +134,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv default -> MangaStatus.UNKNOWN; }; - return new ContentProviderMangaInfoResponseDTO(title, url, status); + return new MangaInfoDTO(title, url, status); } catch (Exception e) { return null; } @@ -148,7 +148,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv } @Override - public Integer getTotalPages() { + public int getTotalPages() { log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_BLOG); try { @@ -166,7 +166,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv return pageNumbers.stream().max(Integer::compareTo).orElse(null); } catch (IOException | NoSuchElementException e) { log.error("Error fetching total pages from MangaLivre", e); - return null; + return 0; } } } diff --git a/src/main/java/com/magamochi/service/providers/impl/MangaLivreProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java similarity index 89% rename from src/main/java/com/magamochi/service/providers/impl/MangaLivreProvider.java rename to src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java index b7d766a..e3dbd57 100644 --- a/src/main/java/com/magamochi/service/providers/impl/MangaLivreProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java @@ -1,15 +1,15 @@ -package com.magamochi.service.providers.impl; +package com.magamochi.ingestion.providers.impl; import static java.util.Objects.nonNull; +import com.magamochi.ingestion.model.dto.MangaInfoDTO; +import com.magamochi.ingestion.providers.ContentProvider; +import com.magamochi.ingestion.providers.ContentProviders; +import com.magamochi.ingestion.providers.PagedContentProvider; import com.magamochi.ingestion.service.FlareService; import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; -import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.model.entity.MangaContentProvider; import com.magamochi.model.enumeration.MangaStatus; -import com.magamochi.service.providers.ContentProvider; -import com.magamochi.service.providers.ContentProviders; -import com.magamochi.service.providers.PagedContentProvider; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -90,7 +90,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider } @Override - public List getMangasFromPage(Integer page) { + public List getMangasFromPage(int page) { log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_TO, page); try { @@ -116,8 +116,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider } } - return new ContentProviderMangaInfoResponseDTO( - title.trim(), url.trim(), MangaStatus.UNKNOWN); + return new MangaInfoDTO(title.trim(), url.trim(), MangaStatus.UNKNOWN); }) .toList(); } catch (NoSuchElementException e) { @@ -127,7 +126,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider } @Override - public Integer getTotalPages() { + public int getTotalPages() { log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_TO); try { @@ -140,7 +139,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider return (int) Math.ceil((double) totalMangas / MANGAS_PER_PAGE); } catch (NoSuchElementException e) { log.error("Error parsing total pages from MangaLivre", e); - return null; + return 0; } } } diff --git a/src/main/java/com/magamochi/service/providers/impl/PinkRosaScanProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java similarity index 90% rename from src/main/java/com/magamochi/service/providers/impl/PinkRosaScanProvider.java rename to src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java index 1cd6a89..c9e7af1 100644 --- a/src/main/java/com/magamochi/service/providers/impl/PinkRosaScanProvider.java +++ b/src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java @@ -1,16 +1,16 @@ -package com.magamochi.service.providers.impl; +package com.magamochi.ingestion.providers.impl; import static java.util.Objects.isNull; import static org.apache.commons.lang3.StringUtils.isBlank; +import com.magamochi.ingestion.model.dto.MangaInfoDTO; +import com.magamochi.ingestion.providers.ContentProvider; +import com.magamochi.ingestion.providers.ContentProviders; +import com.magamochi.ingestion.providers.PagedContentProvider; import com.magamochi.ingestion.service.FlareService; import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; -import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.model.entity.MangaContentProvider; import com.magamochi.model.enumeration.MangaStatus; -import com.magamochi.service.providers.ContentProvider; -import com.magamochi.service.providers.ContentProviders; -import com.magamochi.service.providers.PagedContentProvider; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -110,12 +110,12 @@ public class PinkRosaScanProvider implements ContentProvider, PagedContentProvid } @Override - public Integer getTotalPages() { + public int getTotalPages() { return 1; } @Override - public List getMangasFromPage(Integer page) { + public List getMangasFromPage(int page) { log.info("Getting mangas from {}", ContentProviders.PINK_ROSA_SCAN); try { @@ -142,7 +142,7 @@ public class PinkRosaScanProvider implements ContentProvider, PagedContentProvid var textElement = linkElement.getElementsByTag("h3"); var title = textElement.text().trim(); - return new ContentProviderMangaInfoResponseDTO(title, url, MangaStatus.UNKNOWN); + return new MangaInfoDTO(title, url, MangaStatus.UNKNOWN); }) .toList(); } catch (NoSuchElementException e) { diff --git a/src/main/java/com/magamochi/ingestion/queue/command/ProviderPageIngestCommand.java b/src/main/java/com/magamochi/ingestion/queue/command/ProviderPageIngestCommand.java new file mode 100644 index 0000000..8f6a395 --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/command/ProviderPageIngestCommand.java @@ -0,0 +1,3 @@ +package com.magamochi.ingestion.queue.command; + +public record ProviderPageIngestCommand(long providerId, int page) {} diff --git a/src/main/java/com/magamochi/ingestion/queue/consumer/ProviderPageIngestConsumer.java b/src/main/java/com/magamochi/ingestion/queue/consumer/ProviderPageIngestConsumer.java new file mode 100644 index 0000000..1fef3cb --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/consumer/ProviderPageIngestConsumer.java @@ -0,0 +1,21 @@ +package com.magamochi.ingestion.queue.consumer; + +import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand; +import com.magamochi.ingestion.service.IngestionService; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class ProviderPageIngestConsumer { + private final IngestionService ingestionService; + + @RabbitListener(queues = "${queues.provider-page-ingest}") + public void receiveProviderPageIngestCommand(ProviderPageIngestCommand command) { + log.info("Received provider page ingest command: {}", command); + ingestionService.fetchProviderPageMangas(command.providerId(), command.page()); + } +} diff --git a/src/main/java/com/magamochi/ingestion/queue/producer/MangaIngestProducer.java b/src/main/java/com/magamochi/ingestion/queue/producer/MangaIngestProducer.java new file mode 100644 index 0000000..b67f334 --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/producer/MangaIngestProducer.java @@ -0,0 +1,23 @@ +package com.magamochi.ingestion.queue.producer; + +import com.magamochi.common.queue.command.MangaIngestCommand; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class MangaIngestProducer { + private final RabbitTemplate rabbitTemplate; + + @Value("${queues.manga-ingest}") + private String mangaIngestQueue; + + public void sendMangaIngestCommand(MangaIngestCommand command) { + rabbitTemplate.convertAndSend(mangaIngestQueue, command); + log.info("Sent manga ingest command: {}", command); + } +} diff --git a/src/main/java/com/magamochi/ingestion/queue/producer/ProviderPageIngestProducer.java b/src/main/java/com/magamochi/ingestion/queue/producer/ProviderPageIngestProducer.java new file mode 100644 index 0000000..2bacfca --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/queue/producer/ProviderPageIngestProducer.java @@ -0,0 +1,23 @@ +package com.magamochi.ingestion.queue.producer; + +import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class ProviderPageIngestProducer { + private final RabbitTemplate rabbitTemplate; + + @Value("${queues.provider-page-ingest}") + private String providerPageIngestQueue; + + public void sendProviderPageIngestCommand(ProviderPageIngestCommand command) { + rabbitTemplate.convertAndSend(providerPageIngestQueue, command); + log.info("Sent provider page ingest command: {}", command); + } +} diff --git a/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java b/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java index fb7cbea..026a31c 100644 --- a/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java +++ b/src/main/java/com/magamochi/ingestion/service/ContentProviderService.java @@ -2,6 +2,7 @@ package com.magamochi.ingestion.service; import static java.util.Objects.nonNull; +import com.magamochi.common.exception.NotFoundException; import com.magamochi.ingestion.model.dto.ContentProviderListDTO; import com.magamochi.ingestion.model.entity.ContentProvider; import com.magamochi.ingestion.model.repository.ContentProviderRepository; @@ -22,4 +23,13 @@ public class ContentProviderService { return ContentProviderListDTO.from(providers); } + + public ContentProvider find(Long contentProviderId) { + return contentProviderRepository + .findById(contentProviderId) + .orElseThrow( + () -> + new NotFoundException( + "Content Provider not found (ID: " + contentProviderId + ").")); + } } diff --git a/src/main/java/com/magamochi/ingestion/service/FlareService.java b/src/main/java/com/magamochi/ingestion/service/FlareService.java index b327e92..f0ab713 100644 --- a/src/main/java/com/magamochi/ingestion/service/FlareService.java +++ b/src/main/java/com/magamochi/ingestion/service/FlareService.java @@ -1,6 +1,6 @@ package com.magamochi.ingestion.service; -import com.magamochi.client.FlareClient; +import com.magamochi.ingestion.client.FlareClient; import lombok.RequiredArgsConstructor; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; diff --git a/src/main/java/com/magamochi/ingestion/service/FlareSessionManager.java b/src/main/java/com/magamochi/ingestion/service/FlareSessionManager.java index 81c7a11..251f01a 100644 --- a/src/main/java/com/magamochi/ingestion/service/FlareSessionManager.java +++ b/src/main/java/com/magamochi/ingestion/service/FlareSessionManager.java @@ -3,9 +3,8 @@ package com.magamochi.ingestion.service; import static java.util.Objects.isNull; import static java.util.Objects.nonNull; -import com.magamochi.client.FlareClient; +import com.magamochi.ingestion.client.FlareClient; import com.magamochi.ingestion.model.entity.FlareSession; -import com.magamochi.registry.FlareSessionRegistry; import java.time.Duration; import java.time.Instant; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/magamochi/registry/FlareSessionRegistry.java b/src/main/java/com/magamochi/ingestion/service/FlareSessionRegistry.java similarity index 93% rename from src/main/java/com/magamochi/registry/FlareSessionRegistry.java rename to src/main/java/com/magamochi/ingestion/service/FlareSessionRegistry.java index dbf788c..4242d99 100644 --- a/src/main/java/com/magamochi/registry/FlareSessionRegistry.java +++ b/src/main/java/com/magamochi/ingestion/service/FlareSessionRegistry.java @@ -1,4 +1,4 @@ -package com.magamochi.registry; +package com.magamochi.ingestion.service; import com.magamochi.ingestion.model.entity.FlareSession; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/main/java/com/magamochi/ingestion/service/IngestionService.java b/src/main/java/com/magamochi/ingestion/service/IngestionService.java new file mode 100644 index 0000000..fed47cf --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/service/IngestionService.java @@ -0,0 +1,51 @@ +package com.magamochi.ingestion.service; + +import com.magamochi.common.queue.command.MangaIngestCommand; +import com.magamochi.ingestion.providers.PagedContentProviderFactory; +import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand; +import com.magamochi.ingestion.queue.producer.MangaIngestProducer; +import com.magamochi.ingestion.queue.producer.ProviderPageIngestProducer; +import java.util.stream.IntStream; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class IngestionService { + private final ContentProviderService contentProviderService; + private final PagedContentProviderFactory pagedContentProviderFactory; + + private final ProviderPageIngestProducer providerPageIngestProducer; + private final MangaIngestProducer mangaIngestProducer; + + public void fetchContentProviderMangas(long contentProviderId) { + var contentProvider = contentProviderService.find(contentProviderId); + var pagedContentProvider = + pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName()); + var pages = pagedContentProvider.getTotalPages(); + + IntStream.rangeClosed(1, pages) + .forEach( + page -> + providerPageIngestProducer.sendProviderPageIngestCommand( + new ProviderPageIngestCommand(contentProvider.getId(), page))); + } + + public void fetchProviderPageMangas(long providerId, int page) { + var contentProvider = contentProviderService.find(providerId); + var pagedContentProvider = + pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName()); + + var mangas = pagedContentProvider.getMangasFromPage(page); + + mangas.forEach( + manga -> + mangaIngestProducer.sendMangaIngestCommand( + new MangaIngestCommand(contentProvider.getId(), manga.title(), manga.url()))); + } + + public void fetchAllContentProviderMangas() { + var contentProviders = contentProviderService.getProviders(null); + contentProviders.providers().forEach(dto -> fetchContentProviderMangas(dto.id())); + } +} diff --git a/src/main/java/com/magamochi/ingestion/task/FlareSessionCleanupTask.java b/src/main/java/com/magamochi/ingestion/task/FlareSessionCleanupTask.java index 6e04d46..26610c5 100644 --- a/src/main/java/com/magamochi/ingestion/task/FlareSessionCleanupTask.java +++ b/src/main/java/com/magamochi/ingestion/task/FlareSessionCleanupTask.java @@ -1,7 +1,7 @@ package com.magamochi.ingestion.task; -import com.magamochi.client.FlareClient; -import com.magamochi.registry.FlareSessionRegistry; +import com.magamochi.ingestion.client.FlareClient; +import com.magamochi.ingestion.service.FlareSessionRegistry; import java.time.Duration; import java.time.Instant; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/magamochi/ingestion/task/FlareStartupCleanupTask.java b/src/main/java/com/magamochi/ingestion/task/FlareStartupCleanupTask.java index d426c63..6e0ba9b 100644 --- a/src/main/java/com/magamochi/ingestion/task/FlareStartupCleanupTask.java +++ b/src/main/java/com/magamochi/ingestion/task/FlareStartupCleanupTask.java @@ -1,6 +1,6 @@ package com.magamochi.ingestion.task; -import com.magamochi.client.FlareClient; +import com.magamochi.ingestion.client.FlareClient; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.boot.context.event.ApplicationReadyEvent; diff --git a/src/main/java/com/magamochi/ingestion/task/IngestFromContentProvidersTask.java b/src/main/java/com/magamochi/ingestion/task/IngestFromContentProvidersTask.java new file mode 100644 index 0000000..aa7d823 --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/task/IngestFromContentProvidersTask.java @@ -0,0 +1,31 @@ +package com.magamochi.ingestion.task; + +import com.magamochi.ingestion.service.IngestionService; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Log4j2 +@Component +@RequiredArgsConstructor +public class IngestFromContentProvidersTask { + @Value("${content-providers.update-enabled}") + private Boolean updateEnabled; + + private final IngestionService ingestionService; + + @Scheduled(cron = "${content-providers.cron-expression}") + public void updateMangaListScheduled() { + if (!updateEnabled) { + return; + } + + log.info("Starting scheduling ingest from Content Providers."); + + ingestionService.fetchAllContentProviderMangas(); + + log.info("Finished scheduling ingest from Content Providers."); + } +} diff --git a/src/main/java/com/magamochi/model/dto/ContentProviderMangaInfoResponseDTO.java b/src/main/java/com/magamochi/model/dto/ContentProviderMangaInfoResponseDTO.java deleted file mode 100644 index 82ced22..0000000 --- a/src/main/java/com/magamochi/model/dto/ContentProviderMangaInfoResponseDTO.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.magamochi.model.dto; - -import com.magamochi.model.enumeration.MangaStatus; -import jakarta.validation.constraints.NotBlank; - -public record ContentProviderMangaInfoResponseDTO( - @NotBlank String title, @NotBlank String url, MangaStatus status) {} diff --git a/src/main/java/com/magamochi/model/dto/MangaMessageDTO.java b/src/main/java/com/magamochi/model/dto/MangaMessageDTO.java index d529faa..acda1c3 100644 --- a/src/main/java/com/magamochi/model/dto/MangaMessageDTO.java +++ b/src/main/java/com/magamochi/model/dto/MangaMessageDTO.java @@ -1,6 +1,7 @@ package com.magamochi.model.dto; +import com.magamochi.ingestion.model.dto.MangaInfoDTO; import java.util.List; public record MangaMessageDTO( - String contentProviderName, List mangaInfoResponseDTOs) {} + String contentProviderName, List mangaInfoResponseDTOs) {} diff --git a/src/main/java/com/magamochi/service/MangaChapterService.java b/src/main/java/com/magamochi/service/MangaChapterService.java index 915843a..099973b 100644 --- a/src/main/java/com/magamochi/service/MangaChapterService.java +++ b/src/main/java/com/magamochi/service/MangaChapterService.java @@ -2,6 +2,7 @@ package com.magamochi.service; import com.google.common.util.concurrent.RateLimiter; import com.magamochi.common.exception.UnprocessableException; +import com.magamochi.ingestion.providers.ContentProviderFactory; import com.magamochi.model.dto.MangaChapterArchiveDTO; import com.magamochi.model.dto.MangaChapterImagesDTO; import com.magamochi.model.entity.MangaChapter; @@ -9,7 +10,6 @@ import com.magamochi.model.entity.MangaChapterImage; import com.magamochi.model.enumeration.ArchiveFileType; import com.magamochi.model.repository.MangaChapterImageRepository; import com.magamochi.model.repository.MangaChapterRepository; -import com.magamochi.service.providers.ContentProviderFactory; import io.github.resilience4j.retry.Retry; import io.github.resilience4j.retry.RetryRegistry; import java.io.BufferedInputStream; diff --git a/src/main/java/com/magamochi/service/MangaListService.java b/src/main/java/com/magamochi/service/MangaListService.java index 21e1e08..faf5bc8 100644 --- a/src/main/java/com/magamochi/service/MangaListService.java +++ b/src/main/java/com/magamochi/service/MangaListService.java @@ -2,9 +2,9 @@ package com.magamochi.service; import static java.util.Objects.isNull; +import com.magamochi.ingestion.providers.PagedContentProviderFactory; import com.magamochi.model.entity.MangaContentProvider; import com.magamochi.model.repository.MangaContentProviderRepository; -import com.magamochi.service.providers.PagedContentProviderFactory; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; diff --git a/src/main/java/com/magamochi/service/MangaService.java b/src/main/java/com/magamochi/service/MangaService.java index db0b8f8..8d36731 100644 --- a/src/main/java/com/magamochi/service/MangaService.java +++ b/src/main/java/com/magamochi/service/MangaService.java @@ -4,6 +4,7 @@ import static java.util.Objects.nonNull; import com.magamochi.client.NtfyClient; import com.magamochi.common.exception.NotFoundException; +import com.magamochi.ingestion.providers.ContentProviderFactory; import com.magamochi.model.dto.*; import com.magamochi.model.entity.Manga; import com.magamochi.model.entity.MangaChapter; @@ -12,7 +13,6 @@ import com.magamochi.model.entity.UserMangaFollow; import com.magamochi.model.repository.*; import com.magamochi.model.specification.MangaSpecification; import com.magamochi.queue.MangaChapterDownloadProducer; -import com.magamochi.service.providers.ContentProviderFactory; import java.util.Comparator; import java.util.List; import java.util.Set; diff --git a/src/main/java/com/magamochi/service/ProviderManualMangaImportService.java b/src/main/java/com/magamochi/service/ProviderManualMangaImportService.java index 1b88b71..b23e908 100644 --- a/src/main/java/com/magamochi/service/ProviderManualMangaImportService.java +++ b/src/main/java/com/magamochi/service/ProviderManualMangaImportService.java @@ -6,11 +6,11 @@ import static java.util.Objects.nonNull; import com.magamochi.common.exception.NotFoundException; import com.magamochi.ingestion.model.entity.ContentProvider; import com.magamochi.ingestion.model.repository.ContentProviderRepository; +import com.magamochi.ingestion.providers.ManualImportContentProviderFactory; import com.magamochi.model.dto.ImportMangaResponseDTO; import com.magamochi.model.dto.ImportRequestDTO; import com.magamochi.model.entity.MangaContentProvider; import com.magamochi.model.repository.MangaContentProviderRepository; -import com.magamochi.service.providers.ManualImportContentProviderFactory; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; diff --git a/src/main/java/com/magamochi/service/providers/PagedContentProvider.java b/src/main/java/com/magamochi/service/providers/PagedContentProvider.java deleted file mode 100644 index 8ff4c82..0000000 --- a/src/main/java/com/magamochi/service/providers/PagedContentProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.magamochi.service.providers; - -import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO; -import java.util.List; - -public interface PagedContentProvider { - Integer getTotalPages(); - - List getMangasFromPage(Integer page); -} diff --git a/src/main/java/com/magamochi/service/providers/impl/BatoProvider.java b/src/main/java/com/magamochi/service/providers/impl/BatoProvider.java deleted file mode 100644 index 99328ec..0000000 --- a/src/main/java/com/magamochi/service/providers/impl/BatoProvider.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.magamochi.service.providers.impl; - -import static java.util.Objects.isNull; - -import com.magamochi.common.exception.UnprocessableException; -import com.magamochi.ingestion.service.FlareService; -import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO; -import com.magamochi.model.entity.MangaContentProvider; -import com.magamochi.service.providers.ContentProvider; -import com.magamochi.service.providers.ContentProviders; -import com.magamochi.service.providers.ManualImportContentProvider; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.springframework.stereotype.Service; - -@Log4j2 -@Service(ContentProviders.BATO) -@RequiredArgsConstructor -public class BatoProvider implements ContentProvider, ManualImportContentProvider { - private static final String URL = "https://battwo.com"; - - private final FlareService flareService; - - @Override - public List getAvailableChapters( - MangaContentProvider provider) { - try { - var document = - flareService.getContentAsJsoupDocument(provider.getUrl(), ContentProviders.BATO); - - // Direct selector for chapter links - var chapterLinks = document.select("div.scrollable-panel a[href*=/title/]"); - - // TODO: fix chapter and language code - return chapterLinks.stream() - .map( - chapterLink -> - new ContentProviderMangaChapterResponseDTO( - chapterLink.text(), chapterLink.attr("href"), null, null)) - .toList(); - } catch (Exception e) { - log.warn(e.getMessage()); - return null; - } - } - - @Override - public Map getChapterImagesUrls(String chapterUrl) { - try { - var document = - flareService.getContentAsJsoupDocument( - URL + chapterUrl + "?load=2", ContentProviders.BATO); - - // Select all chapter page images - var imgElements = document.select("img.z-10.w-full.h-full"); - - List imageUrls = new ArrayList<>(); - for (var img : imgElements) { - String src = img.attr("src"); - - // Normalize if needed - if (!src.startsWith("http")) { - src = "https://battwo.com" + src; - } - - imageUrls.add(src); - } - - return IntStream.range(0, imageUrls.size()) - .boxed() - .collect( - Collectors.toMap( - i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new)); - - } catch (Exception e) { - log.warn(e.getMessage()); - return null; - } - } - - @Override - public String getMangaTitle(String value) { - var document = flareService.getContentAsJsoupDocument(value, ContentProviders.BATO); - - var titleElement = document.selectFirst("h3 a[href*=/title/]"); - if (isNull(titleElement)) { - titleElement = document.selectFirst("h3.item-title > a"); - - if (isNull(titleElement)) { - throw new UnprocessableException("Manga title not found for url: " + value); - } - } - - return titleElement.text(); - } -} diff --git a/src/main/java/com/magamochi/task/UpdateMangaDataTask.java b/src/main/java/com/magamochi/task/UpdateMangaDataTask.java deleted file mode 100644 index ff67b48..0000000 --- a/src/main/java/com/magamochi/task/UpdateMangaDataTask.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.magamochi.task; - -import static java.util.Objects.isNull; - -import com.magamochi.model.dto.UpdateMangaDataCommand; -import com.magamochi.model.repository.MangaRepository; -import com.magamochi.queue.UpdateMangaDataProducer; -import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -@Log4j2 -@Component -@RequiredArgsConstructor -public class UpdateMangaDataTask { - private final MangaRepository mangaRepository; - private final UpdateMangaDataProducer updateMangaDataProducer; - - @Scheduled(cron = "@daily") - public void updateMangaData() { - var mangas = - mangaRepository.findAll().stream().filter(manga -> isNull(manga.getScore())).toList(); - - mangas.forEach( - manga -> - updateMangaDataProducer.sendUpdateMangaDataCommand( - new UpdateMangaDataCommand(manga.getId()))); - } -} diff --git a/src/main/java/com/magamochi/task/UpdateMangaListTask.java b/src/main/java/com/magamochi/task/UpdateMangaListTask.java deleted file mode 100644 index 466776a..0000000 --- a/src/main/java/com/magamochi/task/UpdateMangaListTask.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.magamochi.task; - -import com.magamochi.common.exception.NotFoundException; -import com.magamochi.ingestion.model.repository.ContentProviderRepository; -import com.magamochi.model.dto.MangaListUpdateCommand; -import com.magamochi.queue.UpdateMangaListProducer; -import com.magamochi.service.providers.PagedContentProvider; -import com.magamochi.service.providers.PagedContentProviderFactory; -import java.util.stream.IntStream; -import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -@Log4j2 -@Component -@RequiredArgsConstructor -public class UpdateMangaListTask { - @Value("${content-providers.update-enabled}") - private Boolean updateEnabled; - - private final PagedContentProviderFactory contentProviderFactory; - private final UpdateMangaListProducer updateMangaListProducer; - private final ContentProviderRepository contentProviderRepository; - - @Scheduled(cron = "${content-providers.cron-expression}") - public void updateMangaListScheduled() { - if (!updateEnabled) { - return; - } - - updateMangaList(); - } - - public void updateMangaList() { - log.info("Queuing manga list updates..."); - - var contentProviders = contentProviderFactory.getContentProviders(); - contentProviders.forEach(this::updateProviderMangaList); - } - - public void updateProviderMangaList(Long providerId) { - var provider = - contentProviderRepository - .findById(providerId) - .orElseThrow(() -> new NotFoundException("Provider not found")); - var contentProvider = contentProviderFactory.getPagedContentProvider(provider.getName()); - - updateProviderMangaList(provider.getName(), contentProvider); - } - - private void updateProviderMangaList( - String contentProviderName, PagedContentProvider contentProvider) { - log.info("Getting total pages for provider {}", contentProviderName); - - var pages = contentProvider.getTotalPages(); - - IntStream.rangeClosed(1, pages) - .forEach( - page -> - updateMangaListProducer.sendUpdateMangaListCommand( - new MangaListUpdateCommand(contentProviderName, page))); - - log.info("Manga list update queued for content provider {}.", contentProviderName); - } -} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2ab855a..f9395d2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -88,6 +88,10 @@ resilience4j: - java.io.IOException - java.net.SocketTimeoutException +queues: + manga-ingest: ${MANGA_INGEST_QUEUE:mangaIngest} + provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:providerPageIngest} + rabbit-mq: queues: manga-data-update: ${MANGA_DATA_UPDATE_QUEUE:mangaDataUpdateQueue} diff --git a/src/main/resources/db/migration/V0001__CREATE_TABLES.sql b/src/main/resources/db/migration/V0001__CREATE_TABLES.sql index 8f7aa62..df0130f 100644 --- a/src/main/resources/db/migration/V0001__CREATE_TABLES.sql +++ b/src/main/resources/db/migration/V0001__CREATE_TABLES.sql @@ -42,6 +42,7 @@ CREATE TABLE content_providers ( id BIGSERIAL NOT NULL PRIMARY KEY, name VARCHAR NOT NULL, + url VARCHAR NOT NULL, active BOOLEAN NOT NULL DEFAULT TRUE, supports_chapter_fetch BOOLEAN NOT NULL DEFAULT TRUE, manual_import BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/src/main/resources/db/migration/V0002__CONTENT_PROVIDERS.sql b/src/main/resources/db/migration/V0002__CONTENT_PROVIDERS.sql new file mode 100644 index 0000000..dce63ff --- /dev/null +++ b/src/main/resources/db/migration/V0002__CONTENT_PROVIDERS.sql @@ -0,0 +1,4 @@ +INSERT INTO content_providers(name, url, active, supports_chapter_fetch, manual_import) +VALUES ('Manga Livre Blog', 'https://mangalivre.blog', TRUE, TRUE, FALSE), + ('Manga Livre.to', 'https://mangalivre.to', TRUE, TRUE, FALSE), + ('Pink Rosa Scan', 'https://scanpinkrosa.blogspot.com', TRUE, TRUE, FALSE); \ No newline at end of file