diff --git a/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java b/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java index be6984f..263f52e 100644 --- a/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java +++ b/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java @@ -11,6 +11,7 @@ import org.springframework.context.annotation.Configuration; public class RabbitConfig { public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue"; public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue"; + public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue"; @Bean public Queue mangaDataUpdateQueue() { @@ -22,6 +23,11 @@ public class RabbitConfig { return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false); } + @Bean + public Queue mangaListUpdateQueue() { + return new Queue(MANGA_LIST_UPDATE_QUEUE, false); + } + @Bean public Jackson2JsonMessageConverter messageConverter() { return new Jackson2JsonMessageConverter(); diff --git a/src/main/java/com/magamochi/mangamochi/controller/DevController.java b/src/main/java/com/magamochi/mangamochi/controller/DevController.java deleted file mode 100644 index a940242..0000000 --- a/src/main/java/com/magamochi/mangamochi/controller/DevController.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.magamochi.mangamochi.controller; - -import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand; -import com.magamochi.mangamochi.queue.UpdateMangaDataProducer; -import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/records") -@RequiredArgsConstructor -public class DevController { - private final UpdateMangaDataProducer producer; - - @PostMapping - public String sendRecord(@RequestBody UpdateMangaDataCommand command) { - try { - producer.sendUpdateMangaDataCommand(command); - } catch (Exception e) { - return e.getMessage(); - } - return "Command sent to RabbitMQ: " + command.mangaId(); - } -} diff --git a/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java b/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java new file mode 100644 index 0000000..3c40efc --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java @@ -0,0 +1,40 @@ +package com.magamochi.mangamochi.controller; + +import com.magamochi.mangamochi.model.dto.DefaultResponseDTO; +import com.magamochi.mangamochi.task.ImageCleanupTask; +import com.magamochi.mangamochi.task.UpdateMangaListTask; +import io.swagger.v3.oas.annotations.Operation; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/management") +@RequiredArgsConstructor +public class ManagementController { + private final UpdateMangaListTask updateMangaListTask; + private final ImageCleanupTask imageCleanupTask; + + @Operation( + summary = "Queue update manga list", + description = "Queue the retrieval of the manga lists from the content providers", + tags = {"Management"}, + operationId = "updateMangaList") + @PostMapping("update-manga-list") + public DefaultResponseDTO updateMangaList() { + updateMangaListTask.updateMangaList(); + + return DefaultResponseDTO.ok().build(); + } + + @Operation( + summary = "Cleanup unused S3 images", + description = "Triggers the cleanup of untracked S3 images", + tags = {"Management"}, + operationId = "imageCleanup") + @PostMapping("image-cleanup") + public DefaultResponseDTO imageCleanup() { + imageCleanupTask.cleanupImages(); + + return DefaultResponseDTO.ok().build(); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/MangaListUpdateCommand.java b/src/main/java/com/magamochi/mangamochi/model/dto/MangaListUpdateCommand.java new file mode 100644 index 0000000..28f714e --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/model/dto/MangaListUpdateCommand.java @@ -0,0 +1,3 @@ +package com.magamochi.mangamochi.model.dto; + +public record MangaListUpdateCommand(String contentProviderName, Integer page) {} diff --git a/src/main/java/com/magamochi/mangamochi/model/repository/MangaImportReviewRepository.java b/src/main/java/com/magamochi/mangamochi/model/repository/MangaImportReviewRepository.java index 6c2872f..8114803 100644 --- a/src/main/java/com/magamochi/mangamochi/model/repository/MangaImportReviewRepository.java +++ b/src/main/java/com/magamochi/mangamochi/model/repository/MangaImportReviewRepository.java @@ -3,4 +3,6 @@ package com.magamochi.mangamochi.model.repository; import com.magamochi.mangamochi.model.entity.MangaImportReview; import org.springframework.data.jpa.repository.JpaRepository; -public interface MangaImportReviewRepository extends JpaRepository {} +public interface MangaImportReviewRepository extends JpaRepository { + boolean existsByTitleIgnoreCaseAndUrlIgnoreCase(String title, String url); +} diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java new file mode 100644 index 0000000..38925c3 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java @@ -0,0 +1,22 @@ +package com.magamochi.mangamochi.queue; + +import com.magamochi.mangamochi.config.RabbitConfig; +import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand; +import com.magamochi.mangamochi.service.MangaListService; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class UpdateMangaListConsumer { + private final MangaListService mangaListService; + + @RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE) + public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) { + log.info("Received update manga list command: {}", command); + mangaListService.updateMangaList(command.contentProviderName(), command.page()); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java new file mode 100644 index 0000000..167e66b --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java @@ -0,0 +1,20 @@ +package com.magamochi.mangamochi.queue; + +import com.magamochi.mangamochi.config.RabbitConfig; +import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class UpdateMangaListProducer { + private final RabbitTemplate rabbitTemplate; + + public void sendUpdateMangaListCommand(MangaListUpdateCommand command) { + rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command); + log.info("Sent update manga list command: {}", command); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java index bd359b9..5e554ca 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java @@ -91,6 +91,10 @@ public class MangaCreationService { } private void createMangaImportReview(String title, String url, Provider provider) { + if (!mangaImportReviewRepository.existsByTitleIgnoreCaseAndUrlIgnoreCase(title, url)) { + return; + } + mangaImportReviewRepository.save( MangaImportReview.builder().title(title).url(url).provider(provider).build()); } diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaListService.java b/src/main/java/com/magamochi/mangamochi/service/MangaListService.java index c725cd3..aa444ca 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaListService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaListService.java @@ -2,10 +2,9 @@ package com.magamochi.mangamochi.service; import static java.util.Objects.isNull; -import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.mangamochi.model.entity.MangaProvider; import com.magamochi.mangamochi.model.repository.MangaProviderRepository; -import java.util.List; +import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; @@ -16,14 +15,17 @@ import org.springframework.stereotype.Service; public class MangaListService { private final ProviderService providerService; private final MangaCreationService mangaCreationService; + private final PagedContentProviderFactory pagedContentProviderFactory; private final MangaProviderRepository mangaProviderRepository; - public void updateMangaList( - String contentProviderName, List mangaInfoResponseDTOs) { + public void updateMangaList(String contentProviderName, Integer page) { + var contentProvider = pagedContentProviderFactory.getPagedContentProvider(contentProviderName); var provider = providerService.getOrCreateProvider(contentProviderName); - mangaInfoResponseDTOs.forEach( + var mangas = contentProvider.getMangasFromPage(page); + + mangas.forEach( mangaResponse -> { var mangaProvider = mangaProviderRepository.findByMangaTitleIgnoreCaseAndProvider( diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/ContentProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/ContentProvider.java index c239d12..9cb16cb 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/ContentProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/ContentProvider.java @@ -1,14 +1,11 @@ package com.magamochi.mangamochi.service.providers; import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; -import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.mangamochi.model.entity.MangaProvider; import java.util.List; import java.util.Map; public interface ContentProvider { - List getAvailableMangas(); - List getAvailableChapters(MangaProvider provider); Map getChapterImagesUrls(String chapterUrl); diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProvider.java new file mode 100644 index 0000000..7cc59fe --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProvider.java @@ -0,0 +1,10 @@ +package com.magamochi.mangamochi.service.providers; + +import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; +import java.util.List; + +public interface PagedContentProvider { + Integer getTotalPages(); + + List getMangasFromPage(Integer page); +} diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProviderFactory.java b/src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProviderFactory.java new file mode 100644 index 0000000..aa0dfba --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProviderFactory.java @@ -0,0 +1,24 @@ +package com.magamochi.mangamochi.service.providers; + +import java.util.Map; +import java.util.Objects; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +@Getter +@Component +@RequiredArgsConstructor +public class PagedContentProviderFactory { + private final Map contentProviders; + + public PagedContentProvider getPagedContentProvider(String providerName) { + var provider = contentProviders.get(providerName); + + if (Objects.isNull(provider)) { + throw new IllegalArgumentException("No such provider " + providerName); + } + + return provider; + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java index 772b73c..1f937da 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java @@ -7,7 +7,6 @@ import com.magamochi.mangamochi.client.MangaDexClient; import com.magamochi.mangamochi.exception.NotFoundException; import com.magamochi.mangamochi.exception.UnprocessableException; import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; -import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO; import com.magamochi.mangamochi.model.entity.MangaProvider; import com.magamochi.mangamochi.model.entity.Provider; @@ -35,14 +34,6 @@ public class MangaDexProvider implements ContentProvider { private final RateLimiter mangaDexRateLimiter; - @Override - public List getAvailableMangas() { - // MangaDex API does not provide an endpoint to list all mangas directly. - // As there is lots and lots of mangas, this is not feasible to implement here. - // The frontend has a function to import mangas by their IDs instead. - return List.of(); - } - @Override public List getAvailableChapters(MangaProvider provider) { try { diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java index 49eeeec..eec5ba3 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java @@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus; import com.magamochi.mangamochi.service.WebScrapperClientProxyService; import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.service.providers.ContentProviders; +import com.magamochi.mangamochi.service.providers.PagedContentProvider; import java.io.IOException; import java.util.*; import java.util.regex.Pattern; @@ -21,28 +22,13 @@ import org.springframework.stereotype.Service; @Log4j2 @Service(ContentProviders.MANGA_LIVRE_BLOG) @RequiredArgsConstructor -public class MangaLivreBlogProvider implements ContentProvider { +public class MangaLivreBlogProvider implements ContentProvider, PagedContentProvider { private static final Pattern NUMERIC_PATTERN = Pattern.compile("-?\\d+"); private final String url = "https://mangalivre.blog/manga/"; private final WebScrapperClientProxyService webScrapperClientProxyService; - @Override - public List getAvailableMangas() { - var totalPages = getTotalPages(); - - if (Objects.isNull(totalPages) || totalPages < 1) { - return List.of(); - } - - return IntStream.rangeClosed(1, totalPages) - .mapToObj(this::getMangasFromPage) - .filter(Objects::nonNull) - .flatMap(List::stream) - .toList(); - } - @Override public List getAvailableChapters( MangaProvider mangaProvider) { @@ -109,7 +95,8 @@ public class MangaLivreBlogProvider implements ContentProvider { } } - private List getMangasFromPage(int page) { + @Override + public List getMangasFromPage(Integer page) { try { var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page); @@ -159,7 +146,8 @@ public class MangaLivreBlogProvider implements ContentProvider { } } - private Integer getTotalPages() { + @Override + public Integer getTotalPages() { try { var document = webScrapperClientProxyService.scrapeToJsoupDocument(url); diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java index 8b144c2..f00dad1 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java @@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus; import com.magamochi.mangamochi.service.WebScrapperClientProxyService; import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.service.providers.ContentProviders; +import com.magamochi.mangamochi.service.providers.PagedContentProvider; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -18,26 +19,11 @@ import org.springframework.stereotype.Service; @Log4j2 @Service(ContentProviders.MANGA_LIVRE) @RequiredArgsConstructor -public class MangaLivreProvider implements ContentProvider { +public class MangaLivreProvider implements ContentProvider, PagedContentProvider { private final String url = "https://mangalivre.tv/manga/"; private final WebScrapperClientProxyService webScrapperClientProxyService; - @Override - public List getAvailableMangas() { - var totalPages = getTotalPages(); - - if (Objects.isNull(totalPages) || totalPages < 1) { - return List.of(); - } - - return IntStream.rangeClosed(1, totalPages) - .mapToObj(this::getMangasFromPage) - .filter(Objects::nonNull) - .flatMap(List::stream) - .toList(); - } - @Override public List getAvailableChapters(MangaProvider provider) { try { @@ -88,7 +74,8 @@ public class MangaLivreProvider implements ContentProvider { } } - private List getMangasFromPage(int page) { + @Override + public List getMangasFromPage(Integer page) { try { var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page); @@ -134,7 +121,8 @@ public class MangaLivreProvider implements ContentProvider { } } - private Integer getTotalPages() { + @Override + public Integer getTotalPages() { try { var document = webScrapperClientProxyService.scrapeToJsoupDocument(url); diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/PinkRosaScanProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/PinkRosaScanProvider.java index e52f55e..578e971 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/PinkRosaScanProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/PinkRosaScanProvider.java @@ -10,6 +10,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus; import com.magamochi.mangamochi.service.WebScrapperClientProxyService; import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.service.providers.ContentProviders; +import com.magamochi.mangamochi.service.providers.PagedContentProvider; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -21,47 +22,9 @@ import org.springframework.stereotype.Service; @Log4j2 @Service(ContentProviders.PINK_ROSA_SCAN) @RequiredArgsConstructor -public class PinkRosaScanProvider implements ContentProvider { +public class PinkRosaScanProvider implements ContentProvider, PagedContentProvider { private final WebScrapperClientProxyService webScrapperClientProxyService; - @Override - public List getAvailableMangas() { - try { - var document = - webScrapperClientProxyService.scrapeToJsoupDocument( - "https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000"); - - var mangaElements = - document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit"); - - return mangaElements.stream() - .map( - element -> { - var linkElement = - element - .getElementsByClass( - "flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw] w-full") - .getFirst() - .getElementsByTag("div") - .getFirst() - .getElementsByTag("a") - .getFirst(); - - var url = linkElement.attr("href"); - - var textElement = linkElement.getElementsByTag("h3"); - var title = textElement.text().trim(); - - return new ContentProviderMangaInfoResponseDTO( - title, url, null, MangaStatus.UNKNOWN); - }) - .toList(); - } catch (NoSuchElementException | IOException e) { - log.error("Error parsing mangas from Pink Rosa Scan", e); - return List.of(); - } - } - @Override public List getAvailableChapters(MangaProvider provider) { try { @@ -129,4 +92,47 @@ public class PinkRosaScanProvider implements ContentProvider { return Map.of(); } } + + @Override + public Integer getTotalPages() { + return 1; + } + + @Override + public List getMangasFromPage(Integer page) { + try { + var document = + webScrapperClientProxyService.scrapeToJsoupDocument( + "https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000"); + + var mangaElements = + document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit"); + + return mangaElements.stream() + .map( + element -> { + var linkElement = + element + .getElementsByClass( + "flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw] w-full") + .getFirst() + .getElementsByTag("div") + .getFirst() + .getElementsByTag("a") + .getFirst(); + + var url = linkElement.attr("href"); + + var textElement = linkElement.getElementsByTag("h3"); + var title = textElement.text().trim(); + + return new ContentProviderMangaInfoResponseDTO( + title, url, null, MangaStatus.UNKNOWN); + }) + .toList(); + } catch (NoSuchElementException | IOException e) { + log.error("Error parsing mangas from Pink Rosa Scan", e); + return List.of(); + } + } } diff --git a/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java b/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java index 0fce0d3..f9756be 100644 --- a/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java +++ b/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java @@ -20,13 +20,17 @@ public class ImageCleanupTask { private final S3Service s3Service; private final ImageRepository imageRepository; - @Scheduled(cron = "@weekly") - public void cleanupImages() { + @Scheduled(cron = "${image-service.cron-expression}") + public void cleanUpImagesScheduled() { if (!cleanUpEnabled) { log.info("S3 Image cleanup disabled."); return; } + cleanupImages(); + } + + public void cleanupImages() { log.info("Getting unused S3 object keys to remove."); var imageKeys = s3Service.listAllObjectKeys(); diff --git a/src/main/java/com/magamochi/mangamochi/task/UpdateMangaListTask.java b/src/main/java/com/magamochi/mangamochi/task/UpdateMangaListTask.java index ddc8c3e..fdec2da 100644 --- a/src/main/java/com/magamochi/mangamochi/task/UpdateMangaListTask.java +++ b/src/main/java/com/magamochi/mangamochi/task/UpdateMangaListTask.java @@ -1,33 +1,55 @@ package com.magamochi.mangamochi.task; -import com.magamochi.mangamochi.service.MangaListService; -import com.magamochi.mangamochi.service.providers.ContentProvider; -import com.magamochi.mangamochi.service.providers.ContentProviderFactory; +import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand; +import com.magamochi.mangamochi.queue.UpdateMangaListProducer; +import com.magamochi.mangamochi.service.providers.PagedContentProvider; +import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory; +import java.util.stream.IntStream; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @Log4j2 @Component @RequiredArgsConstructor public class UpdateMangaListTask { - private final ContentProviderFactory contentProviderFactory; - private final MangaListService mangaListService; + @Value("${content-providers.update-enabled}") + private Boolean updateEnabled; + + private final PagedContentProviderFactory contentProviderFactory; + private final UpdateMangaListProducer updateMangaListProducer; + + @Scheduled(cron = "${content-providers.cron-expression}") + public void updateMangaListScheduled() { + if (!updateEnabled) { + return; + } + + updateMangaList(); + } - // @Scheduled(fixedDelayString = "1d") public void updateMangaList() { - log.info("Updating manga list..."); + log.info("Queuing manga list updates..."); var contentProviders = contentProviderFactory.getContentProviders(); contentProviders.forEach(this::updateProviderMangaList); - - log.info("Manga list updated."); } private void updateProviderMangaList( - String contentProviderName, ContentProvider contentProvider) { - log.info("Updating manga list for content provider {}", contentProviderName); - mangaListService.updateMangaList(contentProviderName, contentProvider.getAvailableMangas()); - log.info("Manga list for content provider {} updated.", contentProviderName); + String contentProviderName, PagedContentProvider contentProvider) { + log.info("Getting total pages for provider {}", contentProviderName); + + var pages = contentProvider.getTotalPages(); + + IntStream.rangeClosed(1, pages) + .forEach( + page -> { + updateMangaListProducer.sendUpdateMangaListCommand( + new MangaListUpdateCommand(contentProviderName, page)); + }); + + log.info("Manga list update queued for content provider {}.", contentProviderName); } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index ce1f091..f62ee92 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -23,8 +23,8 @@ spring: client: config: web-scrapper: - connect-timeout: 120000 - read-timeout: 120000 + connect-timeout: 240000 + read-timeout: 240000 rabbitmq: host: ${RABBITMQ_HOST} port: ${RABBITMQ_PORT} @@ -71,3 +71,9 @@ resilience4j: image-service: clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false} + cron-expression: "@weekly" + +content-providers: + update-enabled: ${CONTENT_PROVIDER_UPDATE_ENABLED:false} + cron-expression: "@weekly" +