feat: implement queued manga list update

src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java

@@ -11,6 +11,7 @@ import org.springframework.context.annotation.Configuration;
 public class RabbitConfig {
   public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
   public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue";
+  public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue";
 
   @Bean
   public Queue mangaDataUpdateQueue() {
@@ -22,6 +23,11 @@ public class RabbitConfig {
     return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false);
   }
 
+  @Bean
+  public Queue mangaListUpdateQueue() {
+    return new Queue(MANGA_LIST_UPDATE_QUEUE, false);
+  }
+
   @Bean
   public Jackson2JsonMessageConverter messageConverter() {
     return new Jackson2JsonMessageConverter();

src/main/java/com/magamochi/mangamochi/controller/DevController.java

@@ -1,23 +0,0 @@
-package com.magamochi.mangamochi.controller;
-
-import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
-import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
-import lombok.RequiredArgsConstructor;
-import org.springframework.web.bind.annotation.*;
-
-@RestController
-@RequestMapping("/records")
-@RequiredArgsConstructor
-public class DevController {
-  private final UpdateMangaDataProducer producer;
-
-  @PostMapping
-  public String sendRecord(@RequestBody UpdateMangaDataCommand command) {
-    try {
-      producer.sendUpdateMangaDataCommand(command);
-    } catch (Exception e) {
-      return e.getMessage();
-    }
-    return "Command sent to RabbitMQ: " + command.mangaId();
-  }
-}

src/main/java/com/magamochi/mangamochi/controller/ManagementController.java

@@ -0,0 +1,40 @@
+package com.magamochi.mangamochi.controller;
+
+import com.magamochi.mangamochi.model.dto.DefaultResponseDTO;
+import com.magamochi.mangamochi.task.ImageCleanupTask;
+import com.magamochi.mangamochi.task.UpdateMangaListTask;
+import io.swagger.v3.oas.annotations.Operation;
+import lombok.RequiredArgsConstructor;
+import org.springframework.web.bind.annotation.*;
+
+@RestController
+@RequestMapping("/management")
+@RequiredArgsConstructor
+public class ManagementController {
+  private final UpdateMangaListTask updateMangaListTask;
+  private final ImageCleanupTask imageCleanupTask;
+
+  @Operation(
+      summary = "Queue update manga list",
+      description = "Queue the retrieval of the manga lists from the content providers",
+      tags = {"Management"},
+      operationId = "updateMangaList")
+  @PostMapping("update-manga-list")
+  public DefaultResponseDTO<Void> updateMangaList() {
+    updateMangaListTask.updateMangaList();
+
+    return DefaultResponseDTO.ok().build();
+  }
+
+  @Operation(
+      summary = "Cleanup unused S3 images",
+      description = "Triggers the cleanup of untracked S3 images",
+      tags = {"Management"},
+      operationId = "imageCleanup")
+  @PostMapping("image-cleanup")
+  public DefaultResponseDTO<Void> imageCleanup() {
+    imageCleanupTask.cleanupImages();
+
+    return DefaultResponseDTO.ok().build();
+  }
+}

src/main/java/com/magamochi/mangamochi/model/dto/MangaListUpdateCommand.java

@@ -0,0 +1,3 @@
+package com.magamochi.mangamochi.model.dto;
+
+public record MangaListUpdateCommand(String contentProviderName, Integer page) {}

src/main/java/com/magamochi/mangamochi/model/repository/MangaImportReviewRepository.java

@@ -3,4 +3,6 @@ package com.magamochi.mangamochi.model.repository;
 import com.magamochi.mangamochi.model.entity.MangaImportReview;
 import org.springframework.data.jpa.repository.JpaRepository;
 
-public interface MangaImportReviewRepository extends JpaRepository<MangaImportReview, Long> {}
+public interface MangaImportReviewRepository extends JpaRepository<MangaImportReview, Long> {
+  boolean existsByTitleIgnoreCaseAndUrlIgnoreCase(String title, String url);
+}

src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java

@@ -0,0 +1,22 @@
+package com.magamochi.mangamochi.queue;
+
+import com.magamochi.mangamochi.config.RabbitConfig;
+import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
+import com.magamochi.mangamochi.service.MangaListService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class UpdateMangaListConsumer {
+  private final MangaListService mangaListService;
+
+  @RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE)
+  public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) {
+    log.info("Received update manga list command: {}", command);
+    mangaListService.updateMangaList(command.contentProviderName(), command.page());
+  }
+}

src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java

@@ -0,0 +1,20 @@
+package com.magamochi.mangamochi.queue;
+
+import com.magamochi.mangamochi.config.RabbitConfig;
+import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class UpdateMangaListProducer {
+  private final RabbitTemplate rabbitTemplate;
+
+  public void sendUpdateMangaListCommand(MangaListUpdateCommand command) {
+    rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command);
+    log.info("Sent update manga list command: {}", command);
+  }
+}

src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java

@@ -91,6 +91,10 @@ public class MangaCreationService {
   }
 
   private void createMangaImportReview(String title, String url, Provider provider) {
+    if (!mangaImportReviewRepository.existsByTitleIgnoreCaseAndUrlIgnoreCase(title, url)) {
+      return;
+    }
+
     mangaImportReviewRepository.save(
         MangaImportReview.builder().title(title).url(url).provider(provider).build());
   }

src/main/java/com/magamochi/mangamochi/service/MangaListService.java

@@ -2,10 +2,9 @@ package com.magamochi.mangamochi.service;
 
 import static java.util.Objects.isNull;
 
-import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.mangamochi.model.entity.MangaProvider;
 import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
-import java.util.List;
+import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.stereotype.Service;
@@ -16,14 +15,17 @@ import org.springframework.stereotype.Service;
 public class MangaListService {
   private final ProviderService providerService;
   private final MangaCreationService mangaCreationService;
+  private final PagedContentProviderFactory pagedContentProviderFactory;
 
   private final MangaProviderRepository mangaProviderRepository;
 
-  public void updateMangaList(
-      String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) {
+  public void updateMangaList(String contentProviderName, Integer page) {
+    var contentProvider = pagedContentProviderFactory.getPagedContentProvider(contentProviderName);
     var provider = providerService.getOrCreateProvider(contentProviderName);
 
-    mangaInfoResponseDTOs.forEach(
+    var mangas = contentProvider.getMangasFromPage(page);
+
+    mangas.forEach(
         mangaResponse -> {
           var mangaProvider =
               mangaProviderRepository.findByMangaTitleIgnoreCaseAndProvider(

src/main/java/com/magamochi/mangamochi/service/providers/ContentProvider.java

@@ -1,14 +1,11 @@
 package com.magamochi.mangamochi.service.providers;
 
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
-import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.mangamochi.model.entity.MangaProvider;
 import java.util.List;
 import java.util.Map;
 
 public interface ContentProvider {
-  List<ContentProviderMangaInfoResponseDTO> getAvailableMangas();
-
   List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider);
 
   Map<Integer, String> getChapterImagesUrls(String chapterUrl);

src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProvider.java

@@ -0,0 +1,10 @@
+package com.magamochi.mangamochi.service.providers;
+
+import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
+import java.util.List;
+
+public interface PagedContentProvider {
+  Integer getTotalPages();
+
+  List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page);
+}

src/main/java/com/magamochi/mangamochi/service/providers/PagedContentProviderFactory.java

@@ -0,0 +1,24 @@
+package com.magamochi.mangamochi.service.providers;
+
+import java.util.Map;
+import java.util.Objects;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Component;
+
+@Getter
+@Component
+@RequiredArgsConstructor
+public class PagedContentProviderFactory {
+  private final Map<String, PagedContentProvider> contentProviders;
+
+  public PagedContentProvider getPagedContentProvider(String providerName) {
+    var provider = contentProviders.get(providerName);
+
+    if (Objects.isNull(provider)) {
+      throw new IllegalArgumentException("No such provider " + providerName);
+    }
+
+    return provider;
+  }
+}

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java

@@ -7,7 +7,6 @@ import com.magamochi.mangamochi.client.MangaDexClient;
 import com.magamochi.mangamochi.exception.NotFoundException;
 import com.magamochi.mangamochi.exception.UnprocessableException;
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
-import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
 import com.magamochi.mangamochi.model.entity.MangaProvider;
 import com.magamochi.mangamochi.model.entity.Provider;
@@ -35,14 +34,6 @@ public class MangaDexProvider implements ContentProvider {
 
   private final RateLimiter mangaDexRateLimiter;
 
-  @Override
-  public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
-    // MangaDex API does not provide an endpoint to list all mangas directly.
-    // As there is lots and lots of mangas, this is not feasible to implement here.
-    // The frontend has a function to import mangas by their IDs instead.
-    return List.of();
-  }
-
   @Override
   public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
     try {

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java

@@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
 import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
 import com.magamochi.mangamochi.service.providers.ContentProvider;
 import com.magamochi.mangamochi.service.providers.ContentProviders;
+import com.magamochi.mangamochi.service.providers.PagedContentProvider;
 import java.io.IOException;
 import java.util.*;
 import java.util.regex.Pattern;
@@ -21,28 +22,13 @@ import org.springframework.stereotype.Service;
 @Log4j2
 @Service(ContentProviders.MANGA_LIVRE_BLOG)
 @RequiredArgsConstructor
-public class MangaLivreBlogProvider implements ContentProvider {
+public class MangaLivreBlogProvider implements ContentProvider, PagedContentProvider {
   private static final Pattern NUMERIC_PATTERN = Pattern.compile("-?\\d+");
 
   private final String url = "https://mangalivre.blog/manga/";
 
   private final WebScrapperClientProxyService webScrapperClientProxyService;
 
-  @Override
-  public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
-    var totalPages = getTotalPages();
-
-    if (Objects.isNull(totalPages) || totalPages < 1) {
-      return List.of();
-    }
-
-    return IntStream.rangeClosed(1, totalPages)
-        .mapToObj(this::getMangasFromPage)
-        .filter(Objects::nonNull)
-        .flatMap(List::stream)
-        .toList();
-  }
-
   @Override
   public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(
       MangaProvider mangaProvider) {
@@ -109,7 +95,8 @@ public class MangaLivreBlogProvider implements ContentProvider {
     }
   }
 
-  private List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(int page) {
+  @Override
+  public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
     try {
       var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
 
@@ -159,7 +146,8 @@ public class MangaLivreBlogProvider implements ContentProvider {
     }
   }
 
-  private Integer getTotalPages() {
+  @Override
+  public Integer getTotalPages() {
     try {
       var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
 

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java

@@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
 import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
 import com.magamochi.mangamochi.service.providers.ContentProvider;
 import com.magamochi.mangamochi.service.providers.ContentProviders;
+import com.magamochi.mangamochi.service.providers.PagedContentProvider;
 import java.io.IOException;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -18,26 +19,11 @@ import org.springframework.stereotype.Service;
 @Log4j2
 @Service(ContentProviders.MANGA_LIVRE)
 @RequiredArgsConstructor
-public class MangaLivreProvider implements ContentProvider {
+public class MangaLivreProvider implements ContentProvider, PagedContentProvider {
   private final String url = "https://mangalivre.tv/manga/";
 
   private final WebScrapperClientProxyService webScrapperClientProxyService;
 
-  @Override
-  public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
-    var totalPages = getTotalPages();
-
-    if (Objects.isNull(totalPages) || totalPages < 1) {
-      return List.of();
-    }
-
-    return IntStream.rangeClosed(1, totalPages)
-        .mapToObj(this::getMangasFromPage)
-        .filter(Objects::nonNull)
-        .flatMap(List::stream)
-        .toList();
-  }
-
   @Override
   public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
     try {
@@ -88,7 +74,8 @@ public class MangaLivreProvider implements ContentProvider {
     }
   }
 
-  private List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(int page) {
+  @Override
+  public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
     try {
       var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
 
@@ -134,7 +121,8 @@ public class MangaLivreProvider implements ContentProvider {
     }
   }
 
-  private Integer getTotalPages() {
+  @Override
+  public Integer getTotalPages() {
     try {
       var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
 

src/main/java/com/magamochi/mangamochi/service/providers/impl/PinkRosaScanProvider.java

@@ -10,6 +10,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
 import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
 import com.magamochi.mangamochi.service.providers.ContentProvider;
 import com.magamochi.mangamochi.service.providers.ContentProviders;
+import com.magamochi.mangamochi.service.providers.PagedContentProvider;
 import java.io.IOException;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -21,47 +22,9 @@ import org.springframework.stereotype.Service;
 @Log4j2
 @Service(ContentProviders.PINK_ROSA_SCAN)
 @RequiredArgsConstructor
-public class PinkRosaScanProvider implements ContentProvider {
+public class PinkRosaScanProvider implements ContentProvider, PagedContentProvider {
   private final WebScrapperClientProxyService webScrapperClientProxyService;
 
-  @Override
-  public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
-    try {
-      var document =
-          webScrapperClientProxyService.scrapeToJsoupDocument(
-              "https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000");
-
-      var mangaElements =
-          document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit");
-
-      return mangaElements.stream()
-          .map(
-              element -> {
-                var linkElement =
-                    element
-                        .getElementsByClass(
-                            "flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw]  w-full")
-                        .getFirst()
-                        .getElementsByTag("div")
-                        .getFirst()
-                        .getElementsByTag("a")
-                        .getFirst();
-
-                var url = linkElement.attr("href");
-
-                var textElement = linkElement.getElementsByTag("h3");
-                var title = textElement.text().trim();
-
-                return new ContentProviderMangaInfoResponseDTO(
-                    title, url, null, MangaStatus.UNKNOWN);
-              })
-          .toList();
-    } catch (NoSuchElementException | IOException e) {
-      log.error("Error parsing mangas from Pink Rosa Scan", e);
-      return List.of();
-    }
-  }
-
   @Override
   public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
     try {
@@ -129,4 +92,47 @@ public class PinkRosaScanProvider implements ContentProvider {
       return Map.of();
     }
   }
+
+  @Override
+  public Integer getTotalPages() {
+    return 1;
+  }
+
+  @Override
+  public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
+    try {
+      var document =
+          webScrapperClientProxyService.scrapeToJsoupDocument(
+              "https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000");
+
+      var mangaElements =
+          document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit");
+
+      return mangaElements.stream()
+          .map(
+              element -> {
+                var linkElement =
+                    element
+                        .getElementsByClass(
+                            "flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw]  w-full")
+                        .getFirst()
+                        .getElementsByTag("div")
+                        .getFirst()
+                        .getElementsByTag("a")
+                        .getFirst();
+
+                var url = linkElement.attr("href");
+
+                var textElement = linkElement.getElementsByTag("h3");
+                var title = textElement.text().trim();
+
+                return new ContentProviderMangaInfoResponseDTO(
+                    title, url, null, MangaStatus.UNKNOWN);
+              })
+          .toList();
+    } catch (NoSuchElementException | IOException e) {
+      log.error("Error parsing mangas from Pink Rosa Scan", e);
+      return List.of();
+    }
+  }
 }

src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java

@@ -20,13 +20,17 @@ public class ImageCleanupTask {
   private final S3Service s3Service;
   private final ImageRepository imageRepository;
 
-  @Scheduled(cron = "@weekly")
-  public void cleanupImages() {
+  @Scheduled(cron = "${image-service.cron-expression}")
+  public void cleanUpImagesScheduled() {
     if (!cleanUpEnabled) {
       log.info("S3 Image cleanup disabled.");
       return;
     }
 
+    cleanupImages();
+  }
+
+  public void cleanupImages() {
     log.info("Getting unused S3 object keys to remove.");
 
     var imageKeys = s3Service.listAllObjectKeys();

src/main/java/com/magamochi/mangamochi/task/UpdateMangaListTask.java

@@ -1,33 +1,55 @@
 package com.magamochi.mangamochi.task;
 
-import com.magamochi.mangamochi.service.MangaListService;
-import com.magamochi.mangamochi.service.providers.ContentProvider;
-import com.magamochi.mangamochi.service.providers.ContentProviderFactory;
+import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
+import com.magamochi.mangamochi.queue.UpdateMangaListProducer;
+import com.magamochi.mangamochi.service.providers.PagedContentProvider;
+import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory;
+import java.util.stream.IntStream;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 @Log4j2
 @Component
 @RequiredArgsConstructor
 public class UpdateMangaListTask {
-  private final ContentProviderFactory contentProviderFactory;
-  private final MangaListService mangaListService;
+  @Value("${content-providers.update-enabled}")
+  private Boolean updateEnabled;
+
+  private final PagedContentProviderFactory contentProviderFactory;
+  private final UpdateMangaListProducer updateMangaListProducer;
+
+  @Scheduled(cron = "${content-providers.cron-expression}")
+  public void updateMangaListScheduled() {
+    if (!updateEnabled) {
+      return;
+    }
+
+    updateMangaList();
+  }
 
-  //    @Scheduled(fixedDelayString = "1d")
   public void updateMangaList() {
-    log.info("Updating manga list...");
+    log.info("Queuing manga list updates...");
 
     var contentProviders = contentProviderFactory.getContentProviders();
     contentProviders.forEach(this::updateProviderMangaList);
-
-    log.info("Manga list updated.");
   }
 
   private void updateProviderMangaList(
-      String contentProviderName, ContentProvider contentProvider) {
-    log.info("Updating manga list for content provider {}", contentProviderName);
-    mangaListService.updateMangaList(contentProviderName, contentProvider.getAvailableMangas());
-    log.info("Manga list for content provider {} updated.", contentProviderName);
+      String contentProviderName, PagedContentProvider contentProvider) {
+    log.info("Getting total pages for provider {}", contentProviderName);
+
+    var pages = contentProvider.getTotalPages();
+
+    IntStream.rangeClosed(1, pages)
+        .forEach(
+            page -> {
+              updateMangaListProducer.sendUpdateMangaListCommand(
+                  new MangaListUpdateCommand(contentProviderName, page));
+            });
+
+    log.info("Manga list update queued for content provider {}.", contentProviderName);
   }
 }

src/main/resources/application.yml

@@ -23,8 +23,8 @@ spring:
       client:
         config:
           web-scrapper:
-            connect-timeout: 120000
-            read-timeout: 120000
+            connect-timeout: 240000
+            read-timeout: 240000
   rabbitmq:
     host: ${RABBITMQ_HOST}
     port: ${RABBITMQ_PORT}
@@ -71,3 +71,9 @@ resilience4j:
 
 image-service:
   clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
+  cron-expression: "@weekly"
+
+content-providers:
+  update-enabled: ${CONTENT_PROVIDER_UPDATE_ENABLED:false}
+  cron-expression: "@weekly"
+