src/main/java/com/magamochi/common/config/RabbitConfig.java

@@ -1,4 +1,4 @@
-package com.magamochi.config;
+package com.magamochi.common.config;
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.rabbit.connection.ConnectionFactory;
@@ -10,6 +10,24 @@ import org.springframework.context.annotation.Configuration;
 
 @Configuration
 public class RabbitConfig {
+  @Value("${queues.manga-ingest}")
+  private String mangaIngestQueue;
+
+  @Value("${queues.provider-page-ingest}")
+  private String providerPageIngestQueue;
+
+  @Bean
+  public Queue mangaIngestQueue() {
+    return new Queue(mangaIngestQueue, false);
+  }
+
+  @Bean
+  public Queue providerPageIngestQueue() {
+    return new Queue(providerPageIngestQueue, false);
+  }
+
+  // TODO: remove unused queues
+
   @Value("${rabbit-mq.queues.manga-data-update}")
   private String mangaDataUpdateQueue;
 

src/main/java/com/magamochi/common/queue/command/MangaIngestCommand.java

@@ -0,0 +1,6 @@
+package com.magamochi.common.queue.command;
+
+import jakarta.validation.constraints.NotBlank;
+
+public record MangaIngestCommand(
+    long providerId, @NotBlank String mangaTitle, @NotBlank String url) {}

src/main/java/com/magamochi/controller/ManagementController.java

@@ -2,12 +2,12 @@ package com.magamochi.controller;
 
 import com.magamochi.client.NtfyClient;
 import com.magamochi.common.dto.DefaultResponseDTO;
+import com.magamochi.ingestion.task.IngestFromContentProvidersTask;
 import com.magamochi.model.dto.UpdateMangaDataCommand;
 import com.magamochi.model.repository.UserRepository;
 import com.magamochi.queue.UpdateMangaDataProducer;
 import com.magamochi.task.ImageCleanupTask;
 import com.magamochi.task.MangaFollowUpdateTask;
-import com.magamochi.task.UpdateMangaListTask;
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.RequiredArgsConstructor;
 import org.springframework.web.bind.annotation.*;
@@ -16,7 +16,7 @@ import org.springframework.web.bind.annotation.*;
 @RequestMapping("/management")
 @RequiredArgsConstructor
 public class ManagementController {
-  private final UpdateMangaListTask updateMangaListTask;
+  private final IngestFromContentProvidersTask ingestFromContentProvidersTask;
   private final ImageCleanupTask imageCleanupTask;
   private final MangaFollowUpdateTask mangaFollowUpdateTask;
   private final UserRepository userRepository;
@@ -35,30 +35,6 @@ public class ManagementController {
     return DefaultResponseDTO.ok().build();
   }
 
-  @Operation(
-      summary = "Queue update manga list",
-      description = "Queue the retrieval of the manga lists from the content providers",
-      tags = {"Management"},
-      operationId = "updateMangaList")
-  @PostMapping("update-manga-list")
-  public DefaultResponseDTO<Void> updateMangaList() {
-    updateMangaListTask.updateMangaList();
-
-    return DefaultResponseDTO.ok().build();
-  }
-
-  @Operation(
-      summary = "Queue update provider manga list",
-      description = "Queue the retrieval of the manga list for a specific provider",
-      tags = {"Management"},
-      operationId = "updateProviderMangaList")
-  @PostMapping("update-provider-manga-list")
-  public DefaultResponseDTO<Void> updateProviderMangaList(@RequestParam Long providerId) {
-    updateMangaListTask.updateProviderMangaList(providerId);
-
-    return DefaultResponseDTO.ok().build();
-  }
-
   @Operation(
       summary = "Cleanup unused S3 images",
       description = "Triggers the cleanup of untracked S3 images",

src/main/java/com/magamochi/ingestion/client/FlareClient.java

@@ -1,4 +1,4 @@
-package com.magamochi.client;
+package com.magamochi.ingestion.client;
 
 import io.github.resilience4j.retry.annotation.Retry;
 import java.util.List;

src/main/java/com/magamochi/ingestion/controller/IngestionController.java

@@ -3,18 +3,17 @@ package com.magamochi.ingestion.controller;
 import com.magamochi.common.dto.DefaultResponseDTO;
 import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
 import com.magamochi.ingestion.service.ContentProviderService;
+import com.magamochi.ingestion.service.IngestionService;
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.RequiredArgsConstructor;
-import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.*;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
 
 @RestController
 @RequestMapping("/ingestion")
 @RequiredArgsConstructor
 public class IngestionController {
   private final ContentProviderService contentProviderService;
+  private final IngestionService ingestionService;
 
   @Operation(
       summary = "Get a list of content providers",
@@ -26,4 +25,30 @@ public class IngestionController {
       @RequestParam(name = "manualImport", required = false) Boolean manualImport) {
     return DefaultResponseDTO.ok(contentProviderService.getProviders(manualImport));
   }
+
+  @Operation(
+      summary = "Fetch mangas from a content provider",
+      description =
+          "Triggers the ingestion process for a specific content provider, fetching manga data and queuing it for processing.",
+      tags = {"Ingestion"},
+      operationId = "fetchContentProviderMangas")
+  @PostMapping("/providers/{providerId}/fetch")
+  public DefaultResponseDTO<Void> fetchContentProviderMangas(@PathVariable Long providerId) {
+    ingestionService.fetchContentProviderMangas(providerId);
+
+    return DefaultResponseDTO.ok().build();
+  }
+
+  @Operation(
+      summary = "Fetch mangas from all content providers",
+      description =
+          "Triggers the ingestion process for all content providers, fetching manga data and queuing them for processing.",
+      tags = {"Ingestion"},
+      operationId = "fetchAllContentProviderMangas")
+  @PostMapping("/providers/fetch")
+  public DefaultResponseDTO<Void> fetchAllContentProviderMangas() {
+    ingestionService.fetchAllContentProviderMangas();
+
+    return DefaultResponseDTO.ok().build();
+  }
 }

src/main/java/com/magamochi/ingestion/model/dto/ContentProviderListDTO.java

@@ -11,7 +11,7 @@ public record ContentProviderListDTO(@NotNull List<ContentProviderDTO> providers
         contentProviders.stream().map(ContentProviderDTO::from).toList());
   }
 
-  record ContentProviderDTO(long id, @NotBlank String name) {
+  public record ContentProviderDTO(long id, @NotBlank String name) {
     public static ContentProviderDTO from(ContentProvider contentProvider) {
       return new ContentProviderDTO(contentProvider.getId(), contentProvider.getName());
     }

src/main/java/com/magamochi/ingestion/model/dto/MangaInfoDTO.java

@@ -0,0 +1,6 @@
+package com.magamochi.ingestion.model.dto;
+
+import com.magamochi.model.enumeration.MangaStatus;
+import jakarta.validation.constraints.NotBlank;
+
+public record MangaInfoDTO(@NotBlank String title, @NotBlank String url, MangaStatus status) {}

src/main/java/com/magamochi/ingestion/providers/ContentProvider.java

@@ -1,4 +1,4 @@
-package com.magamochi.service.providers;
+package com.magamochi.ingestion.providers;
 
 import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
 import com.magamochi.model.entity.MangaContentProvider;

src/main/java/com/magamochi/ingestion/providers/ContentProviderFactory.java

@@ -1,4 +1,4 @@
-package com.magamochi.service.providers;
+package com.magamochi.ingestion.providers;
 
 import java.util.Map;
 import java.util.Objects;

src/main/java/com/magamochi/ingestion/providers/ContentProviders.java

@@ -1,9 +1,8 @@
-package com.magamochi.service.providers;
+package com.magamochi.ingestion.providers;
 
 public class ContentProviders {
-  public static final String MANGA_LIVRE_TO = "Manga Livre.to";
   public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
-  public static final String MANGA_DEX = "MangaDex";
+  public static final String MANGA_LIVRE_TO = "Manga Livre.to";
   public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
-  public static final String BATO = "Bato";
+  public static final String MANGA_DEX = "MangaDex";
 }

src/main/java/com/magamochi/ingestion/providers/ManualImportContentProvider.java

@@ -1,4 +1,4 @@
-package com.magamochi.service.providers;
+package com.magamochi.ingestion.providers;
 
 public interface ManualImportContentProvider {
   String getMangaTitle(String value);

src/main/java/com/magamochi/ingestion/providers/ManualImportContentProviderFactory.java

@@ -1,4 +1,4 @@
-package com.magamochi.service.providers;
+package com.magamochi.ingestion.providers;
 
 import java.util.Map;
 import java.util.Objects;

src/main/java/com/magamochi/ingestion/providers/PagedContentProvider.java

@@ -0,0 +1,10 @@
+package com.magamochi.ingestion.providers;
+
+import com.magamochi.ingestion.model.dto.MangaInfoDTO;
+import java.util.List;
+
+public interface PagedContentProvider {
+  int getTotalPages();
+
+  List<MangaInfoDTO> getMangasFromPage(int page);
+}

src/main/java/com/magamochi/ingestion/providers/PagedContentProviderFactory.java

@@ -1,4 +1,4 @@
-package com.magamochi.service.providers;
+package com.magamochi.ingestion.providers;
 
 import java.util.Map;
 import java.util.Objects;

src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java

@@ -1,15 +1,15 @@
-package com.magamochi.service.providers.impl;
+package com.magamochi.ingestion.providers.impl;
 
 import static java.util.Objects.isNull;
 
 import com.google.common.util.concurrent.RateLimiter;
 import com.magamochi.client.MangaDexClient;
 import com.magamochi.common.exception.UnprocessableException;
+import com.magamochi.ingestion.providers.ContentProvider;
+import com.magamochi.ingestion.providers.ContentProviders;
+import com.magamochi.ingestion.providers.ManualImportContentProvider;
 import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
 import com.magamochi.model.entity.MangaContentProvider;
-import com.magamochi.service.providers.ContentProvider;
-import com.magamochi.service.providers.ContentProviders;
-import com.magamochi.service.providers.ManualImportContentProvider;
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;

src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java

@@ -1,12 +1,12 @@
-package com.magamochi.service.providers.impl;
+package com.magamochi.ingestion.providers.impl;
 
+import com.magamochi.ingestion.model.dto.MangaInfoDTO;
+import com.magamochi.ingestion.providers.ContentProvider;
+import com.magamochi.ingestion.providers.ContentProviders;
+import com.magamochi.ingestion.providers.PagedContentProvider;
 import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
-import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.model.entity.MangaContentProvider;
 import com.magamochi.model.enumeration.MangaStatus;
-import com.magamochi.service.providers.ContentProvider;
-import com.magamochi.service.providers.ContentProviders;
-import com.magamochi.service.providers.PagedContentProvider;
 import java.io.IOException;
 import java.util.*;
 import java.util.regex.Pattern;
@@ -101,7 +101,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
   }
 
   @Override
-  public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
+  public List<MangaInfoDTO> getMangasFromPage(int page) {
     log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_BLOG, page);
 
     try {
@@ -134,7 +134,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
                         default -> MangaStatus.UNKNOWN;
                       };
 
-                  return new ContentProviderMangaInfoResponseDTO(title, url, status);
+                  return new MangaInfoDTO(title, url, status);
                 } catch (Exception e) {
                   return null;
                 }
@@ -148,7 +148,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
   }
 
   @Override
-  public Integer getTotalPages() {
+  public int getTotalPages() {
     log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_BLOG);
 
     try {
@@ -166,7 +166,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
       return pageNumbers.stream().max(Integer::compareTo).orElse(null);
     } catch (IOException | NoSuchElementException e) {
       log.error("Error fetching total pages from MangaLivre", e);
-      return null;
+      return 0;
     }
   }
 }

src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java

@@ -1,15 +1,15 @@
-package com.magamochi.service.providers.impl;
+package com.magamochi.ingestion.providers.impl;
 
 import static java.util.Objects.nonNull;
 
+import com.magamochi.ingestion.model.dto.MangaInfoDTO;
+import com.magamochi.ingestion.providers.ContentProvider;
+import com.magamochi.ingestion.providers.ContentProviders;
+import com.magamochi.ingestion.providers.PagedContentProvider;
 import com.magamochi.ingestion.service.FlareService;
 import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
-import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.model.entity.MangaContentProvider;
 import com.magamochi.model.enumeration.MangaStatus;
-import com.magamochi.service.providers.ContentProvider;
-import com.magamochi.service.providers.ContentProviders;
-import com.magamochi.service.providers.PagedContentProvider;
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -90,7 +90,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
   }
 
   @Override
-  public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
+  public List<MangaInfoDTO> getMangasFromPage(int page) {
     log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_TO, page);
 
     try {
@@ -116,8 +116,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
                   }
                 }
 
-                return new ContentProviderMangaInfoResponseDTO(
+                return new MangaInfoDTO(title.trim(), url.trim(), MangaStatus.UNKNOWN);
-                    title.trim(), url.trim(), MangaStatus.UNKNOWN);
               })
           .toList();
     } catch (NoSuchElementException e) {
@@ -127,7 +126,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
   }
 
   @Override
-  public Integer getTotalPages() {
+  public int getTotalPages() {
     log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_TO);
 
     try {
@@ -140,7 +139,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
       return (int) Math.ceil((double) totalMangas / MANGAS_PER_PAGE);
     } catch (NoSuchElementException e) {
       log.error("Error parsing total pages from MangaLivre", e);
-      return null;
+      return 0;
     }
   }
 }

src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java

@@ -1,16 +1,16 @@
-package com.magamochi.service.providers.impl;
+package com.magamochi.ingestion.providers.impl;
 
 import static java.util.Objects.isNull;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 
+import com.magamochi.ingestion.model.dto.MangaInfoDTO;
+import com.magamochi.ingestion.providers.ContentProvider;
+import com.magamochi.ingestion.providers.ContentProviders;
+import com.magamochi.ingestion.providers.PagedContentProvider;
 import com.magamochi.ingestion.service.FlareService;
 import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
-import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.model.entity.MangaContentProvider;
 import com.magamochi.model.enumeration.MangaStatus;
-import com.magamochi.service.providers.ContentProvider;
-import com.magamochi.service.providers.ContentProviders;
-import com.magamochi.service.providers.PagedContentProvider;
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -110,12 +110,12 @@ public class PinkRosaScanProvider implements ContentProvider, PagedContentProvid
   }
 
   @Override
-  public Integer getTotalPages() {
+  public int getTotalPages() {
     return 1;
   }
 
   @Override
-  public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
+  public List<MangaInfoDTO> getMangasFromPage(int page) {
     log.info("Getting mangas from {}", ContentProviders.PINK_ROSA_SCAN);
 
     try {
@@ -142,7 +142,7 @@ public class PinkRosaScanProvider implements ContentProvider, PagedContentProvid
                 var textElement = linkElement.getElementsByTag("h3");
                 var title = textElement.text().trim();
 
-                return new ContentProviderMangaInfoResponseDTO(title, url, MangaStatus.UNKNOWN);
+                return new MangaInfoDTO(title, url, MangaStatus.UNKNOWN);
               })
           .toList();
     } catch (NoSuchElementException e) {

src/main/java/com/magamochi/ingestion/queue/command/ProviderPageIngestCommand.java

@@ -0,0 +1,3 @@
+package com.magamochi.ingestion.queue.command;
+
+public record ProviderPageIngestCommand(long providerId, int page) {}

src/main/java/com/magamochi/ingestion/queue/consumer/ProviderPageIngestConsumer.java

@@ -0,0 +1,21 @@
+package com.magamochi.ingestion.queue.consumer;
+
+import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
+import com.magamochi.ingestion.service.IngestionService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class ProviderPageIngestConsumer {
+  private final IngestionService ingestionService;
+
+  @RabbitListener(queues = "${queues.provider-page-ingest}")
+  public void receiveProviderPageIngestCommand(ProviderPageIngestCommand command) {
+    log.info("Received provider page ingest command: {}", command);
+    ingestionService.fetchProviderPageMangas(command.providerId(), command.page());
+  }
+}

src/main/java/com/magamochi/ingestion/queue/producer/MangaIngestProducer.java

@@ -0,0 +1,23 @@
+package com.magamochi.ingestion.queue.producer;
+
+import com.magamochi.common.queue.command.MangaIngestCommand;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class MangaIngestProducer {
+  private final RabbitTemplate rabbitTemplate;
+
+  @Value("${queues.manga-ingest}")
+  private String mangaIngestQueue;
+
+  public void sendMangaIngestCommand(MangaIngestCommand command) {
+    rabbitTemplate.convertAndSend(mangaIngestQueue, command);
+    log.info("Sent manga ingest command: {}", command);
+  }
+}

src/main/java/com/magamochi/ingestion/queue/producer/ProviderPageIngestProducer.java

@@ -0,0 +1,23 @@
+package com.magamochi.ingestion.queue.producer;
+
+import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class ProviderPageIngestProducer {
+  private final RabbitTemplate rabbitTemplate;
+
+  @Value("${queues.provider-page-ingest}")
+  private String providerPageIngestQueue;
+
+  public void sendProviderPageIngestCommand(ProviderPageIngestCommand command) {
+    rabbitTemplate.convertAndSend(providerPageIngestQueue, command);
+    log.info("Sent provider page ingest command: {}", command);
+  }
+}

src/main/java/com/magamochi/ingestion/service/ContentProviderService.java

@@ -2,6 +2,7 @@ package com.magamochi.ingestion.service;
 
 import static java.util.Objects.nonNull;
 
+import com.magamochi.common.exception.NotFoundException;
 import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
 import com.magamochi.ingestion.model.entity.ContentProvider;
 import com.magamochi.ingestion.model.repository.ContentProviderRepository;
@@ -22,4 +23,13 @@ public class ContentProviderService {
 
     return ContentProviderListDTO.from(providers);
   }
+
+  public ContentProvider find(Long contentProviderId) {
+    return contentProviderRepository
+        .findById(contentProviderId)
+        .orElseThrow(
+            () ->
+                new NotFoundException(
+                    "Content Provider not found (ID: " + contentProviderId + ")."));
+  }
 }

src/main/java/com/magamochi/ingestion/service/FlareService.java

@@ -1,6 +1,6 @@
 package com.magamochi.ingestion.service;
 
-import com.magamochi.client.FlareClient;
+import com.magamochi.ingestion.client.FlareClient;
 import lombok.RequiredArgsConstructor;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;

src/main/java/com/magamochi/ingestion/service/FlareSessionManager.java

@@ -3,9 +3,8 @@ package com.magamochi.ingestion.service;
 import static java.util.Objects.isNull;
 import static java.util.Objects.nonNull;
 
-import com.magamochi.client.FlareClient;
+import com.magamochi.ingestion.client.FlareClient;
 import com.magamochi.ingestion.model.entity.FlareSession;
-import com.magamochi.registry.FlareSessionRegistry;
 import java.time.Duration;
 import java.time.Instant;
 import lombok.RequiredArgsConstructor;

src/main/java/com/magamochi/ingestion/service/FlareSessionRegistry.java

@@ -1,4 +1,4 @@
-package com.magamochi.registry;
+package com.magamochi.ingestion.service;
 
 import com.magamochi.ingestion.model.entity.FlareSession;
 import java.util.concurrent.ConcurrentHashMap;

src/main/java/com/magamochi/ingestion/service/IngestionService.java

@@ -0,0 +1,51 @@
+package com.magamochi.ingestion.service;
+
+import com.magamochi.common.queue.command.MangaIngestCommand;
+import com.magamochi.ingestion.providers.PagedContentProviderFactory;
+import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
+import com.magamochi.ingestion.queue.producer.MangaIngestProducer;
+import com.magamochi.ingestion.queue.producer.ProviderPageIngestProducer;
+import java.util.stream.IntStream;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class IngestionService {
+  private final ContentProviderService contentProviderService;
+  private final PagedContentProviderFactory pagedContentProviderFactory;
+
+  private final ProviderPageIngestProducer providerPageIngestProducer;
+  private final MangaIngestProducer mangaIngestProducer;
+
+  public void fetchContentProviderMangas(long contentProviderId) {
+    var contentProvider = contentProviderService.find(contentProviderId);
+    var pagedContentProvider =
+        pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName());
+    var pages = pagedContentProvider.getTotalPages();
+
+    IntStream.rangeClosed(1, pages)
+        .forEach(
+            page ->
+                providerPageIngestProducer.sendProviderPageIngestCommand(
+                    new ProviderPageIngestCommand(contentProvider.getId(), page)));
+  }
+
+  public void fetchProviderPageMangas(long providerId, int page) {
+    var contentProvider = contentProviderService.find(providerId);
+    var pagedContentProvider =
+        pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName());
+
+    var mangas = pagedContentProvider.getMangasFromPage(page);
+
+    mangas.forEach(
+        manga ->
+            mangaIngestProducer.sendMangaIngestCommand(
+                new MangaIngestCommand(contentProvider.getId(), manga.title(), manga.url())));
+  }
+
+  public void fetchAllContentProviderMangas() {
+    var contentProviders = contentProviderService.getProviders(null);
+    contentProviders.providers().forEach(dto -> fetchContentProviderMangas(dto.id()));
+  }
+}

src/main/java/com/magamochi/ingestion/task/FlareSessionCleanupTask.java

@@ -1,7 +1,7 @@
 package com.magamochi.ingestion.task;
 
-import com.magamochi.client.FlareClient;
+import com.magamochi.ingestion.client.FlareClient;
-import com.magamochi.registry.FlareSessionRegistry;
+import com.magamochi.ingestion.service.FlareSessionRegistry;
 import java.time.Duration;
 import java.time.Instant;
 import lombok.RequiredArgsConstructor;

src/main/java/com/magamochi/ingestion/task/FlareStartupCleanupTask.java

@@ -1,6 +1,6 @@
 package com.magamochi.ingestion.task;
 
-import com.magamochi.client.FlareClient;
+import com.magamochi.ingestion.client.FlareClient;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.boot.context.event.ApplicationReadyEvent;

src/main/java/com/magamochi/ingestion/task/IngestFromContentProvidersTask.java

@@ -0,0 +1,31 @@
+package com.magamochi.ingestion.task;
+
+import com.magamochi.ingestion.service.IngestionService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Log4j2
+@Component
+@RequiredArgsConstructor
+public class IngestFromContentProvidersTask {
+  @Value("${content-providers.update-enabled}")
+  private Boolean updateEnabled;
+
+  private final IngestionService ingestionService;
+
+  @Scheduled(cron = "${content-providers.cron-expression}")
+  public void updateMangaListScheduled() {
+    if (!updateEnabled) {
+      return;
+    }
+
+    log.info("Starting scheduling ingest from Content Providers.");
+
+    ingestionService.fetchAllContentProviderMangas();
+
+    log.info("Finished scheduling ingest from Content Providers.");
+  }
+}

src/main/java/com/magamochi/model/dto/ContentProviderMangaInfoResponseDTO.java

@@ -1,7 +0,0 @@
-package com.magamochi.model.dto;
-
-import com.magamochi.model.enumeration.MangaStatus;
-import jakarta.validation.constraints.NotBlank;
-
-public record ContentProviderMangaInfoResponseDTO(
-    @NotBlank String title, @NotBlank String url, MangaStatus status) {}

src/main/java/com/magamochi/model/dto/MangaMessageDTO.java

@@ -1,6 +1,7 @@
 package com.magamochi.model.dto;
 
+import com.magamochi.ingestion.model.dto.MangaInfoDTO;
 import java.util.List;
 
 public record MangaMessageDTO(
-    String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) {}
+    String contentProviderName, List<MangaInfoDTO> mangaInfoResponseDTOs) {}

src/main/java/com/magamochi/service/MangaChapterService.java

@@ -2,6 +2,7 @@ package com.magamochi.service;
 
 import com.google.common.util.concurrent.RateLimiter;
 import com.magamochi.common.exception.UnprocessableException;
+import com.magamochi.ingestion.providers.ContentProviderFactory;
 import com.magamochi.model.dto.MangaChapterArchiveDTO;
 import com.magamochi.model.dto.MangaChapterImagesDTO;
 import com.magamochi.model.entity.MangaChapter;
@@ -9,7 +10,6 @@ import com.magamochi.model.entity.MangaChapterImage;
 import com.magamochi.model.enumeration.ArchiveFileType;
 import com.magamochi.model.repository.MangaChapterImageRepository;
 import com.magamochi.model.repository.MangaChapterRepository;
-import com.magamochi.service.providers.ContentProviderFactory;
 import io.github.resilience4j.retry.Retry;
 import io.github.resilience4j.retry.RetryRegistry;
 import java.io.BufferedInputStream;

src/main/java/com/magamochi/service/MangaListService.java

@@ -2,9 +2,9 @@ package com.magamochi.service;
 
 import static java.util.Objects.isNull;
 
+import com.magamochi.ingestion.providers.PagedContentProviderFactory;
 import com.magamochi.model.entity.MangaContentProvider;
 import com.magamochi.model.repository.MangaContentProviderRepository;
-import com.magamochi.service.providers.PagedContentProviderFactory;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.stereotype.Service;

src/main/java/com/magamochi/service/MangaService.java

@@ -4,6 +4,7 @@ import static java.util.Objects.nonNull;
 
 import com.magamochi.client.NtfyClient;
 import com.magamochi.common.exception.NotFoundException;
+import com.magamochi.ingestion.providers.ContentProviderFactory;
 import com.magamochi.model.dto.*;
 import com.magamochi.model.entity.Manga;
 import com.magamochi.model.entity.MangaChapter;
@@ -12,7 +13,6 @@ import com.magamochi.model.entity.UserMangaFollow;
 import com.magamochi.model.repository.*;
 import com.magamochi.model.specification.MangaSpecification;
 import com.magamochi.queue.MangaChapterDownloadProducer;
-import com.magamochi.service.providers.ContentProviderFactory;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Set;

src/main/java/com/magamochi/service/ProviderManualMangaImportService.java

@@ -6,11 +6,11 @@ import static java.util.Objects.nonNull;
 import com.magamochi.common.exception.NotFoundException;
 import com.magamochi.ingestion.model.entity.ContentProvider;
 import com.magamochi.ingestion.model.repository.ContentProviderRepository;
+import com.magamochi.ingestion.providers.ManualImportContentProviderFactory;
 import com.magamochi.model.dto.ImportMangaResponseDTO;
 import com.magamochi.model.dto.ImportRequestDTO;
 import com.magamochi.model.entity.MangaContentProvider;
 import com.magamochi.model.repository.MangaContentProviderRepository;
-import com.magamochi.service.providers.ManualImportContentProviderFactory;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.stereotype.Service;

src/main/java/com/magamochi/service/providers/PagedContentProvider.java

@@ -1,10 +0,0 @@
-package com.magamochi.service.providers;
-
-import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
-import java.util.List;
-
-public interface PagedContentProvider {
-  Integer getTotalPages();
-
-  List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page);
-}

src/main/java/com/magamochi/service/providers/impl/BatoProvider.java

@@ -1,99 +0,0 @@
-package com.magamochi.service.providers.impl;
-
-import static java.util.Objects.isNull;
-
-import com.magamochi.common.exception.UnprocessableException;
-import com.magamochi.ingestion.service.FlareService;
-import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
-import com.magamochi.model.entity.MangaContentProvider;
-import com.magamochi.service.providers.ContentProvider;
-import com.magamochi.service.providers.ContentProviders;
-import com.magamochi.service.providers.ManualImportContentProvider;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-import org.springframework.stereotype.Service;
-
-@Log4j2
-@Service(ContentProviders.BATO)
-@RequiredArgsConstructor
-public class BatoProvider implements ContentProvider, ManualImportContentProvider {
-  private static final String URL = "https://battwo.com";
-
-  private final FlareService flareService;
-
-  @Override
-  public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(
-      MangaContentProvider provider) {
-    try {
-      var document =
-          flareService.getContentAsJsoupDocument(provider.getUrl(), ContentProviders.BATO);
-
-      // Direct selector for chapter links
-      var chapterLinks = document.select("div.scrollable-panel a[href*=/title/]");
-
-      // TODO: fix chapter and language code
-      return chapterLinks.stream()
-          .map(
-              chapterLink ->
-                  new ContentProviderMangaChapterResponseDTO(
-                      chapterLink.text(), chapterLink.attr("href"), null, null))
-          .toList();
-    } catch (Exception e) {
-      log.warn(e.getMessage());
-      return null;
-    }
-  }
-
-  @Override
-  public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
-    try {
-      var document =
-          flareService.getContentAsJsoupDocument(
-              URL + chapterUrl + "?load=2", ContentProviders.BATO);
-
-      // Select all chapter page images
-      var imgElements = document.select("img.z-10.w-full.h-full");
-
-      List<String> imageUrls = new ArrayList<>();
-      for (var img : imgElements) {
-        String src = img.attr("src");
-
-        // Normalize if needed
-        if (!src.startsWith("http")) {
-          src = "https://battwo.com" + src;
-        }
-
-        imageUrls.add(src);
-      }
-
-      return IntStream.range(0, imageUrls.size())
-          .boxed()
-          .collect(
-              Collectors.toMap(
-                  i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new));
-
-    } catch (Exception e) {
-      log.warn(e.getMessage());
-      return null;
-    }
-  }
-
-  @Override
-  public String getMangaTitle(String value) {
-    var document = flareService.getContentAsJsoupDocument(value, ContentProviders.BATO);
-
-    var titleElement = document.selectFirst("h3 a[href*=/title/]");
-    if (isNull(titleElement)) {
-      titleElement = document.selectFirst("h3.item-title > a");
-
-      if (isNull(titleElement)) {
-        throw new UnprocessableException("Manga title not found for url: " + value);
-      }
-    }
-
-    return titleElement.text();
-  }
-}

src/main/java/com/magamochi/task/UpdateMangaDataTask.java

@@ -1,30 +0,0 @@
-package com.magamochi.task;
-
-import static java.util.Objects.isNull;
-
-import com.magamochi.model.dto.UpdateMangaDataCommand;
-import com.magamochi.model.repository.MangaRepository;
-import com.magamochi.queue.UpdateMangaDataProducer;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-@Log4j2
-@Component
-@RequiredArgsConstructor
-public class UpdateMangaDataTask {
-  private final MangaRepository mangaRepository;
-  private final UpdateMangaDataProducer updateMangaDataProducer;
-
-  @Scheduled(cron = "@daily")
-  public void updateMangaData() {
-    var mangas =
-        mangaRepository.findAll().stream().filter(manga -> isNull(manga.getScore())).toList();
-
-    mangas.forEach(
-        manga ->
-            updateMangaDataProducer.sendUpdateMangaDataCommand(
-                new UpdateMangaDataCommand(manga.getId())));
-  }
-}

src/main/java/com/magamochi/task/UpdateMangaListTask.java

@@ -1,67 +0,0 @@
-package com.magamochi.task;
-
-import com.magamochi.common.exception.NotFoundException;
-import com.magamochi.ingestion.model.repository.ContentProviderRepository;
-import com.magamochi.model.dto.MangaListUpdateCommand;
-import com.magamochi.queue.UpdateMangaListProducer;
-import com.magamochi.service.providers.PagedContentProvider;
-import com.magamochi.service.providers.PagedContentProviderFactory;
-import java.util.stream.IntStream;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-@Log4j2
-@Component
-@RequiredArgsConstructor
-public class UpdateMangaListTask {
-  @Value("${content-providers.update-enabled}")
-  private Boolean updateEnabled;
-
-  private final PagedContentProviderFactory contentProviderFactory;
-  private final UpdateMangaListProducer updateMangaListProducer;
-  private final ContentProviderRepository contentProviderRepository;
-
-  @Scheduled(cron = "${content-providers.cron-expression}")
-  public void updateMangaListScheduled() {
-    if (!updateEnabled) {
-      return;
-    }
-
-    updateMangaList();
-  }
-
-  public void updateMangaList() {
-    log.info("Queuing manga list updates...");
-
-    var contentProviders = contentProviderFactory.getContentProviders();
-    contentProviders.forEach(this::updateProviderMangaList);
-  }
-
-  public void updateProviderMangaList(Long providerId) {
-    var provider =
-        contentProviderRepository
-            .findById(providerId)
-            .orElseThrow(() -> new NotFoundException("Provider not found"));
-    var contentProvider = contentProviderFactory.getPagedContentProvider(provider.getName());
-
-    updateProviderMangaList(provider.getName(), contentProvider);
-  }
-
-  private void updateProviderMangaList(
-      String contentProviderName, PagedContentProvider contentProvider) {
-    log.info("Getting total pages for provider {}", contentProviderName);
-
-    var pages = contentProvider.getTotalPages();
-
-    IntStream.rangeClosed(1, pages)
-        .forEach(
-            page ->
-                updateMangaListProducer.sendUpdateMangaListCommand(
-                    new MangaListUpdateCommand(contentProviderName, page)));
-
-    log.info("Manga list update queued for content provider {}.", contentProviderName);
-  }
-}

src/main/resources/application.yml

@@ -88,6 +88,10 @@ resilience4j:
           - java.io.IOException
           - java.net.SocketTimeoutException
 
+queues:
+  manga-ingest: ${MANGA_INGEST_QUEUE:mangaIngest}
+  provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:providerPageIngest}
+
 rabbit-mq:
   queues:
     manga-data-update: ${MANGA_DATA_UPDATE_QUEUE:mangaDataUpdateQueue}

src/main/resources/db/migration/V0001__CREATE_TABLES.sql

@@ -42,6 +42,7 @@ CREATE TABLE content_providers
 (
     id                     BIGSERIAL NOT NULL PRIMARY KEY,
     name                   VARCHAR   NOT NULL,
+    url                    VARCHAR   NOT NULL,
     active                 BOOLEAN   NOT NULL DEFAULT TRUE,
     supports_chapter_fetch BOOLEAN   NOT NULL DEFAULT TRUE,
     manual_import          BOOLEAN   NOT NULL DEFAULT FALSE,

src/main/resources/db/migration/V0002__CONTENT_PROVIDERS.sql

@@ -0,0 +1,4 @@
+INSERT INTO content_providers(name, url, active, supports_chapter_fetch, manual_import)
+VALUES ('Manga Livre Blog', 'https://mangalivre.blog', TRUE, TRUE, FALSE),
+       ('Manga Livre.to', 'https://mangalivre.to', TRUE, TRUE, FALSE),
+       ('Pink Rosa Scan', 'https://scanpinkrosa.blogspot.com', TRUE, TRUE, FALSE);