feat: integrate RabbitMQ for manga data updates and add rate limiting

.env

@@ -12,4 +12,9 @@ MANGAMATCHER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8000/match-title
 MANGADEX_USER=rocverde
 MANGADEX_PASS=!A3u8e4s0
 MANGADEX_CLIENT_ID=personal-client-3c21667a-6de3-4273-94c4-e6014690f128-68830913
-MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS
+MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS
+
+RABBITMQ_HOST=localhost
+RABBITMQ_PORT=5672
+RABBITMQ_USERNAME=guest
+RABBITMQ_PASSWORD=guest

pom.xml

@@ -121,6 +121,10 @@
 			<version>0.13.0</version>
 			<scope>runtime</scope>
 		</dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-amqp</artifactId>
+        </dependency>
 	</dependencies>
 
 	<build>

src/main/java/com/magamochi/mangamochi/MangamochiApplication.java

@@ -1,5 +1,6 @@
 package com.magamochi.mangamochi;
 
+import org.springframework.amqp.rabbit.annotation.EnableRabbit;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
@@ -8,6 +9,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
 @SpringBootApplication
 @EnableFeignClients
 @EnableScheduling
+@EnableRabbit
 public class MangamochiApplication {
 
   public static void main(String[] args) {

src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java

@@ -0,0 +1,31 @@
+package com.magamochi.mangamochi.config;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.rabbit.connection.ConnectionFactory;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class RabbitConfig {
+  public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
+
+  @Bean
+  public Queue mangaDataUpdateQueue() {
+    return new Queue(MANGA_DATA_UPDATE_QUEUE, false);
+  }
+
+  @Bean
+  public Jackson2JsonMessageConverter messageConverter() {
+    return new Jackson2JsonMessageConverter();
+  }
+
+  @Bean
+  public RabbitTemplate rabbitTemplate(ConnectionFactory connectionFactory) {
+    var rabbitTemplate = new RabbitTemplate(connectionFactory);
+    rabbitTemplate.setMessageConverter(messageConverter());
+
+    return rabbitTemplate;
+  }
+}

src/main/java/com/magamochi/mangamochi/config/RateLimiterConfig.java

@@ -0,0 +1,18 @@
+package com.magamochi.mangamochi.config;
+
+import com.google.common.util.concurrent.RateLimiter;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class RateLimiterConfig {
+  @Bean
+  public RateLimiter mangaDexRateLimiter() {
+    return RateLimiter.create(3);
+  }
+
+  @Bean
+  public RateLimiter jikanRateLimiter() {
+    return RateLimiter.create(1);
+  }
+}

src/main/java/com/magamochi/mangamochi/controller/DevController.java

@@ -0,0 +1,23 @@
+package com.magamochi.mangamochi.controller;
+
+import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
+import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
+import lombok.RequiredArgsConstructor;
+import org.springframework.web.bind.annotation.*;
+
+@RestController
+@RequestMapping("/records")
+@RequiredArgsConstructor
+public class DevController {
+  private final UpdateMangaDataProducer producer;
+
+  @PostMapping
+  public String sendRecord(@RequestBody UpdateMangaDataCommand command) {
+    try {
+      producer.sendUpdateMangaDataCommand(command);
+    } catch (Exception e) {
+      return e.getMessage();
+    }
+    return "Command sent to RabbitMQ: " + command.mangaId();
+  }
+}

src/main/java/com/magamochi/mangamochi/model/dto/UpdateMangaDataCommand.java

@@ -0,0 +1,3 @@
+package com.magamochi.mangamochi.model.dto;
+
+public record UpdateMangaDataCommand(Long mangaId) {}

src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataConsumer.java

@@ -0,0 +1,22 @@
+package com.magamochi.mangamochi.queue;
+
+import com.magamochi.mangamochi.config.RabbitConfig;
+import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
+import com.magamochi.mangamochi.service.MangaImportService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class UpdateMangaDataConsumer {
+  private final MangaImportService mangaImportService;
+
+  @RabbitListener(queues = RabbitConfig.MANGA_DATA_UPDATE_QUEUE)
+  public void receiveUpdateMangaDataCommand(UpdateMangaDataCommand command) {
+    log.info("Received update manga data command: {}", command);
+    mangaImportService.updateMangaData(command.mangaId());
+  }
+}

src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataProducer.java

@@ -0,0 +1,20 @@
+package com.magamochi.mangamochi.queue;
+
+import com.magamochi.mangamochi.config.RabbitConfig;
+import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class UpdateMangaDataProducer {
+  private final RabbitTemplate rabbitTemplate;
+
+  public void sendUpdateMangaDataCommand(UpdateMangaDataCommand command) {
+    rabbitTemplate.convertAndSend(RabbitConfig.MANGA_DATA_UPDATE_QUEUE, command);
+    log.info("Sent update manga data command: {}", command);
+  }
+}

src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java

@@ -1,5 +1,6 @@
 package com.magamochi.mangamochi.service;
 
+import com.google.common.util.concurrent.RateLimiter;
 import com.magamochi.mangamochi.client.JikanClient;
 import com.magamochi.mangamochi.client.RapidFuzzClient;
 import com.magamochi.mangamochi.model.entity.Manga;
@@ -21,12 +22,15 @@ public class MangaCreationService {
   private final JikanClient jikanClient;
   private final RapidFuzzClient rapidFuzzClient;
 
+  private final RateLimiter jikanRateLimiter;
+
   public Manga getOrCreateManga(String title, String url, Provider provider) {
     var existingManga = mangaRepository.findByTitleIgnoreCase(title);
     if (existingManga.isPresent()) {
       return existingManga.get();
     }
 
+    jikanRateLimiter.acquire();
     var jikanResults = jikanClient.mangaSearch(title).data();
     if (jikanResults.isEmpty()) {
       createMangaImportReview(title, url, provider);

src/main/java/com/magamochi/mangamochi/service/MangaImportReviewService.java

@@ -25,7 +25,7 @@ public class MangaImportReviewService {
 
   private final JikanClient jikanClient;
 
-  RateLimiter rateLimiter = RateLimiter.create(1);
+  private final RateLimiter jikanRateLimiter;
 
   public List<ImportReviewDTO> getImportReviews() {
     return mangaImportReviewRepository.findAll().stream().map(ImportReviewDTO::from).toList();
@@ -40,7 +40,7 @@ public class MangaImportReviewService {
   public void resolveImportReview(Long id, String malId) {
     var importReview = getImportReviewThrowIfNotFound(id);
 
-    rateLimiter.acquire();
+    jikanRateLimiter.acquire();
     var jikanResult = jikanClient.getMangaById(Long.parseLong(malId)).data();
 
     if (isNull(jikanResult)) {

src/main/java/com/magamochi/mangamochi/service/MangaImportService.java

@@ -5,6 +5,7 @@ import static java.util.Objects.nonNull;
 
 import com.google.common.util.concurrent.RateLimiter;
 import com.magamochi.mangamochi.client.JikanClient;
+import com.magamochi.mangamochi.exception.NotFoundException;
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
 import com.magamochi.mangamochi.model.entity.*;
 import com.magamochi.mangamochi.model.repository.*;
@@ -44,13 +45,13 @@ public class MangaImportService {
   private final MangaChapterImageRepository mangaChapterImageRepository;
   private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository;
 
-  RateLimiter rateLimiter = RateLimiter.create(1);
+  private final RateLimiter jikanRateLimiter;
 
   public void importMangaFiles(String malId, List<MultipartFile> files) {
     log.info("Importing manga files for MAL ID {}", malId);
     var provider = providerService.getOrCreateProvider("Manual Import", false);
 
-    rateLimiter.acquire();
+    jikanRateLimiter.acquire();
     var mangaData = jikanClient.getMangaById(Long.parseLong(malId));
 
     var mangaProvider = getOrCreateMangaProvider(mangaData.data().title(), provider);
@@ -119,11 +120,20 @@ public class MangaImportService {
     log.info("Import manga files for MAL ID {} completed.", malId);
   }
 
+  public void updateMangaData(Long mangaId) {
+    var manga =
+        mangaRepository
+            .findById(mangaId)
+            .orElseThrow(() -> new NotFoundException("Manga not found for ID: " + mangaId));
+
+    updateMangaData(manga);
+  }
+
   public void updateMangaData(Manga manga) {
     log.info("Updating manga {}", manga.getTitle());
 
     try {
-      rateLimiter.acquire();
+      jikanRateLimiter.acquire();
       var mangaData = jikanClient.getMangaById(manga.getMalId());
 
       manga.setSynopsis(mangaData.data().synopsis());
@@ -248,7 +258,7 @@ public class MangaImportService {
         .findByMangaTitleIgnoreCaseAndProvider(title, provider)
         .orElseGet(
             () -> {
-              rateLimiter.acquire();
+              jikanRateLimiter.acquire();
               var manga = mangaCreationService.getOrCreateManga(title, "manual", provider);
 
               return mangaProviderRepository.save(

src/main/java/com/magamochi/mangamochi/service/MangaListService.java

@@ -2,7 +2,6 @@ package com.magamochi.mangamochi.service;
 
 import static java.util.Objects.isNull;
 
-import com.google.common.util.concurrent.RateLimiter;
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.mangamochi.model.entity.MangaProvider;
 import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
@@ -22,8 +21,6 @@ public class MangaListService {
 
   public void updateMangaList(
       String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) {
-    var rateLimiter = RateLimiter.create(1);
-
     var provider = providerService.getOrCreateProvider(contentProviderName);
 
     mangaInfoResponseDTOs.forEach(
@@ -36,7 +33,6 @@ public class MangaListService {
             return;
           }
 
-          rateLimiter.acquire();
           var manga =
               mangaCreationService.getOrCreateManga(
                   mangaResponse.title(), mangaResponse.url(), provider);

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java

@@ -9,11 +9,13 @@ import com.magamochi.mangamochi.exception.UnprocessableException;
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
 import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
+import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
 import com.magamochi.mangamochi.model.entity.MangaProvider;
 import com.magamochi.mangamochi.model.entity.Provider;
 import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
 import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
 import com.magamochi.mangamochi.model.repository.ProviderRepository;
+import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
 import com.magamochi.mangamochi.service.MangaCreationService;
 import com.magamochi.mangamochi.service.providers.ContentProvider;
 import com.magamochi.mangamochi.service.providers.ContentProviders;
@@ -32,8 +34,9 @@ public class MangaDexProvider implements ContentProvider {
   private final MangaCreationService mangaCreationService;
   private final ProviderRepository providerRepository;
   private final MangaProviderRepository mangaProviderRepository;
+  private final UpdateMangaDataProducer updateMangaDataProducer;
 
-  RateLimiter rateLimiter = RateLimiter.create(3);
+  private final RateLimiter mangaDexRateLimiter;
 
   @Override
   public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
@@ -46,7 +49,7 @@ public class MangaDexProvider implements ContentProvider {
   @Override
   public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
     try {
-      rateLimiter.acquire();
+      mangaDexRateLimiter.acquire();
       var response = mangaDexClient.getMangaFeed(UUID.fromString(provider.getUrl()));
 
       var mangas = new ArrayList<>(response.data());
@@ -58,7 +61,7 @@ public class MangaDexProvider implements ContentProvider {
             .parallel()
             .forEach(
                 i -> {
-                  rateLimiter.acquire();
+                  mangaDexRateLimiter.acquire();
 
                   var pagedResponse =
                       mangaDexClient.getMangaFeed(UUID.fromString(provider.getUrl()), 500, i * 500);
@@ -103,7 +106,7 @@ public class MangaDexProvider implements ContentProvider {
 
   @Override
   public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
-    rateLimiter.acquire();
+    mangaDexRateLimiter.acquire();
     var chapter = mangaDexClient.getMangaChapter(UUID.fromString(chapterUrl));
 
     var chapterImageHashes =
@@ -122,7 +125,7 @@ public class MangaDexProvider implements ContentProvider {
   }
 
   public ImportMangaDexResponseDTO importManga(UUID id) {
-    rateLimiter.acquire();
+    mangaDexRateLimiter.acquire();
     var resultData = mangaDexClient.getManga(id).data();
 
     if (resultData.attributes().title().isEmpty()) {
@@ -161,6 +164,8 @@ public class MangaDexProvider implements ContentProvider {
             .url(id.toString())
             .build());
 
+    updateMangaDataProducer.sendUpdateMangaDataCommand(new UpdateMangaDataCommand(manga.getId()));
+
     return new ImportMangaDexResponseDTO(manga.getId());
   }
 }

src/main/java/com/magamochi/mangamochi/task/UpdateMangaDataTask.java

@@ -2,8 +2,9 @@ package com.magamochi.mangamochi.task;
 
 import static java.util.Objects.isNull;
 
+import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
 import com.magamochi.mangamochi.model.repository.*;
-import com.magamochi.mangamochi.service.MangaImportService;
+import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -14,13 +15,16 @@ import org.springframework.stereotype.Component;
 @RequiredArgsConstructor
 public class UpdateMangaDataTask {
   private final MangaRepository mangaRepository;
-  private final MangaImportService mangaImportService;
+  private final UpdateMangaDataProducer updateMangaDataProducer;
 
-  @Scheduled(fixedDelayString = "1d")
+  @Scheduled(cron = "@daily")
   public void updateMangaData() {
     var mangas =
         mangaRepository.findAll().stream().filter(manga -> isNull(manga.getScore())).toList();
 
-    mangas.forEach(mangaImportService::updateMangaData);
+    mangas.forEach(
+        manga ->
+            updateMangaDataProducer.sendUpdateMangaDataCommand(
+                new UpdateMangaDataCommand(manga.getId())));
   }
 }

src/main/resources/application.yml

@@ -25,6 +25,11 @@ spring:
           web-scrapper:
             connect-timeout: 120000
             read-timeout: 120000
+  rabbitmq:
+    host: ${RABBITMQ_HOST}
+    port: ${RABBITMQ_PORT}
+    username: ${RABBITMQ_USERNAME}
+    password: ${RABBITMQ_PASSWORD}
 
 springdoc:
   api-docs: