Merge pull request 'feat: implement queued manga list update' (#14) from feature/queue-mange-list-update into main

Reviewed-on: #14
This commit is contained in:
rov 2025-11-08 21:10:40 -03:00
commit 0fa47cfacf
19 changed files with 245 additions and 133 deletions

View File

@ -11,6 +11,7 @@ import org.springframework.context.annotation.Configuration;
public class RabbitConfig { public class RabbitConfig {
public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue"; public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue"; public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue";
public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue";
@Bean @Bean
public Queue mangaDataUpdateQueue() { public Queue mangaDataUpdateQueue() {
@ -22,6 +23,11 @@ public class RabbitConfig {
return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false); return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false);
} }
@Bean
public Queue mangaListUpdateQueue() {
return new Queue(MANGA_LIST_UPDATE_QUEUE, false);
}
@Bean @Bean
public Jackson2JsonMessageConverter messageConverter() { public Jackson2JsonMessageConverter messageConverter() {
return new Jackson2JsonMessageConverter(); return new Jackson2JsonMessageConverter();

View File

@ -1,23 +0,0 @@
package com.magamochi.mangamochi.controller;
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/records")
@RequiredArgsConstructor
public class DevController {
private final UpdateMangaDataProducer producer;
@PostMapping
public String sendRecord(@RequestBody UpdateMangaDataCommand command) {
try {
producer.sendUpdateMangaDataCommand(command);
} catch (Exception e) {
return e.getMessage();
}
return "Command sent to RabbitMQ: " + command.mangaId();
}
}

View File

@ -0,0 +1,40 @@
package com.magamochi.mangamochi.controller;
import com.magamochi.mangamochi.model.dto.DefaultResponseDTO;
import com.magamochi.mangamochi.task.ImageCleanupTask;
import com.magamochi.mangamochi.task.UpdateMangaListTask;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/management")
@RequiredArgsConstructor
public class ManagementController {
private final UpdateMangaListTask updateMangaListTask;
private final ImageCleanupTask imageCleanupTask;
@Operation(
summary = "Queue update manga list",
description = "Queue the retrieval of the manga lists from the content providers",
tags = {"Management"},
operationId = "updateMangaList")
@PostMapping("update-manga-list")
public DefaultResponseDTO<Void> updateMangaList() {
updateMangaListTask.updateMangaList();
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Cleanup unused S3 images",
description = "Triggers the cleanup of untracked S3 images",
tags = {"Management"},
operationId = "imageCleanup")
@PostMapping("image-cleanup")
public DefaultResponseDTO<Void> imageCleanup() {
imageCleanupTask.cleanupImages();
return DefaultResponseDTO.ok().build();
}
}

View File

@ -0,0 +1,3 @@
package com.magamochi.mangamochi.model.dto;
public record MangaListUpdateCommand(String contentProviderName, Integer page) {}

View File

@ -3,4 +3,6 @@ package com.magamochi.mangamochi.model.repository;
import com.magamochi.mangamochi.model.entity.MangaImportReview; import com.magamochi.mangamochi.model.entity.MangaImportReview;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
public interface MangaImportReviewRepository extends JpaRepository<MangaImportReview, Long> {} public interface MangaImportReviewRepository extends JpaRepository<MangaImportReview, Long> {
boolean existsByTitleIgnoreCaseAndUrlIgnoreCase(String title, String url);
}

View File

@ -0,0 +1,22 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
import com.magamochi.mangamochi.service.MangaListService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class UpdateMangaListConsumer {
private final MangaListService mangaListService;
@RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE)
public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) {
log.info("Received update manga list command: {}", command);
mangaListService.updateMangaList(command.contentProviderName(), command.page());
}
}

View File

@ -0,0 +1,20 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class UpdateMangaListProducer {
private final RabbitTemplate rabbitTemplate;
public void sendUpdateMangaListCommand(MangaListUpdateCommand command) {
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command);
log.info("Sent update manga list command: {}", command);
}
}

View File

@ -91,6 +91,10 @@ public class MangaCreationService {
} }
private void createMangaImportReview(String title, String url, Provider provider) { private void createMangaImportReview(String title, String url, Provider provider) {
if (!mangaImportReviewRepository.existsByTitleIgnoreCaseAndUrlIgnoreCase(title, url)) {
return;
}
mangaImportReviewRepository.save( mangaImportReviewRepository.save(
MangaImportReview.builder().title(title).url(url).provider(provider).build()); MangaImportReview.builder().title(title).url(url).provider(provider).build());
} }

View File

@ -2,10 +2,9 @@ package com.magamochi.mangamochi.service;
import static java.util.Objects.isNull; import static java.util.Objects.isNull;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.mangamochi.model.entity.MangaProvider; import com.magamochi.mangamochi.model.entity.MangaProvider;
import com.magamochi.mangamochi.model.repository.MangaProviderRepository; import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
import java.util.List; import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -16,14 +15,17 @@ import org.springframework.stereotype.Service;
public class MangaListService { public class MangaListService {
private final ProviderService providerService; private final ProviderService providerService;
private final MangaCreationService mangaCreationService; private final MangaCreationService mangaCreationService;
private final PagedContentProviderFactory pagedContentProviderFactory;
private final MangaProviderRepository mangaProviderRepository; private final MangaProviderRepository mangaProviderRepository;
public void updateMangaList( public void updateMangaList(String contentProviderName, Integer page) {
String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) { var contentProvider = pagedContentProviderFactory.getPagedContentProvider(contentProviderName);
var provider = providerService.getOrCreateProvider(contentProviderName); var provider = providerService.getOrCreateProvider(contentProviderName);
mangaInfoResponseDTOs.forEach( var mangas = contentProvider.getMangasFromPage(page);
mangas.forEach(
mangaResponse -> { mangaResponse -> {
var mangaProvider = var mangaProvider =
mangaProviderRepository.findByMangaTitleIgnoreCaseAndProvider( mangaProviderRepository.findByMangaTitleIgnoreCaseAndProvider(

View File

@ -1,14 +1,11 @@
package com.magamochi.mangamochi.service.providers; package com.magamochi.mangamochi.service.providers;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.mangamochi.model.entity.MangaProvider; import com.magamochi.mangamochi.model.entity.MangaProvider;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
public interface ContentProvider { public interface ContentProvider {
List<ContentProviderMangaInfoResponseDTO> getAvailableMangas();
List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider); List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider);
Map<Integer, String> getChapterImagesUrls(String chapterUrl); Map<Integer, String> getChapterImagesUrls(String chapterUrl);

View File

@ -0,0 +1,10 @@
package com.magamochi.mangamochi.service.providers;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import java.util.List;
public interface PagedContentProvider {
Integer getTotalPages();
List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page);
}

View File

@ -0,0 +1,24 @@
package com.magamochi.mangamochi.service.providers;
import java.util.Map;
import java.util.Objects;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
@Getter
@Component
@RequiredArgsConstructor
public class PagedContentProviderFactory {
private final Map<String, PagedContentProvider> contentProviders;
public PagedContentProvider getPagedContentProvider(String providerName) {
var provider = contentProviders.get(providerName);
if (Objects.isNull(provider)) {
throw new IllegalArgumentException("No such provider " + providerName);
}
return provider;
}
}

View File

@ -7,7 +7,6 @@ import com.magamochi.mangamochi.client.MangaDexClient;
import com.magamochi.mangamochi.exception.NotFoundException; import com.magamochi.mangamochi.exception.NotFoundException;
import com.magamochi.mangamochi.exception.UnprocessableException; import com.magamochi.mangamochi.exception.UnprocessableException;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO; import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
import com.magamochi.mangamochi.model.entity.MangaProvider; import com.magamochi.mangamochi.model.entity.MangaProvider;
import com.magamochi.mangamochi.model.entity.Provider; import com.magamochi.mangamochi.model.entity.Provider;
@ -35,14 +34,6 @@ public class MangaDexProvider implements ContentProvider {
private final RateLimiter mangaDexRateLimiter; private final RateLimiter mangaDexRateLimiter;
@Override
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
// MangaDex API does not provide an endpoint to list all mangas directly.
// As there is lots and lots of mangas, this is not feasible to implement here.
// The frontend has a function to import mangas by their IDs instead.
return List.of();
}
@Override @Override
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) { public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
try { try {

View File

@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
import com.magamochi.mangamochi.service.WebScrapperClientProxyService; import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.service.providers.ContentProvider;
import com.magamochi.mangamochi.service.providers.ContentProviders; import com.magamochi.mangamochi.service.providers.ContentProviders;
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -21,28 +22,13 @@ import org.springframework.stereotype.Service;
@Log4j2 @Log4j2
@Service(ContentProviders.MANGA_LIVRE_BLOG) @Service(ContentProviders.MANGA_LIVRE_BLOG)
@RequiredArgsConstructor @RequiredArgsConstructor
public class MangaLivreBlogProvider implements ContentProvider { public class MangaLivreBlogProvider implements ContentProvider, PagedContentProvider {
private static final Pattern NUMERIC_PATTERN = Pattern.compile("-?\\d+"); private static final Pattern NUMERIC_PATTERN = Pattern.compile("-?\\d+");
private final String url = "https://mangalivre.blog/manga/"; private final String url = "https://mangalivre.blog/manga/";
private final WebScrapperClientProxyService webScrapperClientProxyService; private final WebScrapperClientProxyService webScrapperClientProxyService;
@Override
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
var totalPages = getTotalPages();
if (Objects.isNull(totalPages) || totalPages < 1) {
return List.of();
}
return IntStream.rangeClosed(1, totalPages)
.mapToObj(this::getMangasFromPage)
.filter(Objects::nonNull)
.flatMap(List::stream)
.toList();
}
@Override @Override
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters( public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(
MangaProvider mangaProvider) { MangaProvider mangaProvider) {
@ -109,7 +95,8 @@ public class MangaLivreBlogProvider implements ContentProvider {
} }
} }
private List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(int page) { @Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
try { try {
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page); var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
@ -159,7 +146,8 @@ public class MangaLivreBlogProvider implements ContentProvider {
} }
} }
private Integer getTotalPages() { @Override
public Integer getTotalPages() {
try { try {
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url); var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);

View File

@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
import com.magamochi.mangamochi.service.WebScrapperClientProxyService; import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.service.providers.ContentProvider;
import com.magamochi.mangamochi.service.providers.ContentProviders; import com.magamochi.mangamochi.service.providers.ContentProviders;
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -18,26 +19,11 @@ import org.springframework.stereotype.Service;
@Log4j2 @Log4j2
@Service(ContentProviders.MANGA_LIVRE) @Service(ContentProviders.MANGA_LIVRE)
@RequiredArgsConstructor @RequiredArgsConstructor
public class MangaLivreProvider implements ContentProvider { public class MangaLivreProvider implements ContentProvider, PagedContentProvider {
private final String url = "https://mangalivre.tv/manga/"; private final String url = "https://mangalivre.tv/manga/";
private final WebScrapperClientProxyService webScrapperClientProxyService; private final WebScrapperClientProxyService webScrapperClientProxyService;
@Override
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
var totalPages = getTotalPages();
if (Objects.isNull(totalPages) || totalPages < 1) {
return List.of();
}
return IntStream.rangeClosed(1, totalPages)
.mapToObj(this::getMangasFromPage)
.filter(Objects::nonNull)
.flatMap(List::stream)
.toList();
}
@Override @Override
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) { public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
try { try {
@ -88,7 +74,8 @@ public class MangaLivreProvider implements ContentProvider {
} }
} }
private List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(int page) { @Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
try { try {
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page); var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
@ -134,7 +121,8 @@ public class MangaLivreProvider implements ContentProvider {
} }
} }
private Integer getTotalPages() { @Override
public Integer getTotalPages() {
try { try {
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url); var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);

View File

@ -10,6 +10,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
import com.magamochi.mangamochi.service.WebScrapperClientProxyService; import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.service.providers.ContentProvider;
import com.magamochi.mangamochi.service.providers.ContentProviders; import com.magamochi.mangamochi.service.providers.ContentProviders;
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -21,47 +22,9 @@ import org.springframework.stereotype.Service;
@Log4j2 @Log4j2
@Service(ContentProviders.PINK_ROSA_SCAN) @Service(ContentProviders.PINK_ROSA_SCAN)
@RequiredArgsConstructor @RequiredArgsConstructor
public class PinkRosaScanProvider implements ContentProvider { public class PinkRosaScanProvider implements ContentProvider, PagedContentProvider {
private final WebScrapperClientProxyService webScrapperClientProxyService; private final WebScrapperClientProxyService webScrapperClientProxyService;
@Override
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
try {
var document =
webScrapperClientProxyService.scrapeToJsoupDocument(
"https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000");
var mangaElements =
document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit");
return mangaElements.stream()
.map(
element -> {
var linkElement =
element
.getElementsByClass(
"flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw] w-full")
.getFirst()
.getElementsByTag("div")
.getFirst()
.getElementsByTag("a")
.getFirst();
var url = linkElement.attr("href");
var textElement = linkElement.getElementsByTag("h3");
var title = textElement.text().trim();
return new ContentProviderMangaInfoResponseDTO(
title, url, null, MangaStatus.UNKNOWN);
})
.toList();
} catch (NoSuchElementException | IOException e) {
log.error("Error parsing mangas from Pink Rosa Scan", e);
return List.of();
}
}
@Override @Override
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) { public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
try { try {
@ -129,4 +92,47 @@ public class PinkRosaScanProvider implements ContentProvider {
return Map.of(); return Map.of();
} }
} }
@Override
public Integer getTotalPages() {
return 1;
}
@Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
try {
var document =
webScrapperClientProxyService.scrapeToJsoupDocument(
"https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000");
var mangaElements =
document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit");
return mangaElements.stream()
.map(
element -> {
var linkElement =
element
.getElementsByClass(
"flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw] w-full")
.getFirst()
.getElementsByTag("div")
.getFirst()
.getElementsByTag("a")
.getFirst();
var url = linkElement.attr("href");
var textElement = linkElement.getElementsByTag("h3");
var title = textElement.text().trim();
return new ContentProviderMangaInfoResponseDTO(
title, url, null, MangaStatus.UNKNOWN);
})
.toList();
} catch (NoSuchElementException | IOException e) {
log.error("Error parsing mangas from Pink Rosa Scan", e);
return List.of();
}
}
} }

View File

@ -20,13 +20,17 @@ public class ImageCleanupTask {
private final S3Service s3Service; private final S3Service s3Service;
private final ImageRepository imageRepository; private final ImageRepository imageRepository;
@Scheduled(cron = "@weekly") @Scheduled(cron = "${image-service.cron-expression}")
public void cleanupImages() { public void cleanUpImagesScheduled() {
if (!cleanUpEnabled) { if (!cleanUpEnabled) {
log.info("S3 Image cleanup disabled."); log.info("S3 Image cleanup disabled.");
return; return;
} }
cleanupImages();
}
public void cleanupImages() {
log.info("Getting unused S3 object keys to remove."); log.info("Getting unused S3 object keys to remove.");
var imageKeys = s3Service.listAllObjectKeys(); var imageKeys = s3Service.listAllObjectKeys();

View File

@ -1,33 +1,55 @@
package com.magamochi.mangamochi.task; package com.magamochi.mangamochi.task;
import com.magamochi.mangamochi.service.MangaListService; import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
import com.magamochi.mangamochi.service.providers.ContentProvider; import com.magamochi.mangamochi.queue.UpdateMangaListProducer;
import com.magamochi.mangamochi.service.providers.ContentProviderFactory; import com.magamochi.mangamochi.service.providers.PagedContentProvider;
import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Log4j2 @Log4j2
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class UpdateMangaListTask { public class UpdateMangaListTask {
private final ContentProviderFactory contentProviderFactory; @Value("${content-providers.update-enabled}")
private final MangaListService mangaListService; private Boolean updateEnabled;
private final PagedContentProviderFactory contentProviderFactory;
private final UpdateMangaListProducer updateMangaListProducer;
@Scheduled(cron = "${content-providers.cron-expression}")
public void updateMangaListScheduled() {
if (!updateEnabled) {
return;
}
updateMangaList();
}
// @Scheduled(fixedDelayString = "1d")
public void updateMangaList() { public void updateMangaList() {
log.info("Updating manga list..."); log.info("Queuing manga list updates...");
var contentProviders = contentProviderFactory.getContentProviders(); var contentProviders = contentProviderFactory.getContentProviders();
contentProviders.forEach(this::updateProviderMangaList); contentProviders.forEach(this::updateProviderMangaList);
log.info("Manga list updated.");
} }
private void updateProviderMangaList( private void updateProviderMangaList(
String contentProviderName, ContentProvider contentProvider) { String contentProviderName, PagedContentProvider contentProvider) {
log.info("Updating manga list for content provider {}", contentProviderName); log.info("Getting total pages for provider {}", contentProviderName);
mangaListService.updateMangaList(contentProviderName, contentProvider.getAvailableMangas());
log.info("Manga list for content provider {} updated.", contentProviderName); var pages = contentProvider.getTotalPages();
IntStream.rangeClosed(1, pages)
.forEach(
page -> {
updateMangaListProducer.sendUpdateMangaListCommand(
new MangaListUpdateCommand(contentProviderName, page));
});
log.info("Manga list update queued for content provider {}.", contentProviderName);
} }
} }

View File

@ -23,8 +23,8 @@ spring:
client: client:
config: config:
web-scrapper: web-scrapper:
connect-timeout: 120000 connect-timeout: 240000
read-timeout: 120000 read-timeout: 240000
rabbitmq: rabbitmq:
host: ${RABBITMQ_HOST} host: ${RABBITMQ_HOST}
port: ${RABBITMQ_PORT} port: ${RABBITMQ_PORT}
@ -71,3 +71,9 @@ resilience4j:
image-service: image-service:
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false} clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
cron-expression: "@weekly"
content-providers:
update-enabled: ${CONTENT_PROVIDER_UPDATE_ENABLED:false}
cron-expression: "@weekly"