Merge pull request 'feat: implement queued manga list update' (#14) from feature/queue-mange-list-update into main
Reviewed-on: #14
This commit is contained in:
commit
0fa47cfacf
@ -11,6 +11,7 @@ import org.springframework.context.annotation.Configuration;
|
|||||||
public class RabbitConfig {
|
public class RabbitConfig {
|
||||||
public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
|
public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
|
||||||
public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue";
|
public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue";
|
||||||
|
public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue";
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
public Queue mangaDataUpdateQueue() {
|
public Queue mangaDataUpdateQueue() {
|
||||||
@ -22,6 +23,11 @@ public class RabbitConfig {
|
|||||||
return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false);
|
return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public Queue mangaListUpdateQueue() {
|
||||||
|
return new Queue(MANGA_LIST_UPDATE_QUEUE, false);
|
||||||
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
public Jackson2JsonMessageConverter messageConverter() {
|
public Jackson2JsonMessageConverter messageConverter() {
|
||||||
return new Jackson2JsonMessageConverter();
|
return new Jackson2JsonMessageConverter();
|
||||||
|
|||||||
@ -1,23 +0,0 @@
|
|||||||
package com.magamochi.mangamochi.controller;
|
|
||||||
|
|
||||||
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
|
||||||
import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import org.springframework.web.bind.annotation.*;
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/records")
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class DevController {
|
|
||||||
private final UpdateMangaDataProducer producer;
|
|
||||||
|
|
||||||
@PostMapping
|
|
||||||
public String sendRecord(@RequestBody UpdateMangaDataCommand command) {
|
|
||||||
try {
|
|
||||||
producer.sendUpdateMangaDataCommand(command);
|
|
||||||
} catch (Exception e) {
|
|
||||||
return e.getMessage();
|
|
||||||
}
|
|
||||||
return "Command sent to RabbitMQ: " + command.mangaId();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
package com.magamochi.mangamochi.controller;
|
||||||
|
|
||||||
|
import com.magamochi.mangamochi.model.dto.DefaultResponseDTO;
|
||||||
|
import com.magamochi.mangamochi.task.ImageCleanupTask;
|
||||||
|
import com.magamochi.mangamochi.task.UpdateMangaListTask;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/management")
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class ManagementController {
|
||||||
|
private final UpdateMangaListTask updateMangaListTask;
|
||||||
|
private final ImageCleanupTask imageCleanupTask;
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Queue update manga list",
|
||||||
|
description = "Queue the retrieval of the manga lists from the content providers",
|
||||||
|
tags = {"Management"},
|
||||||
|
operationId = "updateMangaList")
|
||||||
|
@PostMapping("update-manga-list")
|
||||||
|
public DefaultResponseDTO<Void> updateMangaList() {
|
||||||
|
updateMangaListTask.updateMangaList();
|
||||||
|
|
||||||
|
return DefaultResponseDTO.ok().build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Cleanup unused S3 images",
|
||||||
|
description = "Triggers the cleanup of untracked S3 images",
|
||||||
|
tags = {"Management"},
|
||||||
|
operationId = "imageCleanup")
|
||||||
|
@PostMapping("image-cleanup")
|
||||||
|
public DefaultResponseDTO<Void> imageCleanup() {
|
||||||
|
imageCleanupTask.cleanupImages();
|
||||||
|
|
||||||
|
return DefaultResponseDTO.ok().build();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
package com.magamochi.mangamochi.model.dto;
|
||||||
|
|
||||||
|
public record MangaListUpdateCommand(String contentProviderName, Integer page) {}
|
||||||
@ -3,4 +3,6 @@ package com.magamochi.mangamochi.model.repository;
|
|||||||
import com.magamochi.mangamochi.model.entity.MangaImportReview;
|
import com.magamochi.mangamochi.model.entity.MangaImportReview;
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
public interface MangaImportReviewRepository extends JpaRepository<MangaImportReview, Long> {}
|
public interface MangaImportReviewRepository extends JpaRepository<MangaImportReview, Long> {
|
||||||
|
boolean existsByTitleIgnoreCaseAndUrlIgnoreCase(String title, String url);
|
||||||
|
}
|
||||||
|
|||||||
@ -0,0 +1,22 @@
|
|||||||
|
package com.magamochi.mangamochi.queue;
|
||||||
|
|
||||||
|
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||||
|
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
|
||||||
|
import com.magamochi.mangamochi.service.MangaListService;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class UpdateMangaListConsumer {
|
||||||
|
private final MangaListService mangaListService;
|
||||||
|
|
||||||
|
@RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE)
|
||||||
|
public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) {
|
||||||
|
log.info("Received update manga list command: {}", command);
|
||||||
|
mangaListService.updateMangaList(command.contentProviderName(), command.page());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
package com.magamochi.mangamochi.queue;
|
||||||
|
|
||||||
|
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||||
|
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class UpdateMangaListProducer {
|
||||||
|
private final RabbitTemplate rabbitTemplate;
|
||||||
|
|
||||||
|
public void sendUpdateMangaListCommand(MangaListUpdateCommand command) {
|
||||||
|
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command);
|
||||||
|
log.info("Sent update manga list command: {}", command);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -91,6 +91,10 @@ public class MangaCreationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void createMangaImportReview(String title, String url, Provider provider) {
|
private void createMangaImportReview(String title, String url, Provider provider) {
|
||||||
|
if (!mangaImportReviewRepository.existsByTitleIgnoreCaseAndUrlIgnoreCase(title, url)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
mangaImportReviewRepository.save(
|
mangaImportReviewRepository.save(
|
||||||
MangaImportReview.builder().title(title).url(url).provider(provider).build());
|
MangaImportReview.builder().title(title).url(url).provider(provider).build());
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,10 +2,9 @@ package com.magamochi.mangamochi.service;
|
|||||||
|
|
||||||
import static java.util.Objects.isNull;
|
import static java.util.Objects.isNull;
|
||||||
|
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
|
|
||||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||||
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
||||||
import java.util.List;
|
import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@ -16,14 +15,17 @@ import org.springframework.stereotype.Service;
|
|||||||
public class MangaListService {
|
public class MangaListService {
|
||||||
private final ProviderService providerService;
|
private final ProviderService providerService;
|
||||||
private final MangaCreationService mangaCreationService;
|
private final MangaCreationService mangaCreationService;
|
||||||
|
private final PagedContentProviderFactory pagedContentProviderFactory;
|
||||||
|
|
||||||
private final MangaProviderRepository mangaProviderRepository;
|
private final MangaProviderRepository mangaProviderRepository;
|
||||||
|
|
||||||
public void updateMangaList(
|
public void updateMangaList(String contentProviderName, Integer page) {
|
||||||
String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) {
|
var contentProvider = pagedContentProviderFactory.getPagedContentProvider(contentProviderName);
|
||||||
var provider = providerService.getOrCreateProvider(contentProviderName);
|
var provider = providerService.getOrCreateProvider(contentProviderName);
|
||||||
|
|
||||||
mangaInfoResponseDTOs.forEach(
|
var mangas = contentProvider.getMangasFromPage(page);
|
||||||
|
|
||||||
|
mangas.forEach(
|
||||||
mangaResponse -> {
|
mangaResponse -> {
|
||||||
var mangaProvider =
|
var mangaProvider =
|
||||||
mangaProviderRepository.findByMangaTitleIgnoreCaseAndProvider(
|
mangaProviderRepository.findByMangaTitleIgnoreCaseAndProvider(
|
||||||
|
|||||||
@ -1,14 +1,11 @@
|
|||||||
package com.magamochi.mangamochi.service.providers;
|
package com.magamochi.mangamochi.service.providers;
|
||||||
|
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
|
|
||||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public interface ContentProvider {
|
public interface ContentProvider {
|
||||||
List<ContentProviderMangaInfoResponseDTO> getAvailableMangas();
|
|
||||||
|
|
||||||
List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider);
|
List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider);
|
||||||
|
|
||||||
Map<Integer, String> getChapterImagesUrls(String chapterUrl);
|
Map<Integer, String> getChapterImagesUrls(String chapterUrl);
|
||||||
|
|||||||
@ -0,0 +1,10 @@
|
|||||||
|
package com.magamochi.mangamochi.service.providers;
|
||||||
|
|
||||||
|
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface PagedContentProvider {
|
||||||
|
Integer getTotalPages();
|
||||||
|
|
||||||
|
List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page);
|
||||||
|
}
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
package com.magamochi.mangamochi.service.providers;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class PagedContentProviderFactory {
|
||||||
|
private final Map<String, PagedContentProvider> contentProviders;
|
||||||
|
|
||||||
|
public PagedContentProvider getPagedContentProvider(String providerName) {
|
||||||
|
var provider = contentProviders.get(providerName);
|
||||||
|
|
||||||
|
if (Objects.isNull(provider)) {
|
||||||
|
throw new IllegalArgumentException("No such provider " + providerName);
|
||||||
|
}
|
||||||
|
|
||||||
|
return provider;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -7,7 +7,6 @@ import com.magamochi.mangamochi.client.MangaDexClient;
|
|||||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||||
import com.magamochi.mangamochi.exception.UnprocessableException;
|
import com.magamochi.mangamochi.exception.UnprocessableException;
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
|
|
||||||
import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
|
import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
|
||||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||||
import com.magamochi.mangamochi.model.entity.Provider;
|
import com.magamochi.mangamochi.model.entity.Provider;
|
||||||
@ -35,14 +34,6 @@ public class MangaDexProvider implements ContentProvider {
|
|||||||
|
|
||||||
private final RateLimiter mangaDexRateLimiter;
|
private final RateLimiter mangaDexRateLimiter;
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
|
|
||||||
// MangaDex API does not provide an endpoint to list all mangas directly.
|
|
||||||
// As there is lots and lots of mangas, this is not feasible to implement here.
|
|
||||||
// The frontend has a function to import mangas by their IDs instead.
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
|
|||||||
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
||||||
|
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@ -21,28 +22,13 @@ import org.springframework.stereotype.Service;
|
|||||||
@Log4j2
|
@Log4j2
|
||||||
@Service(ContentProviders.MANGA_LIVRE_BLOG)
|
@Service(ContentProviders.MANGA_LIVRE_BLOG)
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class MangaLivreBlogProvider implements ContentProvider {
|
public class MangaLivreBlogProvider implements ContentProvider, PagedContentProvider {
|
||||||
private static final Pattern NUMERIC_PATTERN = Pattern.compile("-?\\d+");
|
private static final Pattern NUMERIC_PATTERN = Pattern.compile("-?\\d+");
|
||||||
|
|
||||||
private final String url = "https://mangalivre.blog/manga/";
|
private final String url = "https://mangalivre.blog/manga/";
|
||||||
|
|
||||||
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
|
|
||||||
var totalPages = getTotalPages();
|
|
||||||
|
|
||||||
if (Objects.isNull(totalPages) || totalPages < 1) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
|
|
||||||
return IntStream.rangeClosed(1, totalPages)
|
|
||||||
.mapToObj(this::getMangasFromPage)
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.flatMap(List::stream)
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(
|
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(
|
||||||
MangaProvider mangaProvider) {
|
MangaProvider mangaProvider) {
|
||||||
@ -109,7 +95,8 @@ public class MangaLivreBlogProvider implements ContentProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(int page) {
|
@Override
|
||||||
|
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
|
||||||
try {
|
try {
|
||||||
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
|
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
|
||||||
|
|
||||||
@ -159,7 +146,8 @@ public class MangaLivreBlogProvider implements ContentProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Integer getTotalPages() {
|
@Override
|
||||||
|
public Integer getTotalPages() {
|
||||||
try {
|
try {
|
||||||
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
|
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
|
||||||
|
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
|
|||||||
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
||||||
|
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
@ -18,26 +19,11 @@ import org.springframework.stereotype.Service;
|
|||||||
@Log4j2
|
@Log4j2
|
||||||
@Service(ContentProviders.MANGA_LIVRE)
|
@Service(ContentProviders.MANGA_LIVRE)
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class MangaLivreProvider implements ContentProvider {
|
public class MangaLivreProvider implements ContentProvider, PagedContentProvider {
|
||||||
private final String url = "https://mangalivre.tv/manga/";
|
private final String url = "https://mangalivre.tv/manga/";
|
||||||
|
|
||||||
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
|
|
||||||
var totalPages = getTotalPages();
|
|
||||||
|
|
||||||
if (Objects.isNull(totalPages) || totalPages < 1) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
|
|
||||||
return IntStream.rangeClosed(1, totalPages)
|
|
||||||
.mapToObj(this::getMangasFromPage)
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.flatMap(List::stream)
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||||
try {
|
try {
|
||||||
@ -88,7 +74,8 @@ public class MangaLivreProvider implements ContentProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(int page) {
|
@Override
|
||||||
|
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
|
||||||
try {
|
try {
|
||||||
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
|
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url + "page/" + page);
|
||||||
|
|
||||||
@ -134,7 +121,8 @@ public class MangaLivreProvider implements ContentProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Integer getTotalPages() {
|
@Override
|
||||||
|
public Integer getTotalPages() {
|
||||||
try {
|
try {
|
||||||
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
|
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
|
||||||
|
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import com.magamochi.mangamochi.model.enumeration.MangaStatus;
|
|||||||
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
||||||
|
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
@ -21,47 +22,9 @@ import org.springframework.stereotype.Service;
|
|||||||
@Log4j2
|
@Log4j2
|
||||||
@Service(ContentProviders.PINK_ROSA_SCAN)
|
@Service(ContentProviders.PINK_ROSA_SCAN)
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class PinkRosaScanProvider implements ContentProvider {
|
public class PinkRosaScanProvider implements ContentProvider, PagedContentProvider {
|
||||||
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
|
|
||||||
try {
|
|
||||||
var document =
|
|
||||||
webScrapperClientProxyService.scrapeToJsoupDocument(
|
|
||||||
"https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000");
|
|
||||||
|
|
||||||
var mangaElements =
|
|
||||||
document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit");
|
|
||||||
|
|
||||||
return mangaElements.stream()
|
|
||||||
.map(
|
|
||||||
element -> {
|
|
||||||
var linkElement =
|
|
||||||
element
|
|
||||||
.getElementsByClass(
|
|
||||||
"flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw] w-full")
|
|
||||||
.getFirst()
|
|
||||||
.getElementsByTag("div")
|
|
||||||
.getFirst()
|
|
||||||
.getElementsByTag("a")
|
|
||||||
.getFirst();
|
|
||||||
|
|
||||||
var url = linkElement.attr("href");
|
|
||||||
|
|
||||||
var textElement = linkElement.getElementsByTag("h3");
|
|
||||||
var title = textElement.text().trim();
|
|
||||||
|
|
||||||
return new ContentProviderMangaInfoResponseDTO(
|
|
||||||
title, url, null, MangaStatus.UNKNOWN);
|
|
||||||
})
|
|
||||||
.toList();
|
|
||||||
} catch (NoSuchElementException | IOException e) {
|
|
||||||
log.error("Error parsing mangas from Pink Rosa Scan", e);
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||||
try {
|
try {
|
||||||
@ -129,4 +92,47 @@ public class PinkRosaScanProvider implements ContentProvider {
|
|||||||
return Map.of();
|
return Map.of();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Integer getTotalPages() {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
|
||||||
|
try {
|
||||||
|
var document =
|
||||||
|
webScrapperClientProxyService.scrapeToJsoupDocument(
|
||||||
|
"https://scanpinkrosa.blogspot.com/search/label/Series?max-results=1000");
|
||||||
|
|
||||||
|
var mangaElements =
|
||||||
|
document.getElementsByClass("grid relative sm:gap-3.5 gap-[2.5vw] w-full h-fit");
|
||||||
|
|
||||||
|
return mangaElements.stream()
|
||||||
|
.map(
|
||||||
|
element -> {
|
||||||
|
var linkElement =
|
||||||
|
element
|
||||||
|
.getElementsByClass(
|
||||||
|
"flex sm:gap-2.5 gap-[2vw] justify-start items-start sm:-mt-0.5 -mt-[0.5vw] w-full")
|
||||||
|
.getFirst()
|
||||||
|
.getElementsByTag("div")
|
||||||
|
.getFirst()
|
||||||
|
.getElementsByTag("a")
|
||||||
|
.getFirst();
|
||||||
|
|
||||||
|
var url = linkElement.attr("href");
|
||||||
|
|
||||||
|
var textElement = linkElement.getElementsByTag("h3");
|
||||||
|
var title = textElement.text().trim();
|
||||||
|
|
||||||
|
return new ContentProviderMangaInfoResponseDTO(
|
||||||
|
title, url, null, MangaStatus.UNKNOWN);
|
||||||
|
})
|
||||||
|
.toList();
|
||||||
|
} catch (NoSuchElementException | IOException e) {
|
||||||
|
log.error("Error parsing mangas from Pink Rosa Scan", e);
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -20,13 +20,17 @@ public class ImageCleanupTask {
|
|||||||
private final S3Service s3Service;
|
private final S3Service s3Service;
|
||||||
private final ImageRepository imageRepository;
|
private final ImageRepository imageRepository;
|
||||||
|
|
||||||
@Scheduled(cron = "@weekly")
|
@Scheduled(cron = "${image-service.cron-expression}")
|
||||||
public void cleanupImages() {
|
public void cleanUpImagesScheduled() {
|
||||||
if (!cleanUpEnabled) {
|
if (!cleanUpEnabled) {
|
||||||
log.info("S3 Image cleanup disabled.");
|
log.info("S3 Image cleanup disabled.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cleanupImages();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void cleanupImages() {
|
||||||
log.info("Getting unused S3 object keys to remove.");
|
log.info("Getting unused S3 object keys to remove.");
|
||||||
|
|
||||||
var imageKeys = s3Service.listAllObjectKeys();
|
var imageKeys = s3Service.listAllObjectKeys();
|
||||||
|
|||||||
@ -1,33 +1,55 @@
|
|||||||
package com.magamochi.mangamochi.task;
|
package com.magamochi.mangamochi.task;
|
||||||
|
|
||||||
import com.magamochi.mangamochi.service.MangaListService;
|
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
import com.magamochi.mangamochi.queue.UpdateMangaListProducer;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProviderFactory;
|
import com.magamochi.mangamochi.service.providers.PagedContentProvider;
|
||||||
|
import com.magamochi.mangamochi.service.providers.PagedContentProviderFactory;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Log4j2
|
@Log4j2
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class UpdateMangaListTask {
|
public class UpdateMangaListTask {
|
||||||
private final ContentProviderFactory contentProviderFactory;
|
@Value("${content-providers.update-enabled}")
|
||||||
private final MangaListService mangaListService;
|
private Boolean updateEnabled;
|
||||||
|
|
||||||
|
private final PagedContentProviderFactory contentProviderFactory;
|
||||||
|
private final UpdateMangaListProducer updateMangaListProducer;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${content-providers.cron-expression}")
|
||||||
|
public void updateMangaListScheduled() {
|
||||||
|
if (!updateEnabled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
updateMangaList();
|
||||||
|
}
|
||||||
|
|
||||||
// @Scheduled(fixedDelayString = "1d")
|
|
||||||
public void updateMangaList() {
|
public void updateMangaList() {
|
||||||
log.info("Updating manga list...");
|
log.info("Queuing manga list updates...");
|
||||||
|
|
||||||
var contentProviders = contentProviderFactory.getContentProviders();
|
var contentProviders = contentProviderFactory.getContentProviders();
|
||||||
contentProviders.forEach(this::updateProviderMangaList);
|
contentProviders.forEach(this::updateProviderMangaList);
|
||||||
|
|
||||||
log.info("Manga list updated.");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void updateProviderMangaList(
|
private void updateProviderMangaList(
|
||||||
String contentProviderName, ContentProvider contentProvider) {
|
String contentProviderName, PagedContentProvider contentProvider) {
|
||||||
log.info("Updating manga list for content provider {}", contentProviderName);
|
log.info("Getting total pages for provider {}", contentProviderName);
|
||||||
mangaListService.updateMangaList(contentProviderName, contentProvider.getAvailableMangas());
|
|
||||||
log.info("Manga list for content provider {} updated.", contentProviderName);
|
var pages = contentProvider.getTotalPages();
|
||||||
|
|
||||||
|
IntStream.rangeClosed(1, pages)
|
||||||
|
.forEach(
|
||||||
|
page -> {
|
||||||
|
updateMangaListProducer.sendUpdateMangaListCommand(
|
||||||
|
new MangaListUpdateCommand(contentProviderName, page));
|
||||||
|
});
|
||||||
|
|
||||||
|
log.info("Manga list update queued for content provider {}.", contentProviderName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -23,8 +23,8 @@ spring:
|
|||||||
client:
|
client:
|
||||||
config:
|
config:
|
||||||
web-scrapper:
|
web-scrapper:
|
||||||
connect-timeout: 120000
|
connect-timeout: 240000
|
||||||
read-timeout: 120000
|
read-timeout: 240000
|
||||||
rabbitmq:
|
rabbitmq:
|
||||||
host: ${RABBITMQ_HOST}
|
host: ${RABBITMQ_HOST}
|
||||||
port: ${RABBITMQ_PORT}
|
port: ${RABBITMQ_PORT}
|
||||||
@ -71,3 +71,9 @@ resilience4j:
|
|||||||
|
|
||||||
image-service:
|
image-service:
|
||||||
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
|
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
|
||||||
|
cron-expression: "@weekly"
|
||||||
|
|
||||||
|
content-providers:
|
||||||
|
update-enabled: ${CONTENT_PROVIDER_UPDATE_ENABLED:false}
|
||||||
|
cron-expression: "@weekly"
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user