Compare commits

..

4 Commits

23 changed files with 277 additions and 134 deletions

21
.env
View File

@ -1,21 +0,0 @@
DB_URL=jdbc:postgresql://localhost:5432/mangamochi?currentSchema=mangamochi
DB_USER=mangamochi
DB_PASS=mangamochi
MINIO_ENDPOINT=http://omv2.badger-pirarucu.ts.net:9000
MINIO_USER=rov
MINIO_PASS=!E9v4i0v3
FLARESOLVERR_ENDPOINT=https://flare-solverr.badger-pirarucu.ts.net
WEBSCRAPPER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8090/url
MANGAMATCHER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8000/match-title
MANGADEX_USER=rocverde
MANGADEX_PASS=!A3u8e4s0
MANGADEX_CLIENT_ID=personal-client-3c21667a-6de3-4273-94c4-e6014690f128-68830913
MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS
RABBITMQ_HOST=localhost
RABBITMQ_PORT=5672
RABBITMQ_USERNAME=guest
RABBITMQ_PASSWORD=guest

19
.env.example Normal file
View File

@ -0,0 +1,19 @@
DB_URL=jdbc:postgresql://localhost:5432/mangamochi?currentSchema=mangamochi
DB_USER=
DB_PASS=
MINIO_ENDPOINT=http://localhost:9000
MINIO_USER=
MINIO_PASS=
FLARESOLVERR_ENDPOINT=localhost:8191
RABBITMQ_HOST=localhost
RABBITMQ_PORT=5672
RABBITMQ_USERNAME=guest
RABBITMQ_PASSWORD=guest
MANGA_DATA_UPDATE_QUEUE=mangaDataUpdateQueueSample
MANGA_CHAPTER_DOWNLOAD_QUEUE=mangaChapterDownloadQueueSample
MANGA_LIST_UPDATE_QUEUE=mangaListUpdateQueueSample
MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE=mangaFollowUpdateChapterQueueSample

2
.gitignore vendored
View File

@ -139,6 +139,8 @@ fabric.properties
*.tar.gz
*.rar
.env
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
replay_pid*

View File

@ -1,16 +0,0 @@
package com.magamochi.mangamochi.client;
import java.util.List;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@FeignClient(name = "rapidFuzz", url = "${manga-matcher.endpoint}")
public interface RapidFuzzClient {
@PostMapping
Response mangaSearch(@RequestBody Request dto);
record Request(String title, List<String> options) {}
record Response(boolean match_found, String best_match, double similarity) {}
}

View File

@ -4,34 +4,42 @@ import org.springframework.amqp.core.Queue;
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class RabbitConfig {
public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue";
public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue";
public static final String MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE = "mangaFollowUpdateChapterQueue";
@Value("${rabbit-mq.queues.manga-data-update}")
private String mangaDataUpdateQueue;
@Value("${rabbit-mq.queues.manga-chapter-download}")
private String mangaChapterDownloadQueue;
@Value("${rabbit-mq.queues.manga-list-update}")
private String mangaListUpdateQueue;
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
private String mangaFollowUpdateChapterQueue;
@Bean
public Queue mangaDataUpdateQueue() {
return new Queue(MANGA_DATA_UPDATE_QUEUE, false);
return new Queue(mangaDataUpdateQueue, false);
}
@Bean
public Queue mangaChapterDownloadQueue() {
return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false);
return new Queue(mangaChapterDownloadQueue, false);
}
@Bean
public Queue mangaListUpdateQueue() {
return new Queue(MANGA_LIST_UPDATE_QUEUE, false);
return new Queue(mangaListUpdateQueue, false);
}
@Bean
public Queue mangaFollowUpdateChapterQueue() {
return new Queue(MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE, false);
return new Queue(mangaFollowUpdateChapterQueue, false);
}
@Bean

View File

@ -6,6 +6,7 @@ import com.magamochi.mangamochi.model.entity.Manga;
import com.magamochi.mangamochi.model.entity.MangaAlternativeTitle;
import com.magamochi.mangamochi.model.entity.MangaChapter;
import com.magamochi.mangamochi.model.entity.MangaProvider;
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import java.time.OffsetDateTime;
@ -53,6 +54,7 @@ public record MangaDTO(
public record MangaProviderDTO(
@NotNull long id,
@NotBlank String providerName,
@NotNull ProviderStatus providerStatus,
@NotNull Integer chaptersAvailable,
@NotNull Integer chaptersDownloaded,
@NotNull Boolean supportsChapterFetch) {
@ -64,6 +66,7 @@ public record MangaDTO(
return new MangaProviderDTO(
mangaProvider.getId(),
mangaProvider.getProvider().getName(),
mangaProvider.getProvider().getStatus(),
chaptersAvailable,
chaptersDownloaded,
mangaProvider.getProvider().getSupportsChapterFetch());

View File

@ -0,0 +1,15 @@
package com.magamochi.mangamochi.model.dto;
import java.util.List;
import lombok.Builder;
import lombok.Getter;
@Getter
@Builder
public class TitleMatchRequestDTO {
private String title;
private List<String> options;
@Builder.Default private int threshold = 85;
}

View File

@ -0,0 +1,12 @@
package com.magamochi.mangamochi.model.dto;
import lombok.Builder;
import lombok.Getter;
@Getter
@Builder
public class TitleMatchResponseDTO {
boolean matchFound;
String bestMatch;
Double similarity;
}

View File

@ -1,6 +1,5 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.MangaChapterDownloadCommand;
import com.magamochi.mangamochi.service.MangaChapterService;
import lombok.RequiredArgsConstructor;
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
public class MangaChapterDownloadConsumer {
private final MangaChapterService mangaChapterService;
@RabbitListener(queues = RabbitConfig.MANGA_CHAPTER_DOWNLOAD_QUEUE)
@RabbitListener(queues = "${rabbit-mq.queues.manga-chapter-download}")
public void receiveMangaChapterDownloadCommand(MangaChapterDownloadCommand command) {
log.info("Received manga chapter download command: {}", command);

View File

@ -1,10 +1,10 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.MangaChapterDownloadCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
public class MangaChapterDownloadProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${rabbit-mq.queues.manga-chapter-download}")
private String mangaChapterDownloadQueue;
public void sendMangaChapterDownloadCommand(MangaChapterDownloadCommand command) {
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_CHAPTER_DOWNLOAD_QUEUE, command);
rabbitTemplate.convertAndSend(mangaChapterDownloadQueue, command);
log.info("Sent manga chapter download command: {}", command);
}
}

View File

@ -1,6 +1,5 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
import com.magamochi.mangamochi.service.MangaImportService;
import lombok.RequiredArgsConstructor;
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
public class UpdateMangaDataConsumer {
private final MangaImportService mangaImportService;
@RabbitListener(queues = RabbitConfig.MANGA_DATA_UPDATE_QUEUE)
@RabbitListener(queues = "${rabbit-mq.queues.manga-data-update}")
public void receiveUpdateMangaDataCommand(UpdateMangaDataCommand command) {
log.info("Received update manga data command: {}", command);
mangaImportService.updateMangaData(command.mangaId());

View File

@ -1,10 +1,10 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
public class UpdateMangaDataProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${rabbit-mq.queues.manga-data-update}")
private String mangaDataUpdateQueue;
public void sendUpdateMangaDataCommand(UpdateMangaDataCommand command) {
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_DATA_UPDATE_QUEUE, command);
rabbitTemplate.convertAndSend(mangaDataUpdateQueue, command);
log.info("Sent update manga data command: {}", command);
}
}

View File

@ -1,6 +1,5 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand;
import com.magamochi.mangamochi.service.MangaService;
import lombok.RequiredArgsConstructor;
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
public class UpdateMangaFollowChapterListConsumer {
private final MangaService mangaService;
@RabbitListener(queues = RabbitConfig.MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE)
@RabbitListener(queues = "${rabbit-mq.queues.manga-follow-update-chapter}")
public void receiveMangaFollowUpdateChapterCommand(UpdateMangaFollowChapterListCommand command) {
log.info("Received update followed manga chapter list command: {}", command);
mangaService.fetchFollowedMangaChapters(command.mangaProviderId());

View File

@ -1,10 +1,10 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
public class UpdateMangaFollowChapterListProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
private String mangaFollowUpdateChapterQueue;
public void sendUpdateMangaFollowChapterListCommand(UpdateMangaFollowChapterListCommand command) {
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE, command);
rabbitTemplate.convertAndSend(mangaFollowUpdateChapterQueue, command);
log.info("Sent update followed manga chapter list command: {}", command);
}
}

View File

@ -1,6 +1,5 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
import com.magamochi.mangamochi.service.MangaListService;
import lombok.RequiredArgsConstructor;
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
public class UpdateMangaListConsumer {
private final MangaListService mangaListService;
@RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE)
@RabbitListener(queues = "${rabbit-mq.queues.manga-list-update}")
public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) {
log.info("Received update manga list command: {}", command);
mangaListService.updateMangaList(command.contentProviderName(), command.page());

View File

@ -1,10 +1,10 @@
package com.magamochi.mangamochi.queue;
import com.magamochi.mangamochi.config.RabbitConfig;
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
public class UpdateMangaListProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${rabbit-mq.queues.manga-list-update}")
private String mangaListUpdateQueue;
public void sendUpdateMangaListCommand(MangaListUpdateCommand command) {
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command);
rabbitTemplate.convertAndSend(mangaListUpdateQueue, command);
log.info("Sent update manga list command: {}", command);
}
}

View File

@ -2,7 +2,7 @@ package com.magamochi.mangamochi.service;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.mangamochi.client.JikanClient;
import com.magamochi.mangamochi.client.RapidFuzzClient;
import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO;
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
import com.magamochi.mangamochi.model.entity.Manga;
import com.magamochi.mangamochi.model.entity.MangaImportReview;
@ -21,8 +21,9 @@ public class MangaCreationService {
private final MangaRepository mangaRepository;
private final MangaImportReviewRepository mangaImportReviewRepository;
private final TitleMatcherService titleMatcherService;
private final JikanClient jikanClient;
private final RapidFuzzClient rapidFuzzClient;
private final RateLimiter jikanRateLimiter;
@ -42,18 +43,20 @@ public class MangaCreationService {
return null;
}
var request =
new RapidFuzzClient.Request(
title,
jikanResults.stream()
.flatMap(
results ->
results.titles().stream()
.map(JikanClient.SearchResponse.MangaData.TitleData::title))
.toList());
var titleMatchResponse =
titleMatcherService.findBestMatch(
TitleMatchRequestDTO.builder()
.title(title)
.options(
jikanResults.stream()
.flatMap(
results ->
results.titles().stream()
.map(JikanClient.SearchResponse.MangaData.TitleData::title))
.toList())
.build());
var fuzzResults = rapidFuzzClient.mangaSearch(request);
if (!fuzzResults.match_found()) {
if (!titleMatchResponse.isMatchFound()) {
createMangaImportReview(title, url, provider);
log.warn("No match found for manga with title {}", title);
return null;
@ -66,7 +69,7 @@ public class MangaCreationService {
results.titles().stream()
.map(JikanClient.SearchResponse.MangaData.TitleData::title)
.toList()
.contains(fuzzResults.best_match()))
.contains(titleMatchResponse.getBestMatch()))
.findFirst();
if (resultOptional.isEmpty()) {
createMangaImportReview(title, url, provider);

View File

@ -0,0 +1,67 @@
package com.magamochi.mangamochi.service;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.springframework.util.CollectionUtils.isEmpty;
import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO;
import com.magamochi.mangamochi.model.dto.TitleMatchResponseDTO;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.text.similarity.LevenshteinDistance;
import org.springframework.stereotype.Service;
@Log4j2
@Service
public class TitleMatcherService {
private final LevenshteinDistance levenshteinDistance = LevenshteinDistance.getDefaultInstance();
public TitleMatchResponseDTO findBestMatch(TitleMatchRequestDTO request) {
if (isBlank(request.getTitle()) || isEmpty(request.getOptions())) {
throw new IllegalArgumentException("Title and options are required");
}
log.info("Finding best match for {}. Options: {}", request.getTitle(), request.getOptions());
String bestMatch = null;
double bestScore = 0.0;
for (var option : request.getOptions()) {
var score = calculateSimilarityScore(request.getTitle(), option);
if (score > bestScore) {
bestScore = score;
bestMatch = option;
}
}
if (bestScore >= request.getThreshold()) {
log.info(
"Found best match for {}: {}. Similarity: {}", request.getTitle(), bestMatch, bestScore);
return TitleMatchResponseDTO.builder()
.matchFound(true)
.bestMatch(bestMatch)
.similarity(bestScore)
.build();
}
log.info("No match found for {}. Threshold: {}", request.getTitle(), request.getThreshold());
return TitleMatchResponseDTO.builder().matchFound(false).build();
}
private double calculateSimilarityScore(String title, String option) {
var dist = levenshteinDistance.apply(title, option);
var maxLength = Math.max(title.length(), option.length());
if (maxLength == 0) {
return 100.0;
}
// Calculate similarity: 100 * (1 - (distance / max_length))
// This scales the distance into a percentage.
var similarity = 100.0 * (1.0 - ((double) dist / maxLength));
// Format to two decimal places for a cleaner result
return Math.round(similarity * 100.0) / 100.0;
}
}

View File

@ -1,7 +1,7 @@
package com.magamochi.mangamochi.service.providers;
public class ContentProviders {
public static final String MANGA_LIVRE = "Manga Livre";
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
public static final String MANGA_DEX = "MangaDex";
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";

View File

@ -1,5 +1,7 @@
package com.magamochi.mangamochi.service.providers.impl;
import static java.util.Objects.nonNull;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.mangamochi.model.entity.MangaProvider;
@ -16,10 +18,11 @@ import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
@Log4j2
@Service(ContentProviders.MANGA_LIVRE)
@Service(ContentProviders.MANGA_LIVRE_TO)
@RequiredArgsConstructor
public class MangaLivreProvider implements ContentProvider, PagedContentProvider {
private final String url = "https://mangalivre.tv/manga/";
private final String url = "https://mangalivre.to/manga/";
private final int MANGAS_PER_PAGE = 10;
private final FlareService flareService;
@ -27,22 +30,27 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
log.info(
"Getting available chapters from {}, manga {}",
ContentProviders.MANGA_LIVRE,
ContentProviders.MANGA_LIVRE_TO,
provider.getManga().getTitle());
try {
var document =
flareService.getContentAsJsoupDocument(provider.getUrl(), ContentProviders.MANGA_LIVRE);
flareService.getContentAsJsoupDocument(
provider.getUrl(), ContentProviders.MANGA_LIVRE_TO);
var chapterItems = document.getElementsByClass("wp-manga-chapter");
var chapterBoxes =
document.selectFirst("div.listing-chapters-wrap").select("div.chapter-box");
return chapterItems.stream()
return chapterBoxes.stream()
.map(
chapterItemElement -> {
var linkElement = chapterItemElement.getElementsByTag("a").getFirst();
chapterBox -> {
var linkElement = chapterBox.selectFirst("a");
var url = linkElement.attr("href");
var title = linkElement.text();
return new ContentProviderMangaChapterResponseDTO(
linkElement.text(), linkElement.attr("href"), null, null);
title.trim(), url.trim(), null, null);
})
.toList();
} catch (NoSuchElementException e) {
@ -53,21 +61,19 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
@Override
public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
log.info("Getting images from {}, url {}", ContentProviders.MANGA_LIVRE, chapterUrl);
log.info("Getting images from {}, url {}", ContentProviders.MANGA_LIVRE_TO, chapterUrl);
try {
var document =
flareService.getContentAsJsoupDocument(chapterUrl, ContentProviders.MANGA_LIVRE);
flareService.getContentAsJsoupDocument(chapterUrl, ContentProviders.MANGA_LIVRE_TO);
var chapterImagesContainer = document.getElementsByClass("chapter-images").getFirst();
var chapterImagesElements = chapterImagesContainer.getElementsByClass("page-break");
var chapterImagesElements = document.select("div.reading-content img.wp-manga-chapter-img");
var imageUrls =
chapterImagesElements.stream()
.map(
chapterImagesElement -> {
var imageElement = chapterImagesElement.getElementsByTag("img").getFirst();
return imageElement.attr("src");
return chapterImagesElement.attr("src");
})
.toList();
@ -77,54 +83,40 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
Collectors.toMap(
i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new));
} catch (NoSuchElementException e) {
log.error("Error parsing mangas from MangaLivre", e);
log.error("Error parsing manga images from MangaLivre", e);
return Map.of();
}
}
@Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE, page);
log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_TO, page);
try {
var document =
flareService.getContentAsJsoupDocument(
url + "page/" + page, ContentProviders.MANGA_LIVRE);
url + "page/" + page, ContentProviders.MANGA_LIVRE_TO);
var mangaElements = document.getElementsByClass("manga__item");
var mangaElements = document.select("div.page-item-detail.manga");
return mangaElements.stream()
.map(
element -> {
var mangaTitleElement =
element
.getElementsByClass("manga__content")
.getFirst()
.getElementsByClass("manga__content_item")
.getFirst()
.getElementsByClass("post-title font-title")
.getFirst()
.getElementsByTag("h2")
.getFirst();
var linkElement = mangaTitleElement.getElementsByTag("a").getFirst();
var linkElement = element.selectFirst(".item-thumb > a");
var url = linkElement.attr("href");
var title = linkElement.text().trim();
var imageElement =
element
.getElementsByClass("manga__thumb")
.getFirst()
.getElementsByClass("manga__thumb_item")
.getFirst()
.getElementsByTag("a")
.getFirst()
.getElementsByTag("img")
.getFirst();
var imgUrl = imageElement.attr("src");
var title = linkElement.attr("title");
// Fallback: If 'title' attribute is empty, try the <img> tag's 'alt' or 'title'
if (title.isBlank()) {
var imgElement = linkElement.selectFirst("img");
if (nonNull(imgElement)) {
title = imgElement.attr("alt");
}
}
return new ContentProviderMangaInfoResponseDTO(
title, url, imgUrl, MangaStatus.UNKNOWN);
title.trim(), url.trim(), null, MangaStatus.UNKNOWN);
})
.toList();
} catch (NoSuchElementException e) {
@ -135,17 +127,16 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
@Override
public Integer getTotalPages() {
log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE);
log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_TO);
try {
var document = flareService.getContentAsJsoupDocument(url, ContentProviders.MANGA_LIVRE);
var document = flareService.getContentAsJsoupDocument(url, ContentProviders.MANGA_LIVRE_TO);
var navLinks = document.getElementsByClass("wp-pagenavi").getFirst();
var lastPageElement = navLinks.getElementsByClass("last").getFirst();
var links = lastPageElement.attr("href");
var fullText = document.selectFirst("div.h4:contains(resultados)").text();
var numberString = fullText.replace("resultados", "").trim().replaceAll("[^0-9]", "");
var totalMangas = Integer.parseInt(numberString);
var totalPages = links.replaceAll("\\D+", "");
return Integer.parseInt(totalPages);
return (int) Math.ceil((double) totalMangas / MANGAS_PER_PAGE);
} catch (NoSuchElementException e) {
log.error("Error parsing total pages from MangaLivre", e);
return null;

View File

@ -2,6 +2,7 @@ package com.magamochi.mangamochi.task;
import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand;
import com.magamochi.mangamochi.model.entity.Manga;
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
import com.magamochi.mangamochi.model.repository.MangaRepository;
import com.magamochi.mangamochi.queue.UpdateMangaFollowChapterListProducer;
import lombok.RequiredArgsConstructor;
@ -43,10 +44,14 @@ public class MangaFollowUpdateTask {
log.info("Fetching available mangas for followed Manga {}", manga.getTitle());
var mangaProviders = manga.getMangaProviders();
mangaProviders.forEach(
mangaProvider ->
producer.sendUpdateMangaFollowChapterListCommand(
new UpdateMangaFollowChapterListCommand(mangaProvider.getId())));
mangaProviders.stream()
.filter(
mangaProvider -> mangaProvider.getProvider().getStatus().equals(ProviderStatus.ACTIVE))
.forEach(
mangaProvider ->
producer.sendUpdateMangaFollowChapterListCommand(
new UpdateMangaFollowChapterListCommand(mangaProvider.getId())));
log.info("Followed Manga ({}) chapter list update queued.", manga.getTitle());
}

View File

@ -53,9 +53,6 @@ jwt:
refresh-secret: MIV9ctIwrImmrZBjh9QueNEcDOLLVv9Rephii+0DKbk=
refresh-expiration: 2629746000
manga-matcher:
endpoint: ${MANGAMATCHER_ENDPOINT}
resilience4j:
retry:
instances:
@ -78,6 +75,13 @@ resilience4j:
retry-exceptions:
- feign.FeignException
rabbit-mq:
queues:
manga-data-update: ${MANGA_DATA_UPDATE_QUEUE:mangaDataUpdateQueue}
manga-chapter-download: ${MANGA_CHAPTER_DOWNLOAD_QUEUE:mangaChapterDownloadQueue}
manga-list-update: ${MANGA_LIST_UPDATE_QUEUE:mangaListUpdateQueue}
manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangaFollowUpdateChapterQueue}
image-service:
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
cron-expression: "@weekly"

View File

@ -0,0 +1,43 @@
DO
$$
DECLARE
_target_provider_name TEXT := 'Manga Livre';
_target_provider_id BIGINT;
_deleted_chapters_count INTEGER;
_deleted_manga_providers_count INTEGER;
BEGIN
SELECT id
INTO _target_provider_id
FROM providers
WHERE name = _target_provider_name;
IF _target_provider_id IS NULL THEN
RAISE EXCEPTION 'Provider with name "%" not found.', _target_provider_name;
END IF;
UPDATE providers SET status = 'INACTIVE' WHERE id = _target_provider_id;
-- Delete non-downloaded manga chapters associated with the target provider
DELETE
FROM manga_chapters mc
USING manga_provider mp
WHERE mc.manga_provider_id = mp.id
AND mp.provider_id = _target_provider_id
AND mc.downloaded = FALSE;
GET DIAGNOSTICS _deleted_chapters_count = ROW_COUNT;
RAISE NOTICE 'Deleted % non-downloaded chapters for provider ID %.', _deleted_chapters_count, _target_provider_id;
-- Delete MangaProvider records ONLY if NO chapters
DELETE
FROM manga_provider mp
WHERE mp.provider_id = _target_provider_id
AND NOT EXISTS (SELECT 1
FROM manga_chapters mc
WHERE mc.manga_provider_id = mp.id);
GET DIAGNOSTICS _deleted_manga_providers_count = ROW_COUNT;
RAISE NOTICE 'Deleted % manga_provider entries', _deleted_manga_providers_count;
END
$$;