Compare commits
4 Commits
4a66797c79
...
fce38466e8
| Author | SHA1 | Date | |
|---|---|---|---|
| fce38466e8 | |||
| 5883007591 | |||
| 4b0a5ab3e5 | |||
| 8b79776b27 |
21
.env
21
.env
@ -1,21 +0,0 @@
|
||||
DB_URL=jdbc:postgresql://localhost:5432/mangamochi?currentSchema=mangamochi
|
||||
DB_USER=mangamochi
|
||||
DB_PASS=mangamochi
|
||||
|
||||
MINIO_ENDPOINT=http://omv2.badger-pirarucu.ts.net:9000
|
||||
MINIO_USER=rov
|
||||
MINIO_PASS=!E9v4i0v3
|
||||
|
||||
FLARESOLVERR_ENDPOINT=https://flare-solverr.badger-pirarucu.ts.net
|
||||
WEBSCRAPPER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8090/url
|
||||
MANGAMATCHER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8000/match-title
|
||||
|
||||
MANGADEX_USER=rocverde
|
||||
MANGADEX_PASS=!A3u8e4s0
|
||||
MANGADEX_CLIENT_ID=personal-client-3c21667a-6de3-4273-94c4-e6014690f128-68830913
|
||||
MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS
|
||||
|
||||
RABBITMQ_HOST=localhost
|
||||
RABBITMQ_PORT=5672
|
||||
RABBITMQ_USERNAME=guest
|
||||
RABBITMQ_PASSWORD=guest
|
||||
19
.env.example
Normal file
19
.env.example
Normal file
@ -0,0 +1,19 @@
|
||||
DB_URL=jdbc:postgresql://localhost:5432/mangamochi?currentSchema=mangamochi
|
||||
DB_USER=
|
||||
DB_PASS=
|
||||
|
||||
MINIO_ENDPOINT=http://localhost:9000
|
||||
MINIO_USER=
|
||||
MINIO_PASS=
|
||||
|
||||
FLARESOLVERR_ENDPOINT=localhost:8191
|
||||
|
||||
RABBITMQ_HOST=localhost
|
||||
RABBITMQ_PORT=5672
|
||||
RABBITMQ_USERNAME=guest
|
||||
RABBITMQ_PASSWORD=guest
|
||||
|
||||
MANGA_DATA_UPDATE_QUEUE=mangaDataUpdateQueueSample
|
||||
MANGA_CHAPTER_DOWNLOAD_QUEUE=mangaChapterDownloadQueueSample
|
||||
MANGA_LIST_UPDATE_QUEUE=mangaListUpdateQueueSample
|
||||
MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE=mangaFollowUpdateChapterQueueSample
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -139,6 +139,8 @@ fabric.properties
|
||||
*.tar.gz
|
||||
*.rar
|
||||
|
||||
.env
|
||||
|
||||
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
|
||||
hs_err_pid*
|
||||
replay_pid*
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
package com.magamochi.mangamochi.client;
|
||||
|
||||
import java.util.List;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
@FeignClient(name = "rapidFuzz", url = "${manga-matcher.endpoint}")
|
||||
public interface RapidFuzzClient {
|
||||
@PostMapping
|
||||
Response mangaSearch(@RequestBody Request dto);
|
||||
|
||||
record Request(String title, List<String> options) {}
|
||||
|
||||
record Response(boolean match_found, String best_match, double similarity) {}
|
||||
}
|
||||
@ -4,34 +4,42 @@ import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class RabbitConfig {
|
||||
public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue";
|
||||
public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue";
|
||||
public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue";
|
||||
public static final String MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE = "mangaFollowUpdateChapterQueue";
|
||||
@Value("${rabbit-mq.queues.manga-data-update}")
|
||||
private String mangaDataUpdateQueue;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-chapter-download}")
|
||||
private String mangaChapterDownloadQueue;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-list-update}")
|
||||
private String mangaListUpdateQueue;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
|
||||
private String mangaFollowUpdateChapterQueue;
|
||||
|
||||
@Bean
|
||||
public Queue mangaDataUpdateQueue() {
|
||||
return new Queue(MANGA_DATA_UPDATE_QUEUE, false);
|
||||
return new Queue(mangaDataUpdateQueue, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaChapterDownloadQueue() {
|
||||
return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false);
|
||||
return new Queue(mangaChapterDownloadQueue, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaListUpdateQueue() {
|
||||
return new Queue(MANGA_LIST_UPDATE_QUEUE, false);
|
||||
return new Queue(mangaListUpdateQueue, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaFollowUpdateChapterQueue() {
|
||||
return new Queue(MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE, false);
|
||||
return new Queue(mangaFollowUpdateChapterQueue, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@ -6,6 +6,7 @@ import com.magamochi.mangamochi.model.entity.Manga;
|
||||
import com.magamochi.mangamochi.model.entity.MangaAlternativeTitle;
|
||||
import com.magamochi.mangamochi.model.entity.MangaChapter;
|
||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import java.time.OffsetDateTime;
|
||||
@ -53,6 +54,7 @@ public record MangaDTO(
|
||||
public record MangaProviderDTO(
|
||||
@NotNull long id,
|
||||
@NotBlank String providerName,
|
||||
@NotNull ProviderStatus providerStatus,
|
||||
@NotNull Integer chaptersAvailable,
|
||||
@NotNull Integer chaptersDownloaded,
|
||||
@NotNull Boolean supportsChapterFetch) {
|
||||
@ -64,6 +66,7 @@ public record MangaDTO(
|
||||
return new MangaProviderDTO(
|
||||
mangaProvider.getId(),
|
||||
mangaProvider.getProvider().getName(),
|
||||
mangaProvider.getProvider().getStatus(),
|
||||
chaptersAvailable,
|
||||
chaptersDownloaded,
|
||||
mangaProvider.getProvider().getSupportsChapterFetch());
|
||||
|
||||
@ -0,0 +1,15 @@
|
||||
package com.magamochi.mangamochi.model.dto;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
public class TitleMatchRequestDTO {
|
||||
private String title;
|
||||
|
||||
private List<String> options;
|
||||
|
||||
@Builder.Default private int threshold = 85;
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package com.magamochi.mangamochi.model.dto;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
public class TitleMatchResponseDTO {
|
||||
boolean matchFound;
|
||||
String bestMatch;
|
||||
Double similarity;
|
||||
}
|
||||
@ -1,6 +1,5 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.MangaChapterDownloadCommand;
|
||||
import com.magamochi.mangamochi.service.MangaChapterService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
public class MangaChapterDownloadConsumer {
|
||||
private final MangaChapterService mangaChapterService;
|
||||
|
||||
@RabbitListener(queues = RabbitConfig.MANGA_CHAPTER_DOWNLOAD_QUEUE)
|
||||
@RabbitListener(queues = "${rabbit-mq.queues.manga-chapter-download}")
|
||||
public void receiveMangaChapterDownloadCommand(MangaChapterDownloadCommand command) {
|
||||
log.info("Received manga chapter download command: {}", command);
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.MangaChapterDownloadCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
|
||||
public class MangaChapterDownloadProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-chapter-download}")
|
||||
private String mangaChapterDownloadQueue;
|
||||
|
||||
public void sendMangaChapterDownloadCommand(MangaChapterDownloadCommand command) {
|
||||
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_CHAPTER_DOWNLOAD_QUEUE, command);
|
||||
rabbitTemplate.convertAndSend(mangaChapterDownloadQueue, command);
|
||||
log.info("Sent manga chapter download command: {}", command);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||
import com.magamochi.mangamochi.service.MangaImportService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaDataConsumer {
|
||||
private final MangaImportService mangaImportService;
|
||||
|
||||
@RabbitListener(queues = RabbitConfig.MANGA_DATA_UPDATE_QUEUE)
|
||||
@RabbitListener(queues = "${rabbit-mq.queues.manga-data-update}")
|
||||
public void receiveUpdateMangaDataCommand(UpdateMangaDataCommand command) {
|
||||
log.info("Received update manga data command: {}", command);
|
||||
mangaImportService.updateMangaData(command.mangaId());
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaDataProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-data-update}")
|
||||
private String mangaDataUpdateQueue;
|
||||
|
||||
public void sendUpdateMangaDataCommand(UpdateMangaDataCommand command) {
|
||||
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_DATA_UPDATE_QUEUE, command);
|
||||
rabbitTemplate.convertAndSend(mangaDataUpdateQueue, command);
|
||||
log.info("Sent update manga data command: {}", command);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.mangamochi.service.MangaService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaFollowChapterListConsumer {
|
||||
private final MangaService mangaService;
|
||||
|
||||
@RabbitListener(queues = RabbitConfig.MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE)
|
||||
@RabbitListener(queues = "${rabbit-mq.queues.manga-follow-update-chapter}")
|
||||
public void receiveMangaFollowUpdateChapterCommand(UpdateMangaFollowChapterListCommand command) {
|
||||
log.info("Received update followed manga chapter list command: {}", command);
|
||||
mangaService.fetchFollowedMangaChapters(command.mangaProviderId());
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaFollowChapterListProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
|
||||
private String mangaFollowUpdateChapterQueue;
|
||||
|
||||
public void sendUpdateMangaFollowChapterListCommand(UpdateMangaFollowChapterListCommand command) {
|
||||
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE, command);
|
||||
rabbitTemplate.convertAndSend(mangaFollowUpdateChapterQueue, command);
|
||||
log.info("Sent update followed manga chapter list command: {}", command);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
|
||||
import com.magamochi.mangamochi.service.MangaListService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -14,7 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaListConsumer {
|
||||
private final MangaListService mangaListService;
|
||||
|
||||
@RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE)
|
||||
@RabbitListener(queues = "${rabbit-mq.queues.manga-list-update}")
|
||||
public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) {
|
||||
log.info("Received update manga list command: {}", command);
|
||||
mangaListService.updateMangaList(command.contentProviderName(), command.page());
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package com.magamochi.mangamochi.queue;
|
||||
|
||||
import com.magamochi.mangamochi.config.RabbitConfig;
|
||||
import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@ -13,8 +13,11 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaListProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-list-update}")
|
||||
private String mangaListUpdateQueue;
|
||||
|
||||
public void sendUpdateMangaListCommand(MangaListUpdateCommand command) {
|
||||
rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command);
|
||||
rabbitTemplate.convertAndSend(mangaListUpdateQueue, command);
|
||||
log.info("Sent update manga list command: {}", command);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,7 +2,7 @@ package com.magamochi.mangamochi.service;
|
||||
|
||||
import com.google.common.util.concurrent.RateLimiter;
|
||||
import com.magamochi.mangamochi.client.JikanClient;
|
||||
import com.magamochi.mangamochi.client.RapidFuzzClient;
|
||||
import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO;
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||
import com.magamochi.mangamochi.model.entity.Manga;
|
||||
import com.magamochi.mangamochi.model.entity.MangaImportReview;
|
||||
@ -21,8 +21,9 @@ public class MangaCreationService {
|
||||
private final MangaRepository mangaRepository;
|
||||
private final MangaImportReviewRepository mangaImportReviewRepository;
|
||||
|
||||
private final TitleMatcherService titleMatcherService;
|
||||
|
||||
private final JikanClient jikanClient;
|
||||
private final RapidFuzzClient rapidFuzzClient;
|
||||
|
||||
private final RateLimiter jikanRateLimiter;
|
||||
|
||||
@ -42,18 +43,20 @@ public class MangaCreationService {
|
||||
return null;
|
||||
}
|
||||
|
||||
var request =
|
||||
new RapidFuzzClient.Request(
|
||||
title,
|
||||
var titleMatchResponse =
|
||||
titleMatcherService.findBestMatch(
|
||||
TitleMatchRequestDTO.builder()
|
||||
.title(title)
|
||||
.options(
|
||||
jikanResults.stream()
|
||||
.flatMap(
|
||||
results ->
|
||||
results.titles().stream()
|
||||
.map(JikanClient.SearchResponse.MangaData.TitleData::title))
|
||||
.toList());
|
||||
.toList())
|
||||
.build());
|
||||
|
||||
var fuzzResults = rapidFuzzClient.mangaSearch(request);
|
||||
if (!fuzzResults.match_found()) {
|
||||
if (!titleMatchResponse.isMatchFound()) {
|
||||
createMangaImportReview(title, url, provider);
|
||||
log.warn("No match found for manga with title {}", title);
|
||||
return null;
|
||||
@ -66,7 +69,7 @@ public class MangaCreationService {
|
||||
results.titles().stream()
|
||||
.map(JikanClient.SearchResponse.MangaData.TitleData::title)
|
||||
.toList()
|
||||
.contains(fuzzResults.best_match()))
|
||||
.contains(titleMatchResponse.getBestMatch()))
|
||||
.findFirst();
|
||||
if (resultOptional.isEmpty()) {
|
||||
createMangaImportReview(title, url, provider);
|
||||
|
||||
@ -0,0 +1,67 @@
|
||||
package com.magamochi.mangamochi.service;
|
||||
|
||||
import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||
import static org.springframework.util.CollectionUtils.isEmpty;
|
||||
|
||||
import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO;
|
||||
import com.magamochi.mangamochi.model.dto.TitleMatchResponseDTO;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.commons.text.similarity.LevenshteinDistance;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
public class TitleMatcherService {
|
||||
private final LevenshteinDistance levenshteinDistance = LevenshteinDistance.getDefaultInstance();
|
||||
|
||||
public TitleMatchResponseDTO findBestMatch(TitleMatchRequestDTO request) {
|
||||
if (isBlank(request.getTitle()) || isEmpty(request.getOptions())) {
|
||||
throw new IllegalArgumentException("Title and options are required");
|
||||
}
|
||||
|
||||
log.info("Finding best match for {}. Options: {}", request.getTitle(), request.getOptions());
|
||||
|
||||
String bestMatch = null;
|
||||
double bestScore = 0.0;
|
||||
|
||||
for (var option : request.getOptions()) {
|
||||
var score = calculateSimilarityScore(request.getTitle(), option);
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestMatch = option;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestScore >= request.getThreshold()) {
|
||||
log.info(
|
||||
"Found best match for {}: {}. Similarity: {}", request.getTitle(), bestMatch, bestScore);
|
||||
|
||||
return TitleMatchResponseDTO.builder()
|
||||
.matchFound(true)
|
||||
.bestMatch(bestMatch)
|
||||
.similarity(bestScore)
|
||||
.build();
|
||||
}
|
||||
|
||||
log.info("No match found for {}. Threshold: {}", request.getTitle(), request.getThreshold());
|
||||
|
||||
return TitleMatchResponseDTO.builder().matchFound(false).build();
|
||||
}
|
||||
|
||||
private double calculateSimilarityScore(String title, String option) {
|
||||
var dist = levenshteinDistance.apply(title, option);
|
||||
|
||||
var maxLength = Math.max(title.length(), option.length());
|
||||
if (maxLength == 0) {
|
||||
return 100.0;
|
||||
}
|
||||
|
||||
// Calculate similarity: 100 * (1 - (distance / max_length))
|
||||
// This scales the distance into a percentage.
|
||||
var similarity = 100.0 * (1.0 - ((double) dist / maxLength));
|
||||
|
||||
// Format to two decimal places for a cleaner result
|
||||
return Math.round(similarity * 100.0) / 100.0;
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
package com.magamochi.mangamochi.service.providers;
|
||||
|
||||
public class ContentProviders {
|
||||
public static final String MANGA_LIVRE = "Manga Livre";
|
||||
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
|
||||
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
|
||||
public static final String MANGA_DEX = "MangaDex";
|
||||
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
package com.magamochi.mangamochi.service.providers.impl;
|
||||
|
||||
import static java.util.Objects.nonNull;
|
||||
|
||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
|
||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||
@ -16,10 +18,11 @@ import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service(ContentProviders.MANGA_LIVRE)
|
||||
@Service(ContentProviders.MANGA_LIVRE_TO)
|
||||
@RequiredArgsConstructor
|
||||
public class MangaLivreProvider implements ContentProvider, PagedContentProvider {
|
||||
private final String url = "https://mangalivre.tv/manga/";
|
||||
private final String url = "https://mangalivre.to/manga/";
|
||||
private final int MANGAS_PER_PAGE = 10;
|
||||
|
||||
private final FlareService flareService;
|
||||
|
||||
@ -27,22 +30,27 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
|
||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||
log.info(
|
||||
"Getting available chapters from {}, manga {}",
|
||||
ContentProviders.MANGA_LIVRE,
|
||||
ContentProviders.MANGA_LIVRE_TO,
|
||||
provider.getManga().getTitle());
|
||||
|
||||
try {
|
||||
var document =
|
||||
flareService.getContentAsJsoupDocument(provider.getUrl(), ContentProviders.MANGA_LIVRE);
|
||||
flareService.getContentAsJsoupDocument(
|
||||
provider.getUrl(), ContentProviders.MANGA_LIVRE_TO);
|
||||
|
||||
var chapterItems = document.getElementsByClass("wp-manga-chapter");
|
||||
var chapterBoxes =
|
||||
document.selectFirst("div.listing-chapters-wrap").select("div.chapter-box");
|
||||
|
||||
return chapterItems.stream()
|
||||
return chapterBoxes.stream()
|
||||
.map(
|
||||
chapterItemElement -> {
|
||||
var linkElement = chapterItemElement.getElementsByTag("a").getFirst();
|
||||
chapterBox -> {
|
||||
var linkElement = chapterBox.selectFirst("a");
|
||||
|
||||
var url = linkElement.attr("href");
|
||||
var title = linkElement.text();
|
||||
|
||||
return new ContentProviderMangaChapterResponseDTO(
|
||||
linkElement.text(), linkElement.attr("href"), null, null);
|
||||
title.trim(), url.trim(), null, null);
|
||||
})
|
||||
.toList();
|
||||
} catch (NoSuchElementException e) {
|
||||
@ -53,21 +61,19 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
|
||||
|
||||
@Override
|
||||
public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
|
||||
log.info("Getting images from {}, url {}", ContentProviders.MANGA_LIVRE, chapterUrl);
|
||||
log.info("Getting images from {}, url {}", ContentProviders.MANGA_LIVRE_TO, chapterUrl);
|
||||
|
||||
try {
|
||||
var document =
|
||||
flareService.getContentAsJsoupDocument(chapterUrl, ContentProviders.MANGA_LIVRE);
|
||||
flareService.getContentAsJsoupDocument(chapterUrl, ContentProviders.MANGA_LIVRE_TO);
|
||||
|
||||
var chapterImagesContainer = document.getElementsByClass("chapter-images").getFirst();
|
||||
var chapterImagesElements = chapterImagesContainer.getElementsByClass("page-break");
|
||||
var chapterImagesElements = document.select("div.reading-content img.wp-manga-chapter-img");
|
||||
|
||||
var imageUrls =
|
||||
chapterImagesElements.stream()
|
||||
.map(
|
||||
chapterImagesElement -> {
|
||||
var imageElement = chapterImagesElement.getElementsByTag("img").getFirst();
|
||||
return imageElement.attr("src");
|
||||
return chapterImagesElement.attr("src");
|
||||
})
|
||||
.toList();
|
||||
|
||||
@ -77,54 +83,40 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
|
||||
Collectors.toMap(
|
||||
i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new));
|
||||
} catch (NoSuchElementException e) {
|
||||
log.error("Error parsing mangas from MangaLivre", e);
|
||||
log.error("Error parsing manga images from MangaLivre", e);
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
|
||||
log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE, page);
|
||||
log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_TO, page);
|
||||
|
||||
try {
|
||||
var document =
|
||||
flareService.getContentAsJsoupDocument(
|
||||
url + "page/" + page, ContentProviders.MANGA_LIVRE);
|
||||
url + "page/" + page, ContentProviders.MANGA_LIVRE_TO);
|
||||
|
||||
var mangaElements = document.getElementsByClass("manga__item");
|
||||
var mangaElements = document.select("div.page-item-detail.manga");
|
||||
|
||||
return mangaElements.stream()
|
||||
.map(
|
||||
element -> {
|
||||
var mangaTitleElement =
|
||||
element
|
||||
.getElementsByClass("manga__content")
|
||||
.getFirst()
|
||||
.getElementsByClass("manga__content_item")
|
||||
.getFirst()
|
||||
.getElementsByClass("post-title font-title")
|
||||
.getFirst()
|
||||
.getElementsByTag("h2")
|
||||
.getFirst();
|
||||
|
||||
var linkElement = mangaTitleElement.getElementsByTag("a").getFirst();
|
||||
var linkElement = element.selectFirst(".item-thumb > a");
|
||||
var url = linkElement.attr("href");
|
||||
var title = linkElement.text().trim();
|
||||
|
||||
var imageElement =
|
||||
element
|
||||
.getElementsByClass("manga__thumb")
|
||||
.getFirst()
|
||||
.getElementsByClass("manga__thumb_item")
|
||||
.getFirst()
|
||||
.getElementsByTag("a")
|
||||
.getFirst()
|
||||
.getElementsByTag("img")
|
||||
.getFirst();
|
||||
var imgUrl = imageElement.attr("src");
|
||||
var title = linkElement.attr("title");
|
||||
|
||||
// Fallback: If 'title' attribute is empty, try the <img> tag's 'alt' or 'title'
|
||||
if (title.isBlank()) {
|
||||
var imgElement = linkElement.selectFirst("img");
|
||||
if (nonNull(imgElement)) {
|
||||
title = imgElement.attr("alt");
|
||||
}
|
||||
}
|
||||
|
||||
return new ContentProviderMangaInfoResponseDTO(
|
||||
title, url, imgUrl, MangaStatus.UNKNOWN);
|
||||
title.trim(), url.trim(), null, MangaStatus.UNKNOWN);
|
||||
})
|
||||
.toList();
|
||||
} catch (NoSuchElementException e) {
|
||||
@ -135,17 +127,16 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
|
||||
|
||||
@Override
|
||||
public Integer getTotalPages() {
|
||||
log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE);
|
||||
log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_TO);
|
||||
|
||||
try {
|
||||
var document = flareService.getContentAsJsoupDocument(url, ContentProviders.MANGA_LIVRE);
|
||||
var document = flareService.getContentAsJsoupDocument(url, ContentProviders.MANGA_LIVRE_TO);
|
||||
|
||||
var navLinks = document.getElementsByClass("wp-pagenavi").getFirst();
|
||||
var lastPageElement = navLinks.getElementsByClass("last").getFirst();
|
||||
var links = lastPageElement.attr("href");
|
||||
var fullText = document.selectFirst("div.h4:contains(resultados)").text();
|
||||
var numberString = fullText.replace("resultados", "").trim().replaceAll("[^0-9]", "");
|
||||
var totalMangas = Integer.parseInt(numberString);
|
||||
|
||||
var totalPages = links.replaceAll("\\D+", "");
|
||||
return Integer.parseInt(totalPages);
|
||||
return (int) Math.ceil((double) totalMangas / MANGAS_PER_PAGE);
|
||||
} catch (NoSuchElementException e) {
|
||||
log.error("Error parsing total pages from MangaLivre", e);
|
||||
return null;
|
||||
|
||||
@ -2,6 +2,7 @@ package com.magamochi.mangamochi.task;
|
||||
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.mangamochi.model.entity.Manga;
|
||||
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
||||
import com.magamochi.mangamochi.model.repository.MangaRepository;
|
||||
import com.magamochi.mangamochi.queue.UpdateMangaFollowChapterListProducer;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -43,7 +44,11 @@ public class MangaFollowUpdateTask {
|
||||
log.info("Fetching available mangas for followed Manga {}", manga.getTitle());
|
||||
|
||||
var mangaProviders = manga.getMangaProviders();
|
||||
mangaProviders.forEach(
|
||||
|
||||
mangaProviders.stream()
|
||||
.filter(
|
||||
mangaProvider -> mangaProvider.getProvider().getStatus().equals(ProviderStatus.ACTIVE))
|
||||
.forEach(
|
||||
mangaProvider ->
|
||||
producer.sendUpdateMangaFollowChapterListCommand(
|
||||
new UpdateMangaFollowChapterListCommand(mangaProvider.getId())));
|
||||
|
||||
@ -53,9 +53,6 @@ jwt:
|
||||
refresh-secret: MIV9ctIwrImmrZBjh9QueNEcDOLLVv9Rephii+0DKbk=
|
||||
refresh-expiration: 2629746000
|
||||
|
||||
manga-matcher:
|
||||
endpoint: ${MANGAMATCHER_ENDPOINT}
|
||||
|
||||
resilience4j:
|
||||
retry:
|
||||
instances:
|
||||
@ -78,6 +75,13 @@ resilience4j:
|
||||
retry-exceptions:
|
||||
- feign.FeignException
|
||||
|
||||
rabbit-mq:
|
||||
queues:
|
||||
manga-data-update: ${MANGA_DATA_UPDATE_QUEUE:mangaDataUpdateQueue}
|
||||
manga-chapter-download: ${MANGA_CHAPTER_DOWNLOAD_QUEUE:mangaChapterDownloadQueue}
|
||||
manga-list-update: ${MANGA_LIST_UPDATE_QUEUE:mangaListUpdateQueue}
|
||||
manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangaFollowUpdateChapterQueue}
|
||||
|
||||
image-service:
|
||||
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
|
||||
cron-expression: "@weekly"
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
DO
|
||||
$$
|
||||
DECLARE
|
||||
_target_provider_name TEXT := 'Manga Livre';
|
||||
_target_provider_id BIGINT;
|
||||
_deleted_chapters_count INTEGER;
|
||||
_deleted_manga_providers_count INTEGER;
|
||||
|
||||
BEGIN
|
||||
SELECT id
|
||||
INTO _target_provider_id
|
||||
FROM providers
|
||||
WHERE name = _target_provider_name;
|
||||
|
||||
IF _target_provider_id IS NULL THEN
|
||||
RAISE EXCEPTION 'Provider with name "%" not found.', _target_provider_name;
|
||||
END IF;
|
||||
|
||||
UPDATE providers SET status = 'INACTIVE' WHERE id = _target_provider_id;
|
||||
|
||||
-- Delete non-downloaded manga chapters associated with the target provider
|
||||
DELETE
|
||||
FROM manga_chapters mc
|
||||
USING manga_provider mp
|
||||
WHERE mc.manga_provider_id = mp.id
|
||||
AND mp.provider_id = _target_provider_id
|
||||
AND mc.downloaded = FALSE;
|
||||
|
||||
GET DIAGNOSTICS _deleted_chapters_count = ROW_COUNT;
|
||||
RAISE NOTICE 'Deleted % non-downloaded chapters for provider ID %.', _deleted_chapters_count, _target_provider_id;
|
||||
|
||||
-- Delete MangaProvider records ONLY if NO chapters
|
||||
DELETE
|
||||
FROM manga_provider mp
|
||||
WHERE mp.provider_id = _target_provider_id
|
||||
AND NOT EXISTS (SELECT 1
|
||||
FROM manga_chapters mc
|
||||
WHERE mc.manga_provider_id = mp.id);
|
||||
|
||||
GET DIAGNOSTICS _deleted_manga_providers_count = ROW_COUNT;
|
||||
RAISE NOTICE 'Deleted % manga_provider entries', _deleted_manga_providers_count;
|
||||
END
|
||||
$$;
|
||||
Loading…
x
Reference in New Issue
Block a user