diff --git a/.env b/.env deleted file mode 100644 index 1167246..0000000 --- a/.env +++ /dev/null @@ -1,21 +0,0 @@ -DB_URL=jdbc:postgresql://localhost:5432/mangamochi?currentSchema=mangamochi -DB_USER=mangamochi -DB_PASS=mangamochi - -MINIO_ENDPOINT=http://omv2.badger-pirarucu.ts.net:9000 -MINIO_USER=rov -MINIO_PASS=!E9v4i0v3 - -FLARESOLVERR_ENDPOINT=https://flare-solverr.badger-pirarucu.ts.net -WEBSCRAPPER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8090/url -MANGAMATCHER_ENDPOINT=http://mangamochi.badger-pirarucu.ts.net:8000/match-title - -MANGADEX_USER=rocverde -MANGADEX_PASS=!A3u8e4s0 -MANGADEX_CLIENT_ID=personal-client-3c21667a-6de3-4273-94c4-e6014690f128-68830913 -MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS - -RABBITMQ_HOST=localhost -RABBITMQ_PORT=5672 -RABBITMQ_USERNAME=guest -RABBITMQ_PASSWORD=guest diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9623c6d --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +DB_URL=jdbc:postgresql://localhost:5432/mangamochi?currentSchema=mangamochi +DB_USER= +DB_PASS= + +MINIO_ENDPOINT=http://localhost:9000 +MINIO_USER= +MINIO_PASS= + +FLARESOLVERR_ENDPOINT=localhost:8191 + +RABBITMQ_HOST=localhost +RABBITMQ_PORT=5672 +RABBITMQ_USERNAME=guest +RABBITMQ_PASSWORD=guest + +MANGA_DATA_UPDATE_QUEUE=mangaDataUpdateQueueSample +MANGA_CHAPTER_DOWNLOAD_QUEUE=mangaChapterDownloadQueueSample +MANGA_LIST_UPDATE_QUEUE=mangaListUpdateQueueSample +MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE=mangaFollowUpdateChapterQueueSample \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6ffd5a5..8dd4618 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,8 @@ fabric.properties *.tar.gz *.rar +.env + # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* replay_pid* diff --git a/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java b/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java deleted file mode 100644 index f2f7741..0000000 --- a/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.magamochi.mangamochi.client; - -import java.util.List; -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; - -@FeignClient(name = "rapidFuzz", url = "${manga-matcher.endpoint}") -public interface RapidFuzzClient { - @PostMapping - Response mangaSearch(@RequestBody Request dto); - - record Request(String title, List options) {} - - record Response(boolean match_found, String best_match, double similarity) {} -} diff --git a/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java b/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java index eaa7780..09e13f0 100644 --- a/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java +++ b/src/main/java/com/magamochi/mangamochi/config/RabbitConfig.java @@ -4,34 +4,42 @@ import org.springframework.amqp.core.Queue; import org.springframework.amqp.rabbit.connection.ConnectionFactory; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class RabbitConfig { - public static final String MANGA_DATA_UPDATE_QUEUE = "mangaDataUpdateQueue"; - public static final String MANGA_CHAPTER_DOWNLOAD_QUEUE = "mangaChapterDownloadQueue"; - public static final String MANGA_LIST_UPDATE_QUEUE = "mangaListUpdateQueue"; - public static final String MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE = "mangaFollowUpdateChapterQueue"; + @Value("${rabbit-mq.queues.manga-data-update}") + private String mangaDataUpdateQueue; + + @Value("${rabbit-mq.queues.manga-chapter-download}") + private String mangaChapterDownloadQueue; + + @Value("${rabbit-mq.queues.manga-list-update}") + private String mangaListUpdateQueue; + + @Value("${rabbit-mq.queues.manga-follow-update-chapter}") + private String mangaFollowUpdateChapterQueue; @Bean public Queue mangaDataUpdateQueue() { - return new Queue(MANGA_DATA_UPDATE_QUEUE, false); + return new Queue(mangaDataUpdateQueue, false); } @Bean public Queue mangaChapterDownloadQueue() { - return new Queue(MANGA_CHAPTER_DOWNLOAD_QUEUE, false); + return new Queue(mangaChapterDownloadQueue, false); } @Bean public Queue mangaListUpdateQueue() { - return new Queue(MANGA_LIST_UPDATE_QUEUE, false); + return new Queue(mangaListUpdateQueue, false); } @Bean public Queue mangaFollowUpdateChapterQueue() { - return new Queue(MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE, false); + return new Queue(mangaFollowUpdateChapterQueue, false); } @Bean diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/MangaDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/MangaDTO.java index 58bab02..454a8e7 100644 --- a/src/main/java/com/magamochi/mangamochi/model/dto/MangaDTO.java +++ b/src/main/java/com/magamochi/mangamochi/model/dto/MangaDTO.java @@ -6,6 +6,7 @@ import com.magamochi.mangamochi.model.entity.Manga; import com.magamochi.mangamochi.model.entity.MangaAlternativeTitle; import com.magamochi.mangamochi.model.entity.MangaChapter; import com.magamochi.mangamochi.model.entity.MangaProvider; +import com.magamochi.mangamochi.model.enumeration.ProviderStatus; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import java.time.OffsetDateTime; @@ -53,6 +54,7 @@ public record MangaDTO( public record MangaProviderDTO( @NotNull long id, @NotBlank String providerName, + @NotNull ProviderStatus providerStatus, @NotNull Integer chaptersAvailable, @NotNull Integer chaptersDownloaded, @NotNull Boolean supportsChapterFetch) { @@ -64,6 +66,7 @@ public record MangaDTO( return new MangaProviderDTO( mangaProvider.getId(), mangaProvider.getProvider().getName(), + mangaProvider.getProvider().getStatus(), chaptersAvailable, chaptersDownloaded, mangaProvider.getProvider().getSupportsChapterFetch()); diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/TitleMatchRequestDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/TitleMatchRequestDTO.java new file mode 100644 index 0000000..a31b88c --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/model/dto/TitleMatchRequestDTO.java @@ -0,0 +1,15 @@ +package com.magamochi.mangamochi.model.dto; + +import java.util.List; +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class TitleMatchRequestDTO { + private String title; + + private List options; + + @Builder.Default private int threshold = 85; +} diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/TitleMatchResponseDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/TitleMatchResponseDTO.java new file mode 100644 index 0000000..c2a3362 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/model/dto/TitleMatchResponseDTO.java @@ -0,0 +1,12 @@ +package com.magamochi.mangamochi.model.dto; + +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class TitleMatchResponseDTO { + boolean matchFound; + String bestMatch; + Double similarity; +} diff --git a/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadConsumer.java b/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadConsumer.java index 3a9c018..3c286c2 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadConsumer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadConsumer.java @@ -1,6 +1,5 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.MangaChapterDownloadCommand; import com.magamochi.mangamochi.service.MangaChapterService; import lombok.RequiredArgsConstructor; @@ -14,7 +13,7 @@ import org.springframework.stereotype.Service; public class MangaChapterDownloadConsumer { private final MangaChapterService mangaChapterService; - @RabbitListener(queues = RabbitConfig.MANGA_CHAPTER_DOWNLOAD_QUEUE) + @RabbitListener(queues = "${rabbit-mq.queues.manga-chapter-download}") public void receiveMangaChapterDownloadCommand(MangaChapterDownloadCommand command) { log.info("Received manga chapter download command: {}", command); diff --git a/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadProducer.java b/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadProducer.java index 2b638ae..dfafb53 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadProducer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/MangaChapterDownloadProducer.java @@ -1,10 +1,10 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.MangaChapterDownloadCommand; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @Log4j2 @@ -13,8 +13,11 @@ import org.springframework.stereotype.Service; public class MangaChapterDownloadProducer { private final RabbitTemplate rabbitTemplate; + @Value("${rabbit-mq.queues.manga-chapter-download}") + private String mangaChapterDownloadQueue; + public void sendMangaChapterDownloadCommand(MangaChapterDownloadCommand command) { - rabbitTemplate.convertAndSend(RabbitConfig.MANGA_CHAPTER_DOWNLOAD_QUEUE, command); + rabbitTemplate.convertAndSend(mangaChapterDownloadQueue, command); log.info("Sent manga chapter download command: {}", command); } } diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataConsumer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataConsumer.java index d0cd79a..519c5bc 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataConsumer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataConsumer.java @@ -1,6 +1,5 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand; import com.magamochi.mangamochi.service.MangaImportService; import lombok.RequiredArgsConstructor; @@ -14,7 +13,7 @@ import org.springframework.stereotype.Service; public class UpdateMangaDataConsumer { private final MangaImportService mangaImportService; - @RabbitListener(queues = RabbitConfig.MANGA_DATA_UPDATE_QUEUE) + @RabbitListener(queues = "${rabbit-mq.queues.manga-data-update}") public void receiveUpdateMangaDataCommand(UpdateMangaDataCommand command) { log.info("Received update manga data command: {}", command); mangaImportService.updateMangaData(command.mangaId()); diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataProducer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataProducer.java index 9e03594..12f67a9 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataProducer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaDataProducer.java @@ -1,10 +1,10 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @Log4j2 @@ -13,8 +13,11 @@ import org.springframework.stereotype.Service; public class UpdateMangaDataProducer { private final RabbitTemplate rabbitTemplate; + @Value("${rabbit-mq.queues.manga-data-update}") + private String mangaDataUpdateQueue; + public void sendUpdateMangaDataCommand(UpdateMangaDataCommand command) { - rabbitTemplate.convertAndSend(RabbitConfig.MANGA_DATA_UPDATE_QUEUE, command); + rabbitTemplate.convertAndSend(mangaDataUpdateQueue, command); log.info("Sent update manga data command: {}", command); } } diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListConsumer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListConsumer.java index 0695bba..4d1632f 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListConsumer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListConsumer.java @@ -1,6 +1,5 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand; import com.magamochi.mangamochi.service.MangaService; import lombok.RequiredArgsConstructor; @@ -14,7 +13,7 @@ import org.springframework.stereotype.Service; public class UpdateMangaFollowChapterListConsumer { private final MangaService mangaService; - @RabbitListener(queues = RabbitConfig.MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE) + @RabbitListener(queues = "${rabbit-mq.queues.manga-follow-update-chapter}") public void receiveMangaFollowUpdateChapterCommand(UpdateMangaFollowChapterListCommand command) { log.info("Received update followed manga chapter list command: {}", command); mangaService.fetchFollowedMangaChapters(command.mangaProviderId()); diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListProducer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListProducer.java index d1e3c2f..14ee2c9 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListProducer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaFollowChapterListProducer.java @@ -1,10 +1,10 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @Log4j2 @@ -13,8 +13,11 @@ import org.springframework.stereotype.Service; public class UpdateMangaFollowChapterListProducer { private final RabbitTemplate rabbitTemplate; + @Value("${rabbit-mq.queues.manga-follow-update-chapter}") + private String mangaFollowUpdateChapterQueue; + public void sendUpdateMangaFollowChapterListCommand(UpdateMangaFollowChapterListCommand command) { - rabbitTemplate.convertAndSend(RabbitConfig.MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE, command); + rabbitTemplate.convertAndSend(mangaFollowUpdateChapterQueue, command); log.info("Sent update followed manga chapter list command: {}", command); } } diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java index 38925c3..e8e3a0b 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListConsumer.java @@ -1,6 +1,5 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand; import com.magamochi.mangamochi.service.MangaListService; import lombok.RequiredArgsConstructor; @@ -14,7 +13,7 @@ import org.springframework.stereotype.Service; public class UpdateMangaListConsumer { private final MangaListService mangaListService; - @RabbitListener(queues = RabbitConfig.MANGA_LIST_UPDATE_QUEUE) + @RabbitListener(queues = "${rabbit-mq.queues.manga-list-update}") public void receiveUpdateMangaListCommand(MangaListUpdateCommand command) { log.info("Received update manga list command: {}", command); mangaListService.updateMangaList(command.contentProviderName(), command.page()); diff --git a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java index 167e66b..e154d0d 100644 --- a/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java +++ b/src/main/java/com/magamochi/mangamochi/queue/UpdateMangaListProducer.java @@ -1,10 +1,10 @@ package com.magamochi.mangamochi.queue; -import com.magamochi.mangamochi.config.RabbitConfig; import com.magamochi.mangamochi.model.dto.MangaListUpdateCommand; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @Log4j2 @@ -13,8 +13,11 @@ import org.springframework.stereotype.Service; public class UpdateMangaListProducer { private final RabbitTemplate rabbitTemplate; + @Value("${rabbit-mq.queues.manga-list-update}") + private String mangaListUpdateQueue; + public void sendUpdateMangaListCommand(MangaListUpdateCommand command) { - rabbitTemplate.convertAndSend(RabbitConfig.MANGA_LIST_UPDATE_QUEUE, command); + rabbitTemplate.convertAndSend(mangaListUpdateQueue, command); log.info("Sent update manga list command: {}", command); } } diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java index da0542a..a703844 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java @@ -2,7 +2,7 @@ package com.magamochi.mangamochi.service; import com.google.common.util.concurrent.RateLimiter; import com.magamochi.mangamochi.client.JikanClient; -import com.magamochi.mangamochi.client.RapidFuzzClient; +import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO; import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand; import com.magamochi.mangamochi.model.entity.Manga; import com.magamochi.mangamochi.model.entity.MangaImportReview; @@ -21,8 +21,9 @@ public class MangaCreationService { private final MangaRepository mangaRepository; private final MangaImportReviewRepository mangaImportReviewRepository; + private final TitleMatcherService titleMatcherService; + private final JikanClient jikanClient; - private final RapidFuzzClient rapidFuzzClient; private final RateLimiter jikanRateLimiter; @@ -42,18 +43,20 @@ public class MangaCreationService { return null; } - var request = - new RapidFuzzClient.Request( - title, - jikanResults.stream() - .flatMap( - results -> - results.titles().stream() - .map(JikanClient.SearchResponse.MangaData.TitleData::title)) - .toList()); + var titleMatchResponse = + titleMatcherService.findBestMatch( + TitleMatchRequestDTO.builder() + .title(title) + .options( + jikanResults.stream() + .flatMap( + results -> + results.titles().stream() + .map(JikanClient.SearchResponse.MangaData.TitleData::title)) + .toList()) + .build()); - var fuzzResults = rapidFuzzClient.mangaSearch(request); - if (!fuzzResults.match_found()) { + if (!titleMatchResponse.isMatchFound()) { createMangaImportReview(title, url, provider); log.warn("No match found for manga with title {}", title); return null; @@ -66,7 +69,7 @@ public class MangaCreationService { results.titles().stream() .map(JikanClient.SearchResponse.MangaData.TitleData::title) .toList() - .contains(fuzzResults.best_match())) + .contains(titleMatchResponse.getBestMatch())) .findFirst(); if (resultOptional.isEmpty()) { createMangaImportReview(title, url, provider); diff --git a/src/main/java/com/magamochi/mangamochi/service/TitleMatcherService.java b/src/main/java/com/magamochi/mangamochi/service/TitleMatcherService.java new file mode 100644 index 0000000..de6ab3a --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/TitleMatcherService.java @@ -0,0 +1,67 @@ +package com.magamochi.mangamochi.service; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.springframework.util.CollectionUtils.isEmpty; + +import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO; +import com.magamochi.mangamochi.model.dto.TitleMatchResponseDTO; +import lombok.extern.log4j.Log4j2; +import org.apache.commons.text.similarity.LevenshteinDistance; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +public class TitleMatcherService { + private final LevenshteinDistance levenshteinDistance = LevenshteinDistance.getDefaultInstance(); + + public TitleMatchResponseDTO findBestMatch(TitleMatchRequestDTO request) { + if (isBlank(request.getTitle()) || isEmpty(request.getOptions())) { + throw new IllegalArgumentException("Title and options are required"); + } + + log.info("Finding best match for {}. Options: {}", request.getTitle(), request.getOptions()); + + String bestMatch = null; + double bestScore = 0.0; + + for (var option : request.getOptions()) { + var score = calculateSimilarityScore(request.getTitle(), option); + + if (score > bestScore) { + bestScore = score; + bestMatch = option; + } + } + + if (bestScore >= request.getThreshold()) { + log.info( + "Found best match for {}: {}. Similarity: {}", request.getTitle(), bestMatch, bestScore); + + return TitleMatchResponseDTO.builder() + .matchFound(true) + .bestMatch(bestMatch) + .similarity(bestScore) + .build(); + } + + log.info("No match found for {}. Threshold: {}", request.getTitle(), request.getThreshold()); + + return TitleMatchResponseDTO.builder().matchFound(false).build(); + } + + private double calculateSimilarityScore(String title, String option) { + var dist = levenshteinDistance.apply(title, option); + + var maxLength = Math.max(title.length(), option.length()); + if (maxLength == 0) { + return 100.0; + } + + // Calculate similarity: 100 * (1 - (distance / max_length)) + // This scales the distance into a percentage. + var similarity = 100.0 * (1.0 - ((double) dist / maxLength)); + + // Format to two decimal places for a cleaner result + return Math.round(similarity * 100.0) / 100.0; + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java b/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java index 2981df7..c7f347a 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java @@ -1,7 +1,7 @@ package com.magamochi.mangamochi.service.providers; public class ContentProviders { - public static final String MANGA_LIVRE = "Manga Livre"; + public static final String MANGA_LIVRE_TO = "Manga Livre.to"; public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog"; public static final String MANGA_DEX = "MangaDex"; public static final String PINK_ROSA_SCAN = "Pink Rosa Scan"; diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java index 96b28c2..7533d3d 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java @@ -1,5 +1,7 @@ package com.magamochi.mangamochi.service.providers.impl; +import static java.util.Objects.nonNull; + import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; import com.magamochi.mangamochi.model.entity.MangaProvider; @@ -16,10 +18,11 @@ import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; @Log4j2 -@Service(ContentProviders.MANGA_LIVRE) +@Service(ContentProviders.MANGA_LIVRE_TO) @RequiredArgsConstructor public class MangaLivreProvider implements ContentProvider, PagedContentProvider { - private final String url = "https://mangalivre.tv/manga/"; + private final String url = "https://mangalivre.to/manga/"; + private final int MANGAS_PER_PAGE = 10; private final FlareService flareService; @@ -27,22 +30,27 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider public List getAvailableChapters(MangaProvider provider) { log.info( "Getting available chapters from {}, manga {}", - ContentProviders.MANGA_LIVRE, + ContentProviders.MANGA_LIVRE_TO, provider.getManga().getTitle()); try { var document = - flareService.getContentAsJsoupDocument(provider.getUrl(), ContentProviders.MANGA_LIVRE); + flareService.getContentAsJsoupDocument( + provider.getUrl(), ContentProviders.MANGA_LIVRE_TO); - var chapterItems = document.getElementsByClass("wp-manga-chapter"); + var chapterBoxes = + document.selectFirst("div.listing-chapters-wrap").select("div.chapter-box"); - return chapterItems.stream() + return chapterBoxes.stream() .map( - chapterItemElement -> { - var linkElement = chapterItemElement.getElementsByTag("a").getFirst(); + chapterBox -> { + var linkElement = chapterBox.selectFirst("a"); + + var url = linkElement.attr("href"); + var title = linkElement.text(); return new ContentProviderMangaChapterResponseDTO( - linkElement.text(), linkElement.attr("href"), null, null); + title.trim(), url.trim(), null, null); }) .toList(); } catch (NoSuchElementException e) { @@ -53,21 +61,19 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider @Override public Map getChapterImagesUrls(String chapterUrl) { - log.info("Getting images from {}, url {}", ContentProviders.MANGA_LIVRE, chapterUrl); + log.info("Getting images from {}, url {}", ContentProviders.MANGA_LIVRE_TO, chapterUrl); try { var document = - flareService.getContentAsJsoupDocument(chapterUrl, ContentProviders.MANGA_LIVRE); + flareService.getContentAsJsoupDocument(chapterUrl, ContentProviders.MANGA_LIVRE_TO); - var chapterImagesContainer = document.getElementsByClass("chapter-images").getFirst(); - var chapterImagesElements = chapterImagesContainer.getElementsByClass("page-break"); + var chapterImagesElements = document.select("div.reading-content img.wp-manga-chapter-img"); var imageUrls = chapterImagesElements.stream() .map( chapterImagesElement -> { - var imageElement = chapterImagesElement.getElementsByTag("img").getFirst(); - return imageElement.attr("src"); + return chapterImagesElement.attr("src"); }) .toList(); @@ -77,54 +83,40 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider Collectors.toMap( i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new)); } catch (NoSuchElementException e) { - log.error("Error parsing mangas from MangaLivre", e); + log.error("Error parsing manga images from MangaLivre", e); return Map.of(); } } @Override public List getMangasFromPage(Integer page) { - log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE, page); + log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_TO, page); try { var document = flareService.getContentAsJsoupDocument( - url + "page/" + page, ContentProviders.MANGA_LIVRE); + url + "page/" + page, ContentProviders.MANGA_LIVRE_TO); - var mangaElements = document.getElementsByClass("manga__item"); + var mangaElements = document.select("div.page-item-detail.manga"); return mangaElements.stream() .map( element -> { - var mangaTitleElement = - element - .getElementsByClass("manga__content") - .getFirst() - .getElementsByClass("manga__content_item") - .getFirst() - .getElementsByClass("post-title font-title") - .getFirst() - .getElementsByTag("h2") - .getFirst(); - - var linkElement = mangaTitleElement.getElementsByTag("a").getFirst(); + var linkElement = element.selectFirst(".item-thumb > a"); var url = linkElement.attr("href"); - var title = linkElement.text().trim(); - var imageElement = - element - .getElementsByClass("manga__thumb") - .getFirst() - .getElementsByClass("manga__thumb_item") - .getFirst() - .getElementsByTag("a") - .getFirst() - .getElementsByTag("img") - .getFirst(); - var imgUrl = imageElement.attr("src"); + var title = linkElement.attr("title"); + + // Fallback: If 'title' attribute is empty, try the tag's 'alt' or 'title' + if (title.isBlank()) { + var imgElement = linkElement.selectFirst("img"); + if (nonNull(imgElement)) { + title = imgElement.attr("alt"); + } + } return new ContentProviderMangaInfoResponseDTO( - title, url, imgUrl, MangaStatus.UNKNOWN); + title.trim(), url.trim(), null, MangaStatus.UNKNOWN); }) .toList(); } catch (NoSuchElementException e) { @@ -135,17 +127,16 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider @Override public Integer getTotalPages() { - log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE); + log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_TO); try { - var document = flareService.getContentAsJsoupDocument(url, ContentProviders.MANGA_LIVRE); + var document = flareService.getContentAsJsoupDocument(url, ContentProviders.MANGA_LIVRE_TO); - var navLinks = document.getElementsByClass("wp-pagenavi").getFirst(); - var lastPageElement = navLinks.getElementsByClass("last").getFirst(); - var links = lastPageElement.attr("href"); + var fullText = document.selectFirst("div.h4:contains(resultados)").text(); + var numberString = fullText.replace("resultados", "").trim().replaceAll("[^0-9]", ""); + var totalMangas = Integer.parseInt(numberString); - var totalPages = links.replaceAll("\\D+", ""); - return Integer.parseInt(totalPages); + return (int) Math.ceil((double) totalMangas / MANGAS_PER_PAGE); } catch (NoSuchElementException e) { log.error("Error parsing total pages from MangaLivre", e); return null; diff --git a/src/main/java/com/magamochi/mangamochi/task/MangaFollowUpdateTask.java b/src/main/java/com/magamochi/mangamochi/task/MangaFollowUpdateTask.java index 0bac214..8e67714 100644 --- a/src/main/java/com/magamochi/mangamochi/task/MangaFollowUpdateTask.java +++ b/src/main/java/com/magamochi/mangamochi/task/MangaFollowUpdateTask.java @@ -2,6 +2,7 @@ package com.magamochi.mangamochi.task; import com.magamochi.mangamochi.model.dto.UpdateMangaFollowChapterListCommand; import com.magamochi.mangamochi.model.entity.Manga; +import com.magamochi.mangamochi.model.enumeration.ProviderStatus; import com.magamochi.mangamochi.model.repository.MangaRepository; import com.magamochi.mangamochi.queue.UpdateMangaFollowChapterListProducer; import lombok.RequiredArgsConstructor; @@ -43,10 +44,14 @@ public class MangaFollowUpdateTask { log.info("Fetching available mangas for followed Manga {}", manga.getTitle()); var mangaProviders = manga.getMangaProviders(); - mangaProviders.forEach( - mangaProvider -> - producer.sendUpdateMangaFollowChapterListCommand( - new UpdateMangaFollowChapterListCommand(mangaProvider.getId()))); + + mangaProviders.stream() + .filter( + mangaProvider -> mangaProvider.getProvider().getStatus().equals(ProviderStatus.ACTIVE)) + .forEach( + mangaProvider -> + producer.sendUpdateMangaFollowChapterListCommand( + new UpdateMangaFollowChapterListCommand(mangaProvider.getId()))); log.info("Followed Manga ({}) chapter list update queued.", manga.getTitle()); } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index a98a321..d68da5c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -53,9 +53,6 @@ jwt: refresh-secret: MIV9ctIwrImmrZBjh9QueNEcDOLLVv9Rephii+0DKbk= refresh-expiration: 2629746000 -manga-matcher: - endpoint: ${MANGAMATCHER_ENDPOINT} - resilience4j: retry: instances: @@ -78,6 +75,13 @@ resilience4j: retry-exceptions: - feign.FeignException +rabbit-mq: + queues: + manga-data-update: ${MANGA_DATA_UPDATE_QUEUE:mangaDataUpdateQueue} + manga-chapter-download: ${MANGA_CHAPTER_DOWNLOAD_QUEUE:mangaChapterDownloadQueue} + manga-list-update: ${MANGA_LIST_UPDATE_QUEUE:mangaListUpdateQueue} + manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangaFollowUpdateChapterQueue} + image-service: clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false} cron-expression: "@weekly" diff --git a/src/main/resources/db/migration/V0017__MANGA_LIVRE_CLEANUP.sql b/src/main/resources/db/migration/V0017__MANGA_LIVRE_CLEANUP.sql new file mode 100644 index 0000000..3530988 --- /dev/null +++ b/src/main/resources/db/migration/V0017__MANGA_LIVRE_CLEANUP.sql @@ -0,0 +1,43 @@ +DO +$$ + DECLARE + _target_provider_name TEXT := 'Manga Livre'; + _target_provider_id BIGINT; + _deleted_chapters_count INTEGER; + _deleted_manga_providers_count INTEGER; + + BEGIN + SELECT id + INTO _target_provider_id + FROM providers + WHERE name = _target_provider_name; + + IF _target_provider_id IS NULL THEN + RAISE EXCEPTION 'Provider with name "%" not found.', _target_provider_name; + END IF; + + UPDATE providers SET status = 'INACTIVE' WHERE id = _target_provider_id; + + -- Delete non-downloaded manga chapters associated with the target provider + DELETE + FROM manga_chapters mc + USING manga_provider mp + WHERE mc.manga_provider_id = mp.id + AND mp.provider_id = _target_provider_id + AND mc.downloaded = FALSE; + + GET DIAGNOSTICS _deleted_chapters_count = ROW_COUNT; + RAISE NOTICE 'Deleted % non-downloaded chapters for provider ID %.', _deleted_chapters_count, _target_provider_id; + + -- Delete MangaProvider records ONLY if NO chapters + DELETE + FROM manga_provider mp + WHERE mp.provider_id = _target_provider_id + AND NOT EXISTS (SELECT 1 + FROM manga_chapters mc + WHERE mc.manga_provider_id = mp.id); + + GET DIAGNOSTICS _deleted_manga_providers_count = ROW_COUNT; + RAISE NOTICE 'Deleted % manga_provider entries', _deleted_manga_providers_count; + END +$$; \ No newline at end of file