diff --git a/src/main/java/com/magamochi/common/ContentProviders.java b/src/main/java/com/magamochi/common/ContentProviders.java index 59a4b3b..52be56c 100644 --- a/src/main/java/com/magamochi/common/ContentProviders.java +++ b/src/main/java/com/magamochi/common/ContentProviders.java @@ -7,4 +7,5 @@ public class ContentProviders { public static final String TAIMU = "Taimu"; public static final String MANGA_DEX = "MangaDex"; public static final String MANUAL_IMPORT = "Manual Import"; + public static final String ROLIA_SCAN = "Rolia Scan"; } diff --git a/src/main/java/com/magamochi/common/config/RateLimiterConfig.java b/src/main/java/com/magamochi/common/config/RateLimiterConfig.java index ade59f9..948022e 100644 --- a/src/main/java/com/magamochi/common/config/RateLimiterConfig.java +++ b/src/main/java/com/magamochi/common/config/RateLimiterConfig.java @@ -16,6 +16,11 @@ public class RateLimiterConfig { return RateLimiter.create(1); } + @Bean + public RateLimiter roliaScanRateLimiter() { + return RateLimiter.create(1); + } + @Bean public RateLimiter aniListRateLimiter() { return RateLimiter.create(0.5); diff --git a/src/main/java/com/magamochi/image/queue/consumer/ImageFetchConsumer.java b/src/main/java/com/magamochi/image/queue/consumer/ImageFetchConsumer.java index 6bfc5bf..1490f39 100644 --- a/src/main/java/com/magamochi/image/queue/consumer/ImageFetchConsumer.java +++ b/src/main/java/com/magamochi/image/queue/consumer/ImageFetchConsumer.java @@ -1,11 +1,11 @@ package com.magamochi.image.queue.consumer; import com.magamochi.common.exception.ExcludedImageException; +import com.magamochi.common.model.enumeration.ContentType; import com.magamochi.common.queue.command.ImageFetchCommand; import com.magamochi.common.queue.command.ImageUpdateCommand; import com.magamochi.common.queue.command.MangaContentImageDeleteCommand; import com.magamochi.image.queue.producer.ImageUpdateProducer; -import com.magamochi.common.model.enumeration.ContentType; import com.magamochi.image.service.ImageFetchService; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -19,7 +19,9 @@ public class ImageFetchConsumer { private final ImageFetchService imageFetchService; private final ImageUpdateProducer imageUpdateProducer; - @RabbitListener(queues = "${queues.image-fetch}", concurrency = "${queues.image-fetch-concurrency:5-10}") + @RabbitListener( + queues = "${queues.image-fetch}", + concurrency = "${queues.image-fetch-concurrency:5-10}") public void receiveImageFetchCommand(ImageFetchCommand command) { log.info("Received image fetch command: {}", command); diff --git a/src/main/java/com/magamochi/image/queue/producer/ImageUpdateProducer.java b/src/main/java/com/magamochi/image/queue/producer/ImageUpdateProducer.java index 9be1b45..50b114e 100644 --- a/src/main/java/com/magamochi/image/queue/producer/ImageUpdateProducer.java +++ b/src/main/java/com/magamochi/image/queue/producer/ImageUpdateProducer.java @@ -30,7 +30,8 @@ public class ImageUpdateProducer { rabbitTemplate.convertAndSend(imageUpdatesTopic, routingKey, command); } - public void publishImageDeleteCommand(MangaContentImageDeleteCommand command, ContentType contentType) { + public void publishImageDeleteCommand( + MangaContentImageDeleteCommand command, ContentType contentType) { var routingKey = String.format(imageDeleteRoutingKey + ".%s", contentType.name().toLowerCase()); rabbitTemplate.convertAndSend(imageUpdatesTopic, routingKey, command); diff --git a/src/main/java/com/magamochi/ingestion/client/RoliaScanClient.java b/src/main/java/com/magamochi/ingestion/client/RoliaScanClient.java new file mode 100644 index 0000000..35f682f --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/client/RoliaScanClient.java @@ -0,0 +1,101 @@ +package com.magamochi.ingestion.client; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.resilience.annotation.Retryable; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; + +@FeignClient(name = "roliaScan", url = "https://roliascan.com") +public interface RoliaScanClient { + + @PostMapping("/wp-json/manga/v1/load") + @Retryable( + maxRetries = 3, + delay = 1000, + multiplier = 2, + maxDelay = 5000, + includes = Exception.class) + List loadMangas(@RequestBody RoliaScanMangaRequest request); + + @GetMapping("/auth/manga-chapters") + @Retryable( + maxRetries = 3, + delay = 1000, + multiplier = 2, + maxDelay = 5000, + includes = Exception.class) + RoliaScanChaptersResponse getChapters( + @RequestParam("manga_id") String mangaId, + @RequestParam int offset, + @RequestParam int limit, + @RequestParam String order, + @RequestParam("_t") String token, + @RequestParam("_ts") long timestamp); + + @GetMapping("/auth/chapter-content") + @Retryable( + maxRetries = 3, + delay = 1000, + multiplier = 2, + maxDelay = 5000, + includes = Exception.class) + RoliaScanChapterContentResponse getChapterContent(@RequestParam("chapter_id") String chapterId); + + record RoliaScanMangaRequest( + Integer page, + String search, + String years, + String genres, + String types, + String statuses, + String sort, + String genreMatchMode) { + public RoliaScanMangaRequest(Integer page) { + this(page, "", "[]", "[]", "[]", "[]", "post_desc", "any"); + } + } + + record RoliaScanMangaItem( + String id, + String title, + String url, + String cover, + String score, + Integer votes, + String status, + String year, + String type, + String description) {} + + record RoliaScanChaptersResponse( + boolean success, + List chapters, + int total, + int offset, + int limit, + @JsonProperty("has_more") boolean hasMore) {} + + record RoliaScanChapterItem( + String id, + String chapter, + String title, + String date, + @JsonProperty("chapter_type") String chapterType, + @JsonProperty("group_id") String groupId, + String language, + @JsonProperty("group_name") String groupName, + @JsonProperty("group_avatar") String groupAvatar, + String likes, + String url) {} + + record RoliaScanChapterContentResponse( + boolean success, + @JsonProperty("chapter_id") String chapterId, + @JsonProperty("chapter_type") String chapterType, + List images, + int total) {} +} diff --git a/src/main/java/com/magamochi/ingestion/providers/impl/RoliaScanProvider.java b/src/main/java/com/magamochi/ingestion/providers/impl/RoliaScanProvider.java new file mode 100644 index 0000000..cf0c0cf --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/providers/impl/RoliaScanProvider.java @@ -0,0 +1,212 @@ +package com.magamochi.ingestion.providers.impl; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +import com.google.common.util.concurrent.RateLimiter; +import com.magamochi.catalog.model.entity.MangaContentProvider; +import com.magamochi.common.ContentProviders; +import com.magamochi.ingestion.client.RoliaScanClient; +import com.magamochi.ingestion.client.RoliaScanClient.RoliaScanMangaRequest; +import com.magamochi.ingestion.model.dto.ContentImageInfoDTO; +import com.magamochi.ingestion.model.dto.ContentInfoDTO; +import com.magamochi.ingestion.model.dto.MangaInfoDTO; +import com.magamochi.ingestion.providers.ContentProvider; +import com.magamochi.ingestion.providers.PagedContentProvider; +import com.magamochi.ingestion.service.FlareService; +import com.magamochi.ingestion.service.RoliaScanAuthService; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.IntStream; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service(ContentProviders.ROLIA_SCAN) +@RequiredArgsConstructor +public class RoliaScanProvider implements ContentProvider, PagedContentProvider { + private final FlareService flareService; + private final RoliaScanClient roliaScanClient; + private final RoliaScanAuthService roliaScanAuthService; + private final RateLimiter roliaScanRateLimiter; + + @Override + public List getAvailableChapters(MangaContentProvider provider) { + log.info( + "Getting available chapters from {}, manga {}", + ContentProviders.ROLIA_SCAN, + provider.getManga().getTitle()); + + try { + var mangaId = extractMangaId(provider.getUrl()); + + log.info("Extracted manga id {} from {}", mangaId, provider.getUrl()); + + var auth = roliaScanAuthService.generateToken(); + + return fetchAllChapters(mangaId, auth.token(), auth.timestamp()); + } catch (NoSuchElementException e) { + log.error("Error parsing chapters from Rolia Scan", e); + return List.of(); + } catch (Exception e) { + log.error("Error fetching chapters from Rolia Scan API", e); + return List.of(); + } + } + + private String extractMangaId(String mangaUrl) { + var document = flareService.getContentAsJsoupDocument(mangaUrl, ContentProviders.ROLIA_SCAN); + + var mangaId = document.body().attr("data-manga-id"); + + if (isBlank(mangaId)) { + throw new NoSuchElementException("data-manga-id not found in page body"); + } + + return mangaId; + } + + private List fetchAllChapters(String mangaId, String token, long timestamp) { + log.info("Fetching all chapters for manga {}", mangaId); + + var allChapters = new ArrayList(); + + int offset = 0; + boolean hasMore = true; + int page = 0; + + while (hasMore) { + page++; + + log.info("Fetching chapters for manga {}, page {}, offset {}", mangaId, page, offset); + + roliaScanRateLimiter.acquire(); + var response = roliaScanClient.getChapters(mangaId, offset, 500, "DESC", token, timestamp); + + log.info( + "Page {} fetched {} chapters for manga {} (offset {}, total {}, hasMore {})", + page, + response.chapters().size(), + mangaId, + offset, + response.total(), + response.hasMore()); + + response.chapters().stream().map(this::toContentInfo).forEach(allChapters::add); + + hasMore = response.hasMore(); + offset += 500; + } + + log.info( + "Fetched total {} chapters for manga {} across {} pages", + allChapters.size(), + mangaId, + page); + + return allChapters; + } + + private ContentInfoDTO toContentInfo(RoliaScanClient.RoliaScanChapterItem chapter) { + var title = + isBlank(chapter.title()) || "N/A".equals(chapter.title()) + ? "Ch. " + chapter.chapter() + : chapter.chapter() + " - " + chapter.title(); + + return new ContentInfoDTO(title, chapter.url(), mapLanguage(chapter.language())); + } + + private static String mapLanguage(String language) { + return switch (language) { + case "en" -> "en-US"; + case "pt" -> "pt-BR"; + default -> language; + }; + } + + @Override + public List getContentImages(String chapterUrl) { + log.info("Getting content images from {}, url {}", ContentProviders.ROLIA_SCAN, chapterUrl); + + try { + var chapterId = extractChapterId(chapterUrl); + + log.info("Extracted chapter id {} from url {}", chapterId, chapterUrl); + + roliaScanRateLimiter.acquire(); + var response = roliaScanClient.getChapterContent(chapterId); + + log.info("Fetched {} images for chapter {}", response.images().size(), chapterId); + + return IntStream.range(0, response.images().size()) + .boxed() + .map(position -> new ContentImageInfoDTO(position, response.images().get(position))) + .toList(); + } catch (Exception e) { + log.error("Error fetching chapter images from Rolia Scan API, url {}", chapterUrl, e); + return List.of(); + } + } + + private String extractChapterId(String chapterUrl) { + var lastDash = chapterUrl.lastIndexOf('-'); + if (lastDash < 0) { + throw new NoSuchElementException("Invalid chapter URL format: " + chapterUrl); + } + return chapterUrl.substring(lastDash + 1); + } + + @Override + public int getTotalPages() { + return 1; + } + + @Override + public List getMangasFromPage(int page) { + log.info("Getting mangas from {}, starting at page {}", ContentProviders.ROLIA_SCAN, page); + + try { + var allMangas = new ArrayList(); + + int currentPage = page; + while (true) { + log.info("Fetching mangas from {}, page {}", ContentProviders.ROLIA_SCAN, currentPage); + + roliaScanRateLimiter.acquire(); + var items = roliaScanClient.loadMangas(new RoliaScanMangaRequest(currentPage)); + + var filteredItems = + items.stream().filter(item -> !"Novel".equalsIgnoreCase(item.type())).toList(); + + log.info( + "Page {} returned {} mangas ({} novels filtered out)", + currentPage, + filteredItems.size(), + items.size() - filteredItems.size()); + + if (filteredItems.isEmpty()) { + log.info("No more mangas on page {}, stopping pagination", currentPage); + break; + } + + filteredItems.stream() + .map(item -> new MangaInfoDTO(item.title(), item.url())) + .forEach(allMangas::add); + + currentPage++; + } + + log.info( + "Loaded {} mangas from {} across {} pages", + allMangas.size(), + ContentProviders.ROLIA_SCAN, + currentPage - page); + + return allMangas; + } catch (Exception e) { + log.error("Error loading mangas from Rolia Scan API, page {}", page, e); + return List.of(); + } + } +} diff --git a/src/main/java/com/magamochi/ingestion/service/RoliaScanAuthService.java b/src/main/java/com/magamochi/ingestion/service/RoliaScanAuthService.java new file mode 100644 index 0000000..36b7661 --- /dev/null +++ b/src/main/java/com/magamochi/ingestion/service/RoliaScanAuthService.java @@ -0,0 +1,42 @@ +package com.magamochi.ingestion.service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import org.springframework.stereotype.Service; + +@Service +public class RoliaScanAuthService { + + public AuthToken generateToken() { + var timestamp = Instant.now().getEpochSecond(); + var hour = getCurrentHourUtc(); + var secret = "mng_ch_" + hour; + var hash = md5(timestamp + secret).substring(0, 16); + return new AuthToken(hash, timestamp); + } + + private String getCurrentHourUtc() { + return LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyyMMddHH")); + } + + private String md5(String input) { + try { + var digest = MessageDigest.getInstance("MD5"); + var hashBytes = digest.digest(input.getBytes(StandardCharsets.UTF_8)); + var hexString = new StringBuilder(); + for (byte b : hashBytes) { + hexString.append(String.format("%02x", b)); + } + return hexString.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("MD5 algorithm not available", e); + } + } + + public record AuthToken(String token, long timestamp) {} +} diff --git a/src/main/resources/db/migration/V0014__ROLIA_PROVIDER.sql b/src/main/resources/db/migration/V0014__ROLIA_PROVIDER.sql new file mode 100644 index 0000000..6330027 --- /dev/null +++ b/src/main/resources/db/migration/V0014__ROLIA_PROVIDER.sql @@ -0,0 +1,2 @@ +INSERT INTO content_providers(name, url, active, supports_content_fetch, manual_import) +VALUES ('Rolia Scan', 'https://roliascan.com', TRUE, TRUE, FALSE); \ No newline at end of file