213 lines
6.7 KiB
Java
213 lines
6.7 KiB
Java
package com.magamochi.ingestion.providers.impl;
|
|
|
|
import static org.apache.commons.lang3.StringUtils.isBlank;
|
|
|
|
import com.google.common.util.concurrent.RateLimiter;
|
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
|
import com.magamochi.common.ContentProviders;
|
|
import com.magamochi.ingestion.client.RoliaScanClient;
|
|
import com.magamochi.ingestion.client.RoliaScanClient.RoliaScanMangaRequest;
|
|
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
|
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
|
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
|
import com.magamochi.ingestion.providers.ContentProvider;
|
|
import com.magamochi.ingestion.providers.PagedContentProvider;
|
|
import com.magamochi.ingestion.service.FlareService;
|
|
import com.magamochi.ingestion.service.RoliaScanAuthService;
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import java.util.NoSuchElementException;
|
|
import java.util.stream.IntStream;
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.extern.log4j.Log4j2;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
@Log4j2
|
|
@Service(ContentProviders.ROLIA_SCAN)
|
|
@RequiredArgsConstructor
|
|
public class RoliaScanProvider implements ContentProvider, PagedContentProvider {
|
|
private final FlareService flareService;
|
|
private final RoliaScanClient roliaScanClient;
|
|
private final RoliaScanAuthService roliaScanAuthService;
|
|
private final RateLimiter roliaScanRateLimiter;
|
|
|
|
@Override
|
|
public List<ContentInfoDTO> getAvailableChapters(MangaContentProvider provider) {
|
|
log.info(
|
|
"Getting available chapters from {}, manga {}",
|
|
ContentProviders.ROLIA_SCAN,
|
|
provider.getManga().getTitle());
|
|
|
|
try {
|
|
var mangaId = extractMangaId(provider.getUrl());
|
|
|
|
log.info("Extracted manga id {} from {}", mangaId, provider.getUrl());
|
|
|
|
var auth = roliaScanAuthService.generateToken();
|
|
|
|
return fetchAllChapters(mangaId, auth.token(), auth.timestamp());
|
|
} catch (NoSuchElementException e) {
|
|
log.error("Error parsing chapters from Rolia Scan", e);
|
|
return List.of();
|
|
} catch (Exception e) {
|
|
log.error("Error fetching chapters from Rolia Scan API", e);
|
|
return List.of();
|
|
}
|
|
}
|
|
|
|
private String extractMangaId(String mangaUrl) {
|
|
var document = flareService.getContentAsJsoupDocument(mangaUrl, ContentProviders.ROLIA_SCAN);
|
|
|
|
var mangaId = document.body().attr("data-manga-id");
|
|
|
|
if (isBlank(mangaId)) {
|
|
throw new NoSuchElementException("data-manga-id not found in page body");
|
|
}
|
|
|
|
return mangaId;
|
|
}
|
|
|
|
private List<ContentInfoDTO> fetchAllChapters(String mangaId, String token, long timestamp) {
|
|
log.info("Fetching all chapters for manga {}", mangaId);
|
|
|
|
var allChapters = new ArrayList<ContentInfoDTO>();
|
|
|
|
int offset = 0;
|
|
boolean hasMore = true;
|
|
int page = 0;
|
|
|
|
while (hasMore) {
|
|
page++;
|
|
|
|
log.info("Fetching chapters for manga {}, page {}, offset {}", mangaId, page, offset);
|
|
|
|
roliaScanRateLimiter.acquire();
|
|
var response = roliaScanClient.getChapters(mangaId, offset, 500, "DESC", token, timestamp);
|
|
|
|
log.info(
|
|
"Page {} fetched {} chapters for manga {} (offset {}, total {}, hasMore {})",
|
|
page,
|
|
response.chapters().size(),
|
|
mangaId,
|
|
offset,
|
|
response.total(),
|
|
response.hasMore());
|
|
|
|
response.chapters().stream().map(this::toContentInfo).forEach(allChapters::add);
|
|
|
|
hasMore = response.hasMore();
|
|
offset += 500;
|
|
}
|
|
|
|
log.info(
|
|
"Fetched total {} chapters for manga {} across {} pages",
|
|
allChapters.size(),
|
|
mangaId,
|
|
page);
|
|
|
|
return allChapters;
|
|
}
|
|
|
|
private ContentInfoDTO toContentInfo(RoliaScanClient.RoliaScanChapterItem chapter) {
|
|
var title =
|
|
isBlank(chapter.title()) || "N/A".equals(chapter.title())
|
|
? "Ch. " + chapter.chapter()
|
|
: chapter.chapter() + " - " + chapter.title();
|
|
|
|
return new ContentInfoDTO(title, chapter.url(), mapLanguage(chapter.language()));
|
|
}
|
|
|
|
private static String mapLanguage(String language) {
|
|
return switch (language) {
|
|
case "en" -> "en-US";
|
|
case "pt" -> "pt-BR";
|
|
default -> language;
|
|
};
|
|
}
|
|
|
|
@Override
|
|
public List<ContentImageInfoDTO> getContentImages(String chapterUrl) {
|
|
log.info("Getting content images from {}, url {}", ContentProviders.ROLIA_SCAN, chapterUrl);
|
|
|
|
try {
|
|
var chapterId = extractChapterId(chapterUrl);
|
|
|
|
log.info("Extracted chapter id {} from url {}", chapterId, chapterUrl);
|
|
|
|
roliaScanRateLimiter.acquire();
|
|
var response = roliaScanClient.getChapterContent(chapterId);
|
|
|
|
log.info("Fetched {} images for chapter {}", response.images().size(), chapterId);
|
|
|
|
return IntStream.range(0, response.images().size())
|
|
.boxed()
|
|
.map(position -> new ContentImageInfoDTO(position, response.images().get(position)))
|
|
.toList();
|
|
} catch (Exception e) {
|
|
log.error("Error fetching chapter images from Rolia Scan API, url {}", chapterUrl, e);
|
|
return List.of();
|
|
}
|
|
}
|
|
|
|
private String extractChapterId(String chapterUrl) {
|
|
var lastDash = chapterUrl.lastIndexOf('-');
|
|
if (lastDash < 0) {
|
|
throw new NoSuchElementException("Invalid chapter URL format: " + chapterUrl);
|
|
}
|
|
return chapterUrl.substring(lastDash + 1);
|
|
}
|
|
|
|
@Override
|
|
public int getTotalPages() {
|
|
return 1;
|
|
}
|
|
|
|
@Override
|
|
public List<MangaInfoDTO> getMangasFromPage(int page) {
|
|
log.info("Getting mangas from {}, starting at page {}", ContentProviders.ROLIA_SCAN, page);
|
|
|
|
try {
|
|
var allMangas = new ArrayList<MangaInfoDTO>();
|
|
|
|
int currentPage = page;
|
|
while (true) {
|
|
log.info("Fetching mangas from {}, page {}", ContentProviders.ROLIA_SCAN, currentPage);
|
|
|
|
roliaScanRateLimiter.acquire();
|
|
var items = roliaScanClient.loadMangas(new RoliaScanMangaRequest(currentPage));
|
|
|
|
var filteredItems =
|
|
items.stream().filter(item -> !"Novel".equalsIgnoreCase(item.type())).toList();
|
|
|
|
log.info(
|
|
"Page {} returned {} mangas ({} novels filtered out)",
|
|
currentPage,
|
|
filteredItems.size(),
|
|
items.size() - filteredItems.size());
|
|
|
|
if (filteredItems.isEmpty()) {
|
|
log.info("No more mangas on page {}, stopping pagination", currentPage);
|
|
break;
|
|
}
|
|
|
|
filteredItems.stream()
|
|
.map(item -> new MangaInfoDTO(item.title(), item.url()))
|
|
.forEach(allMangas::add);
|
|
|
|
currentPage++;
|
|
}
|
|
|
|
log.info(
|
|
"Loaded {} mangas from {} across {} pages",
|
|
allMangas.size(),
|
|
ContentProviders.ROLIA_SCAN,
|
|
currentPage - page);
|
|
|
|
return allMangas;
|
|
} catch (Exception e) {
|
|
log.error("Error loading mangas from Rolia Scan API, page {}", page, e);
|
|
return List.of();
|
|
}
|
|
}
|
|
}
|