mangamochi/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(taimu): update Taimu API integration and URL

Browse Source
This commit is contained in:
Rodrigo Verdiani 2026-06-23 20:51:48 -03:00
parent 8043761bce
commit 119fe51b3f
8 changed files with 140 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/magamochi/common/config/RateLimiterConfig.java
View File

@ -21,6 +21,11 @@ public class RateLimiterConfig {
return RateLimiter.create(1);
}
@Bean
public RateLimiter taimuRateLimiter() {
return RateLimiter.create(1);
}
@Bean
public RateLimiter aniListRateLimiter() {
return RateLimiter.create(0.5);

24
src/main/java/com/magamochi/ingestion/client/ScrollableScrapperClient.java
View File

@ -1,24 +0,0 @@
package com.magamochi.ingestion.client;
import lombok.Builder;
import lombok.Getter;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@FeignClient(name = "scrollable-scrapper", url = "${scrollable-scrapper.endpoint}")
public interface ScrollableScrapperClient {
@PostMapping(
consumes = MediaType.APPLICATION_JSON_VALUE,
produces = MediaType.APPLICATION_JSON_VALUE)
GetResponse get(@RequestBody GetRequest request);
@Getter
@Builder
class GetRequest {
private final String url;
}
record GetResponse(String pageSource) {}
}

73
src/main/java/com/magamochi/ingestion/client/TaimuClient.java Normal file
View File

@ -0,0 +1,73 @@
package com.magamochi.ingestion.client;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.resilience.annotation.Retryable;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
@FeignClient(name = "taimu", url = "https://apiv2.taimumangas.com")
public interface TaimuClient {
@GetMapping("/api/v1/reader/library")
@Retryable(
maxRetries = 3,
delay = 1000,
multiplier = 2,
maxDelay = 5000,
includes = Exception.class)
TaimuLibraryResponse getLibrary(
@RequestParam int page,
@RequestParam("per_page") int perPage,
@RequestParam String sort,
@RequestParam("adult") boolean adult);
record TaimuLibraryResponse(
List<TaimuMangaItem> items, int total, int page, @JsonProperty("per_page") int perPage) {}
record TaimuMangaItem(
String title,
String identifier,
String cover,
@JsonProperty("chapter_count") int chapterCount,
String status,
String type,
@JsonProperty("release_year") Integer releaseYear) {}
@GetMapping("/api/v1/reader/series/{seriesId}/chapters")
@Retryable(
maxRetries = 3,
delay = 1000,
multiplier = 2,
maxDelay = 5000,
includes = Exception.class)
TaimuChaptersResponse getChapters(
@PathVariable String seriesId,
@RequestParam int page,
@RequestParam("per_page") int perPage,
@RequestParam String order);
record TaimuChaptersResponse(
@JsonProperty("has_more") boolean hasMore,
List<TaimuChapterItem> items,
int page,
@JsonProperty("per_page") int perPage) {}
record TaimuChapterItem(String identifier, double number) {}
@GetMapping("/api/v1/reader/chapters/{identifier}")
@Retryable(
maxRetries = 3,
delay = 1000,
multiplier = 2,
maxDelay = 5000,
includes = Exception.class)
TaimuChapterResponse getChapter(
@PathVariable String identifier, @RequestParam("adult") boolean adult);
record TaimuChapterResponse(List<TaimuPageItem> pages) {}
record TaimuPageItem(int number, String url) {}
}

150
src/main/java/com/magamochi/ingestion/providers/impl/TaimuProvider.java
View File

@ -1,34 +1,29 @@
package com.magamochi.ingestion.providers.impl;
import static java.util.Objects.isNull;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.common.ContentProviders;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.ingestion.client.TaimuClient;
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.PagedContentProvider;
import com.magamochi.ingestion.service.FlareService;
import com.magamochi.ingestion.service.ScrollableScrapperService;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.jsoup.nodes.Document;
import org.springframework.stereotype.Service;
@Log4j2
@Service(ContentProviders.TAIMU)
@RequiredArgsConstructor
public class TaimuProvider implements ContentProvider, PagedContentProvider {
private final String baseUrl = "https://taimumangas.rzword.xyz";
private final String baseUrl = "https://beta.taimumangas.com";
private final FlareService flareService;
private final ScrollableScrapperService scrollableScrapperService;
private final TaimuClient taimuClient;
private final RateLimiter taimuRateLimiter;
@Override
public List<ContentInfoDTO> getAvailableChapters(MangaContentProvider provider) {
@ -38,59 +33,45 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
provider.getManga().getTitle());
try {
var document =
flareService.getContentAsJsoupDocument(
provider.getUrl() + "?page=1&order=desc", ContentProviders.TAIMU);
var seriesId = extractSeriesId(provider.getUrl());
var allChapters = new ArrayList<ContentInfoDTO>();
var totalPages = extractContentPagesFromDocument(document);
var contentInfoList = new ArrayList<>(extractContentInfoFromDocument(document));
int page = 1;
boolean hasMore = true;
for (int page = 2; page <= totalPages; page++) {
var pageDocument =
flareService.getContentAsJsoupDocument(
provider.getUrl() + "?page=" + page + "&order=desc", ContentProviders.TAIMU);
contentInfoList.addAll(extractContentInfoFromDocument(pageDocument));
while (hasMore) {
taimuRateLimiter.acquire();
var response = taimuClient.getChapters(seriesId, page, 30, "desc");
response.items().stream()
.map(
item -> {
var number = item.number();
var formatted =
number % 1 == 0 ? String.valueOf((long) number) : String.valueOf(number);
return new ContentInfoDTO(
"Capítulo " + formatted, baseUrl + "/reader/" + item.identifier(), "pt-BR");
})
.forEach(allChapters::add);
hasMore = response.hasMore();
page++;
}
return contentInfoList;
} catch (NoSuchElementException e) {
log.error("Error parsing mangas from MangaLivre", e);
log.info("Fetched {} chapters for series {}", allChapters.size(), seriesId);
return allChapters;
} catch (Exception e) {
log.error("Error parsing chapters from " + ContentProviders.TAIMU, e);
return List.of();
}
}
private int extractContentPagesFromDocument(Document document) {
try {
var buttonsContainer =
document.selectFirst(
"div.flex.items-center.justify-between.gap-3.mt-4.pt-4.border-t div.flex.items-center.gap-1");
var highNumberButton = buttonsContainer.selectFirst("button:nth-last-child(2)");
return Integer.parseInt(highNumberButton.text());
} catch (Exception e) {
// In case of any error during parsing, we assume there is only one page of content
return 1;
}
}
private List<ContentInfoDTO> extractContentInfoFromDocument(Document document) {
try {
var grid = document.selectFirst("div.grid.grid-cols-1.gap-1");
var chapters = grid.select("a");
return chapters.stream()
.map(
chapter -> {
var chapterUrl = baseUrl + chapter.attr("href");
var title =
chapter.selectFirst("div.flex-1.min-w-0 div.flex.items-center.gap-2 p").text();
return new ContentInfoDTO(title, chapterUrl.trim(), "pt-BR");
})
.toList();
} catch (Exception e) {
log.error("Error parsing content info from " + ContentProviders.TAIMU, e);
return List.of();
private String extractSeriesId(String url) {
var lastSlash = url.lastIndexOf('/');
if (lastSlash < 0) {
throw new NoSuchElementException("Invalid series URL: " + url);
}
return url.substring(lastSlash + 1);
}
@Override
@ -98,20 +79,14 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
log.info("Getting images from {}, url {}", ContentProviders.TAIMU, chapterUrl);
try {
var document = scrollableScrapperService.getContentAsJsoupDocument(chapterUrl);
var identifier = extractSeriesId(chapterUrl);
taimuRateLimiter.acquire();
var response = taimuClient.getChapter(identifier, true);
var chapterImages = document.select("img.w-full.h-auto.object-contain.cursor-pointer");
var imageUrls =
chapterImages.stream()
.map(chapterImagesElement -> chapterImagesElement.attr("src"))
.toList();
return IntStream.range(0, imageUrls.size())
.boxed()
.map(position -> new ContentImageInfoDTO(position, imageUrls.get(position)))
return response.pages().stream()
.map(page -> new ContentImageInfoDTO(page.number() - 1, page.url()))
.toList();
} catch (NoSuchElementException e) {
} catch (Exception e) {
log.error("Error parsing manga images from " + ContentProviders.TAIMU, e);
return List.of();
}
@ -122,22 +97,13 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
log.info("Getting mangas from {}, page {}", ContentProviders.TAIMU, page);
try {
var document =
flareService.getContentAsJsoupDocument(
baseUrl + "/biblioteca?page=" + page, ContentProviders.TAIMU);
taimuRateLimiter.acquire();
var response = taimuClient.getLibrary(page, 24, "added", true);
var mangas = document.select("a.group");
return mangas.stream()
.map(
element -> {
var mangaUrl = element.attr("href");
var title = element.selectFirst("div h3").text();
return new MangaInfoDTO(title.trim(), baseUrl + mangaUrl.trim());
})
return response.items().stream()
.map(item -> new MangaInfoDTO(item.title(), baseUrl + "/series/" + item.identifier()))
.toList();
} catch (NoSuchElementException e) {
} catch (Exception e) {
log.error("Error parsing mangas from " + ContentProviders.TAIMU, e);
return List.of();
}
@ -148,27 +114,11 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
log.info("Getting total pages for {}", ContentProviders.TAIMU);
try {
var document =
flareService.getContentAsJsoupDocument(baseUrl + "/biblioteca", ContentProviders.TAIMU);
var container = document.selectFirst("div.flex.flex-col.items-center.space-y-3.mt-6");
var pagination = container.selectFirst("div.flex.items-center.gap-2");
var buttonsContainer = pagination.selectFirst("div.flex.items-center.gap-1");
var lastButton = buttonsContainer.select("button").last();
if (isNull(lastButton)) {
throw new UnprocessableException(
"Pagination buttons not found in " + ContentProviders.TAIMU);
}
var buttonText = lastButton.text();
return Integer.parseInt(buttonText);
taimuRateLimiter.acquire();
var response = taimuClient.getLibrary(1, 24, "added", true);
return (int) Math.ceil((double) response.total() / response.perPage());
} catch (Exception e) {
log.error(
"Error parsing total pages from "
+ ContentProviders.TAIMU
+ ": pagination container not found");
log.error("Error parsing total pages from " + ContentProviders.TAIMU, e);
return 0;
}
}

22
src/main/java/com/magamochi/ingestion/service/ScrollableScrapperService.java
View File

@ -1,22 +0,0 @@
package com.magamochi.ingestion.service;
import com.magamochi.ingestion.client.ScrollableScrapperClient;
import lombok.RequiredArgsConstructor;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class ScrollableScrapperService {
private final ScrollableScrapperClient client;
public Document getContentAsJsoupDocument(String url) {
return Jsoup.parse(getContent(url));
}
private String getContent(String url) {
return client.get(ScrollableScrapperClient.GetRequest.builder().url(url).build()).pageSource();
}
}

8
src/main/resources/application.yml
View File

@ -20,11 +20,6 @@ spring:
max-request-size: 4GB
cloud:
openfeign:
client:
config:
scrollable-scrapper:
connect-timeout: 480000
read-timeout: 480000
rabbitmq:
host: ${RABBITMQ_HOST}
port: ${RABBITMQ_PORT}
@ -47,9 +42,6 @@ springdoc:
flare-solverr:
endpoint: ${FLARESOLVERR_ENDPOINT}
scrollable-scrapper:
endpoint: ${SCROLLABLE_SCRAPPER_ENDPOINT}
minio:
endpoint: ${MINIO_ENDPOINT}
accessKey: ${MINIO_USER}

3
src/main/resources/db/migration/V0015__UPDATE_TAIMU_URL.sql Normal file
View File

@ -0,0 +1,3 @@
UPDATE content_providers
SET url = 'https://beta.taimumangas.com/'
WHERE name = 'Taimu';

9
src/main/resources/db/migration/V0016__CLEAR_TAIMU_CONTENT.sql Normal file
View File

@ -0,0 +1,9 @@
DELETE FROM manga_ingest_reviews
WHERE content_provider_id = (SELECT id FROM content_providers WHERE name = 'Taimu');
DELETE FROM manga_content_provider
WHERE content_provider_id = (SELECT id FROM content_providers WHERE name = 'Taimu');
DELETE FROM images
WHERE id NOT IN (SELECT DISTINCT image_id FROM manga_content_images WHERE image_id IS NOT NULL)
AND id NOT IN (SELECT DISTINCT cover_image_id FROM mangas WHERE cover_image_id IS NOT NULL);