feat: implement Bato content provider for manga import and chapter fetching
This commit is contained in:
parent
d1741bcd32
commit
3f3a3739b6
@ -2,6 +2,7 @@ package com.magamochi.mangamochi.controller;
|
|||||||
|
|
||||||
import com.magamochi.mangamochi.model.dto.*;
|
import com.magamochi.mangamochi.model.dto.*;
|
||||||
import com.magamochi.mangamochi.service.MangaImportService;
|
import com.magamochi.mangamochi.service.MangaImportService;
|
||||||
|
import com.magamochi.mangamochi.service.providers.impl.BatoProvider;
|
||||||
import com.magamochi.mangamochi.service.providers.impl.MangaDexProvider;
|
import com.magamochi.mangamochi.service.providers.impl.MangaDexProvider;
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.Parameter;
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
@ -19,6 +20,7 @@ import org.springframework.web.multipart.MultipartFile;
|
|||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class MangaImportController {
|
public class MangaImportController {
|
||||||
private final MangaDexProvider mangaDexProvider;
|
private final MangaDexProvider mangaDexProvider;
|
||||||
|
private final BatoProvider batoProvider;
|
||||||
private final MangaImportService mangaImportService;
|
private final MangaImportService mangaImportService;
|
||||||
|
|
||||||
@Operation(
|
@Operation(
|
||||||
@ -27,11 +29,22 @@ public class MangaImportController {
|
|||||||
tags = {"Manga Import"},
|
tags = {"Manga Import"},
|
||||||
operationId = "importFromMangaDex")
|
operationId = "importFromMangaDex")
|
||||||
@PostMapping("/manga-dex")
|
@PostMapping("/manga-dex")
|
||||||
public DefaultResponseDTO<ImportMangaDexResponseDTO> importFromMangaDex(
|
public DefaultResponseDTO<ImportMangaResponseDTO> importFromMangaDex(
|
||||||
@RequestBody ImportMangaDexRequestDTO requestDTO) {
|
@RequestBody ImportRequestDTO requestDTO) {
|
||||||
return DefaultResponseDTO.ok(mangaDexProvider.importManga(requestDTO.id()));
|
return DefaultResponseDTO.ok(mangaDexProvider.importManga(requestDTO.id()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Import manga from Bato",
|
||||||
|
description = "Imports manga data from Bato into the local database.",
|
||||||
|
tags = {"Manga Import"},
|
||||||
|
operationId = "importFromBato")
|
||||||
|
@PostMapping("/bato")
|
||||||
|
public DefaultResponseDTO<ImportMangaResponseDTO> importFromBato(
|
||||||
|
@RequestBody ImportRequestDTO requestDTO) {
|
||||||
|
return DefaultResponseDTO.ok(batoProvider.importManga(requestDTO.id()));
|
||||||
|
}
|
||||||
|
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "Upload multiple files",
|
summary = "Upload multiple files",
|
||||||
description = "Accepts multiple files via multipart/form-data and processes them.",
|
description = "Accepts multiple files via multipart/form-data and processes them.",
|
||||||
|
|||||||
@ -2,4 +2,4 @@ package com.magamochi.mangamochi.model.dto;
|
|||||||
|
|
||||||
import jakarta.validation.constraints.NotNull;
|
import jakarta.validation.constraints.NotNull;
|
||||||
|
|
||||||
public record ImportMangaDexResponseDTO(@NotNull Long id) {}
|
public record ImportMangaResponseDTO(@NotNull Long id) {}
|
||||||
@ -1,6 +1,5 @@
|
|||||||
package com.magamochi.mangamochi.model.dto;
|
package com.magamochi.mangamochi.model.dto;
|
||||||
|
|
||||||
import jakarta.validation.constraints.NotNull;
|
import jakarta.validation.constraints.NotNull;
|
||||||
import java.util.UUID;
|
|
||||||
|
|
||||||
public record ImportMangaDexRequestDTO(@NotNull UUID id) {}
|
public record ImportRequestDTO(@NotNull String id) {}
|
||||||
@ -57,7 +57,7 @@ public class Manga {
|
|||||||
@OneToMany(mappedBy = "manga")
|
@OneToMany(mappedBy = "manga")
|
||||||
private List<MangaAlternativeTitle> alternativeTitles;
|
private List<MangaAlternativeTitle> alternativeTitles;
|
||||||
|
|
||||||
private Integer chapterCount;
|
@Builder.Default private Integer chapterCount = 0;
|
||||||
|
|
||||||
private Boolean follow;
|
@Builder.Default private Boolean follow = false;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -32,5 +32,5 @@ public class Provider {
|
|||||||
@OneToMany(mappedBy = "provider")
|
@OneToMany(mappedBy = "provider")
|
||||||
private List<MangaProvider> mangaProviders;
|
private List<MangaProvider> mangaProviders;
|
||||||
|
|
||||||
private Boolean supportsChapterFetch;
|
@Builder.Default private Boolean supportsChapterFetch = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,12 +6,14 @@ import com.google.common.util.concurrent.RateLimiter;
|
|||||||
import com.magamochi.mangamochi.client.JikanClient;
|
import com.magamochi.mangamochi.client.JikanClient;
|
||||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||||
import com.magamochi.mangamochi.model.dto.ImportReviewDTO;
|
import com.magamochi.mangamochi.model.dto.ImportReviewDTO;
|
||||||
|
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||||
import com.magamochi.mangamochi.model.entity.Manga;
|
import com.magamochi.mangamochi.model.entity.Manga;
|
||||||
import com.magamochi.mangamochi.model.entity.MangaImportReview;
|
import com.magamochi.mangamochi.model.entity.MangaImportReview;
|
||||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||||
import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository;
|
import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository;
|
||||||
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
||||||
import com.magamochi.mangamochi.model.repository.MangaRepository;
|
import com.magamochi.mangamochi.model.repository.MangaRepository;
|
||||||
|
import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@ -27,6 +29,8 @@ public class MangaImportReviewService {
|
|||||||
|
|
||||||
private final RateLimiter jikanRateLimiter;
|
private final RateLimiter jikanRateLimiter;
|
||||||
|
|
||||||
|
private final UpdateMangaDataProducer updateMangaDataProducer;
|
||||||
|
|
||||||
public List<ImportReviewDTO> getImportReviews() {
|
public List<ImportReviewDTO> getImportReviews() {
|
||||||
return mangaImportReviewRepository.findAll().stream().map(ImportReviewDTO::from).toList();
|
return mangaImportReviewRepository.findAll().stream().map(ImportReviewDTO::from).toList();
|
||||||
}
|
}
|
||||||
@ -67,6 +71,8 @@ public class MangaImportReviewService {
|
|||||||
.build());
|
.build());
|
||||||
|
|
||||||
mangaImportReviewRepository.delete(importReview);
|
mangaImportReviewRepository.delete(importReview);
|
||||||
|
|
||||||
|
updateMangaDataProducer.sendUpdateMangaDataCommand(new UpdateMangaDataCommand(manga.getId()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private MangaImportReview getImportReviewThrowIfNotFound(Long id) {
|
private MangaImportReview getImportReviewThrowIfNotFound(Long id) {
|
||||||
|
|||||||
@ -5,4 +5,5 @@ public class ContentProviders {
|
|||||||
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
|
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
|
||||||
public static final String MANGA_DEX = "MangaDex";
|
public static final String MANGA_DEX = "MangaDex";
|
||||||
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
|
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
|
||||||
|
public static final String BATO = "Bato";
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,122 @@
|
|||||||
|
package com.magamochi.mangamochi.service.providers.impl;
|
||||||
|
|
||||||
|
import static java.util.Objects.isNull;
|
||||||
|
|
||||||
|
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||||
|
import com.magamochi.mangamochi.exception.UnprocessableException;
|
||||||
|
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||||
|
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
|
||||||
|
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||||
|
import com.magamochi.mangamochi.model.entity.Provider;
|
||||||
|
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
||||||
|
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
||||||
|
import com.magamochi.mangamochi.model.repository.ProviderRepository;
|
||||||
|
import com.magamochi.mangamochi.service.MangaCreationService;
|
||||||
|
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
||||||
|
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
||||||
|
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
@Service(ContentProviders.BATO)
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class BatoProvider implements ContentProvider {
|
||||||
|
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
||||||
|
private final MangaCreationService mangaCreationService;
|
||||||
|
private final ProviderRepository providerRepository;
|
||||||
|
private final MangaProviderRepository mangaProviderRepository;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||||
|
try {
|
||||||
|
var document = webScrapperClientProxyService.scrapeToJsoupDocument(provider.getUrl());
|
||||||
|
|
||||||
|
// Direct selector for chapter links
|
||||||
|
var chapterLinks = document.select("div.scrollable-panel a[href*=/title/]");
|
||||||
|
|
||||||
|
return chapterLinks.stream()
|
||||||
|
.map(
|
||||||
|
chapterLink ->
|
||||||
|
new ContentProviderMangaChapterResponseDTO(
|
||||||
|
chapterLink.text(), chapterLink.attr("href"), null, null))
|
||||||
|
.toList();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn(e.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
|
||||||
|
try {
|
||||||
|
var document =
|
||||||
|
webScrapperClientProxyService.scrapeToJsoupDocument(
|
||||||
|
"https://battwo.com" + chapterUrl + "?load=2");
|
||||||
|
|
||||||
|
var imgElements = document.select("img[src*='media/mbch']");
|
||||||
|
|
||||||
|
List<String> imageUrls = new ArrayList<>();
|
||||||
|
for (var img : imgElements) {
|
||||||
|
String src = img.attr("src");
|
||||||
|
if (src.startsWith("http") && src.contains("/media/mbch/")) {
|
||||||
|
imageUrls.add(src);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return IntStream.range(0, imageUrls.size())
|
||||||
|
.boxed()
|
||||||
|
.collect(
|
||||||
|
Collectors.toMap(
|
||||||
|
i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn(e.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ImportMangaResponseDTO importManga(String url) {
|
||||||
|
try {
|
||||||
|
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
|
||||||
|
// Method 1: Look for the main title in the manga info section
|
||||||
|
var titleElement = document.selectFirst("h3 a[href*=/title/]");
|
||||||
|
if (isNull(titleElement)) {
|
||||||
|
throw new UnprocessableException("Manga title not found for url: " + url);
|
||||||
|
}
|
||||||
|
|
||||||
|
var mangaTitle = titleElement.text();
|
||||||
|
|
||||||
|
var provider =
|
||||||
|
providerRepository
|
||||||
|
.findByNameIgnoreCase("Bato")
|
||||||
|
.orElseGet(
|
||||||
|
() ->
|
||||||
|
providerRepository.save(
|
||||||
|
Provider.builder().name("Bato").status(ProviderStatus.ACTIVE).build()));
|
||||||
|
|
||||||
|
var manga = mangaCreationService.getOrCreateManga(mangaTitle, url, provider);
|
||||||
|
|
||||||
|
if (isNull(manga)) {
|
||||||
|
throw new NotFoundException("Manga could not be found or created for url: " + url);
|
||||||
|
}
|
||||||
|
|
||||||
|
mangaProviderRepository.save(
|
||||||
|
MangaProvider.builder()
|
||||||
|
.manga(manga)
|
||||||
|
.mangaTitle(mangaTitle)
|
||||||
|
.provider(provider)
|
||||||
|
.url(url)
|
||||||
|
.build());
|
||||||
|
|
||||||
|
return new ImportMangaResponseDTO(manga.getId());
|
||||||
|
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -7,7 +7,7 @@ import com.magamochi.mangamochi.client.MangaDexClient;
|
|||||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||||
import com.magamochi.mangamochi.exception.UnprocessableException;
|
import com.magamochi.mangamochi.exception.UnprocessableException;
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||||
import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
|
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
|
||||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||||
import com.magamochi.mangamochi.model.entity.Provider;
|
import com.magamochi.mangamochi.model.entity.Provider;
|
||||||
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
||||||
@ -112,9 +112,9 @@ public class MangaDexProvider implements ContentProvider {
|
|||||||
LinkedHashMap::new));
|
LinkedHashMap::new));
|
||||||
}
|
}
|
||||||
|
|
||||||
public ImportMangaDexResponseDTO importManga(UUID id) {
|
public ImportMangaResponseDTO importManga(String id) {
|
||||||
mangaDexRateLimiter.acquire();
|
mangaDexRateLimiter.acquire();
|
||||||
var resultData = mangaDexClient.getManga(id).data();
|
var resultData = mangaDexClient.getManga(UUID.fromString(id)).data();
|
||||||
|
|
||||||
if (resultData.attributes().title().isEmpty()) {
|
if (resultData.attributes().title().isEmpty()) {
|
||||||
throw new UnprocessableException("Manga title not found for ID: " + id);
|
throw new UnprocessableException("Manga title not found for ID: " + id);
|
||||||
@ -152,6 +152,6 @@ public class MangaDexProvider implements ContentProvider {
|
|||||||
.url(id.toString())
|
.url(id.toString())
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
return new ImportMangaDexResponseDTO(manga.getId());
|
return new ImportMangaResponseDTO(manga.getId());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user