feat: implement Bato content provider for manga import and chapter fetching #19
@ -2,6 +2,7 @@ package com.magamochi.mangamochi.controller;
|
||||
|
||||
import com.magamochi.mangamochi.model.dto.*;
|
||||
import com.magamochi.mangamochi.service.MangaImportService;
|
||||
import com.magamochi.mangamochi.service.providers.impl.BatoProvider;
|
||||
import com.magamochi.mangamochi.service.providers.impl.MangaDexProvider;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
@ -19,6 +20,7 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
@RequiredArgsConstructor
|
||||
public class MangaImportController {
|
||||
private final MangaDexProvider mangaDexProvider;
|
||||
private final BatoProvider batoProvider;
|
||||
private final MangaImportService mangaImportService;
|
||||
|
||||
@Operation(
|
||||
@ -27,11 +29,22 @@ public class MangaImportController {
|
||||
tags = {"Manga Import"},
|
||||
operationId = "importFromMangaDex")
|
||||
@PostMapping("/manga-dex")
|
||||
public DefaultResponseDTO<ImportMangaDexResponseDTO> importFromMangaDex(
|
||||
@RequestBody ImportMangaDexRequestDTO requestDTO) {
|
||||
public DefaultResponseDTO<ImportMangaResponseDTO> importFromMangaDex(
|
||||
@RequestBody ImportRequestDTO requestDTO) {
|
||||
return DefaultResponseDTO.ok(mangaDexProvider.importManga(requestDTO.id()));
|
||||
}
|
||||
|
||||
@Operation(
|
||||
summary = "Import manga from Bato",
|
||||
description = "Imports manga data from Bato into the local database.",
|
||||
tags = {"Manga Import"},
|
||||
operationId = "importFromBato")
|
||||
@PostMapping("/bato")
|
||||
public DefaultResponseDTO<ImportMangaResponseDTO> importFromBato(
|
||||
@RequestBody ImportRequestDTO requestDTO) {
|
||||
return DefaultResponseDTO.ok(batoProvider.importManga(requestDTO.id()));
|
||||
}
|
||||
|
||||
@Operation(
|
||||
summary = "Upload multiple files",
|
||||
description = "Accepts multiple files via multipart/form-data and processes them.",
|
||||
|
||||
@ -2,4 +2,4 @@ package com.magamochi.mangamochi.model.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
public record ImportMangaDexResponseDTO(@NotNull Long id) {}
|
||||
public record ImportMangaResponseDTO(@NotNull Long id) {}
|
||||
@ -1,6 +1,5 @@
|
||||
package com.magamochi.mangamochi.model.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import java.util.UUID;
|
||||
|
||||
public record ImportMangaDexRequestDTO(@NotNull UUID id) {}
|
||||
public record ImportRequestDTO(@NotNull String id) {}
|
||||
@ -57,7 +57,7 @@ public class Manga {
|
||||
@OneToMany(mappedBy = "manga")
|
||||
private List<MangaAlternativeTitle> alternativeTitles;
|
||||
|
||||
private Integer chapterCount;
|
||||
@Builder.Default private Integer chapterCount = 0;
|
||||
|
||||
private Boolean follow;
|
||||
@Builder.Default private Boolean follow = false;
|
||||
}
|
||||
|
||||
@ -32,5 +32,5 @@ public class Provider {
|
||||
@OneToMany(mappedBy = "provider")
|
||||
private List<MangaProvider> mangaProviders;
|
||||
|
||||
private Boolean supportsChapterFetch;
|
||||
@Builder.Default private Boolean supportsChapterFetch = true;
|
||||
}
|
||||
|
||||
@ -6,12 +6,14 @@ import com.google.common.util.concurrent.RateLimiter;
|
||||
import com.magamochi.mangamochi.client.JikanClient;
|
||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||
import com.magamochi.mangamochi.model.dto.ImportReviewDTO;
|
||||
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||
import com.magamochi.mangamochi.model.entity.Manga;
|
||||
import com.magamochi.mangamochi.model.entity.MangaImportReview;
|
||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||
import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository;
|
||||
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
||||
import com.magamochi.mangamochi.model.repository.MangaRepository;
|
||||
import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
@ -27,6 +29,8 @@ public class MangaImportReviewService {
|
||||
|
||||
private final RateLimiter jikanRateLimiter;
|
||||
|
||||
private final UpdateMangaDataProducer updateMangaDataProducer;
|
||||
|
||||
public List<ImportReviewDTO> getImportReviews() {
|
||||
return mangaImportReviewRepository.findAll().stream().map(ImportReviewDTO::from).toList();
|
||||
}
|
||||
@ -67,6 +71,8 @@ public class MangaImportReviewService {
|
||||
.build());
|
||||
|
||||
mangaImportReviewRepository.delete(importReview);
|
||||
|
||||
updateMangaDataProducer.sendUpdateMangaDataCommand(new UpdateMangaDataCommand(manga.getId()));
|
||||
}
|
||||
|
||||
private MangaImportReview getImportReviewThrowIfNotFound(Long id) {
|
||||
|
||||
@ -5,4 +5,5 @@ public class ContentProviders {
|
||||
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
|
||||
public static final String MANGA_DEX = "MangaDex";
|
||||
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
|
||||
public static final String BATO = "Bato";
|
||||
}
|
||||
|
||||
@ -0,0 +1,122 @@
|
||||
package com.magamochi.mangamochi.service.providers.impl;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
|
||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||
import com.magamochi.mangamochi.exception.UnprocessableException;
|
||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
|
||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||
import com.magamochi.mangamochi.model.entity.Provider;
|
||||
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
||||
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
|
||||
import com.magamochi.mangamochi.model.repository.ProviderRepository;
|
||||
import com.magamochi.mangamochi.service.MangaCreationService;
|
||||
import com.magamochi.mangamochi.service.WebScrapperClientProxyService;
|
||||
import com.magamochi.mangamochi.service.providers.ContentProvider;
|
||||
import com.magamochi.mangamochi.service.providers.ContentProviders;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service(ContentProviders.BATO)
|
||||
@RequiredArgsConstructor
|
||||
public class BatoProvider implements ContentProvider {
|
||||
private final WebScrapperClientProxyService webScrapperClientProxyService;
|
||||
private final MangaCreationService mangaCreationService;
|
||||
private final ProviderRepository providerRepository;
|
||||
private final MangaProviderRepository mangaProviderRepository;
|
||||
|
||||
@Override
|
||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||
try {
|
||||
var document = webScrapperClientProxyService.scrapeToJsoupDocument(provider.getUrl());
|
||||
|
||||
// Direct selector for chapter links
|
||||
var chapterLinks = document.select("div.scrollable-panel a[href*=/title/]");
|
||||
|
||||
return chapterLinks.stream()
|
||||
.map(
|
||||
chapterLink ->
|
||||
new ContentProviderMangaChapterResponseDTO(
|
||||
chapterLink.text(), chapterLink.attr("href"), null, null))
|
||||
.toList();
|
||||
} catch (Exception e) {
|
||||
log.warn(e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
|
||||
try {
|
||||
var document =
|
||||
webScrapperClientProxyService.scrapeToJsoupDocument(
|
||||
"https://battwo.com" + chapterUrl + "?load=2");
|
||||
|
||||
var imgElements = document.select("img[src*='media/mbch']");
|
||||
|
||||
List<String> imageUrls = new ArrayList<>();
|
||||
for (var img : imgElements) {
|
||||
String src = img.attr("src");
|
||||
if (src.startsWith("http") && src.contains("/media/mbch/")) {
|
||||
imageUrls.add(src);
|
||||
}
|
||||
}
|
||||
|
||||
return IntStream.range(0, imageUrls.size())
|
||||
.boxed()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new));
|
||||
} catch (Exception e) {
|
||||
log.warn(e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public ImportMangaResponseDTO importManga(String url) {
|
||||
try {
|
||||
var document = webScrapperClientProxyService.scrapeToJsoupDocument(url);
|
||||
// Method 1: Look for the main title in the manga info section
|
||||
var titleElement = document.selectFirst("h3 a[href*=/title/]");
|
||||
if (isNull(titleElement)) {
|
||||
throw new UnprocessableException("Manga title not found for url: " + url);
|
||||
}
|
||||
|
||||
var mangaTitle = titleElement.text();
|
||||
|
||||
var provider =
|
||||
providerRepository
|
||||
.findByNameIgnoreCase("Bato")
|
||||
.orElseGet(
|
||||
() ->
|
||||
providerRepository.save(
|
||||
Provider.builder().name("Bato").status(ProviderStatus.ACTIVE).build()));
|
||||
|
||||
var manga = mangaCreationService.getOrCreateManga(mangaTitle, url, provider);
|
||||
|
||||
if (isNull(manga)) {
|
||||
throw new NotFoundException("Manga could not be found or created for url: " + url);
|
||||
}
|
||||
|
||||
mangaProviderRepository.save(
|
||||
MangaProvider.builder()
|
||||
.manga(manga)
|
||||
.mangaTitle(mangaTitle)
|
||||
.provider(provider)
|
||||
.url(url)
|
||||
.build());
|
||||
|
||||
return new ImportMangaResponseDTO(manga.getId());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,7 @@ import com.magamochi.mangamochi.client.MangaDexClient;
|
||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||
import com.magamochi.mangamochi.exception.UnprocessableException;
|
||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||
import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
|
||||
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
|
||||
import com.magamochi.mangamochi.model.entity.MangaProvider;
|
||||
import com.magamochi.mangamochi.model.entity.Provider;
|
||||
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
|
||||
@ -112,9 +112,9 @@ public class MangaDexProvider implements ContentProvider {
|
||||
LinkedHashMap::new));
|
||||
}
|
||||
|
||||
public ImportMangaDexResponseDTO importManga(UUID id) {
|
||||
public ImportMangaResponseDTO importManga(String id) {
|
||||
mangaDexRateLimiter.acquire();
|
||||
var resultData = mangaDexClient.getManga(id).data();
|
||||
var resultData = mangaDexClient.getManga(UUID.fromString(id)).data();
|
||||
|
||||
if (resultData.attributes().title().isEmpty()) {
|
||||
throw new UnprocessableException("Manga title not found for ID: " + id);
|
||||
@ -152,6 +152,6 @@ public class MangaDexProvider implements ContentProvider {
|
||||
.url(id.toString())
|
||||
.build());
|
||||
|
||||
return new ImportMangaDexResponseDTO(manga.getId());
|
||||
return new ImportMangaResponseDTO(manga.getId());
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user