feat(provider-import): generalize content provider manga import logic #24

Merged
rov merged 1 commits from feature/provider-import into main 2025-12-14 00:49:32 -03:00
17 changed files with 235 additions and 133 deletions

View File

@ -2,8 +2,7 @@ package com.magamochi.mangamochi.controller;
import com.magamochi.mangamochi.model.dto.*;
import com.magamochi.mangamochi.service.MangaImportService;
import com.magamochi.mangamochi.service.providers.impl.BatoProvider;
import com.magamochi.mangamochi.service.providers.impl.MangaDexProvider;
import com.magamochi.mangamochi.service.ProviderManualMangaImportService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
@ -19,30 +18,19 @@ import org.springframework.web.multipart.MultipartFile;
@RequestMapping("/manga/import")
@RequiredArgsConstructor
public class MangaImportController {
private final MangaDexProvider mangaDexProvider;
private final BatoProvider batoProvider;
private final MangaImportService mangaImportService;
private final ProviderManualMangaImportService providerManualMangaImportService;
@Operation(
summary = "Import manga from MangaDex",
description = "Imports manga data from MangaDex into the local database.",
summary = "Import manga from content provider",
description = "Imports manga data from content provider into the local database.",
tags = {"Manga Import"},
operationId = "importFromMangaDex")
@PostMapping("/manga-dex")
public DefaultResponseDTO<ImportMangaResponseDTO> importFromMangaDex(
@RequestBody ImportRequestDTO requestDTO) {
return DefaultResponseDTO.ok(mangaDexProvider.importManga(requestDTO.id()));
}
@Operation(
summary = "Import manga from Bato",
description = "Imports manga data from Bato into the local database.",
tags = {"Manga Import"},
operationId = "importFromBato")
@PostMapping("/bato")
public DefaultResponseDTO<ImportMangaResponseDTO> importFromBato(
@RequestBody ImportRequestDTO requestDTO) {
return DefaultResponseDTO.ok(batoProvider.importManga(requestDTO.id()));
operationId = "importFromProvider")
@PostMapping("/provider/{providerId}")
public DefaultResponseDTO<ImportMangaResponseDTO> importFromProvider(
@PathVariable Long providerId, @RequestBody ImportRequestDTO requestDTO) {
return DefaultResponseDTO.ok(
providerManualMangaImportService.importFromProvider(providerId, requestDTO));
}
@Operation(

View File

@ -0,0 +1,25 @@
package com.magamochi.mangamochi.controller;
import com.magamochi.mangamochi.model.dto.*;
import com.magamochi.mangamochi.service.ProviderService;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/providers")
@RequiredArgsConstructor
public class ProviderController {
private final ProviderService providerService;
@Operation(
summary = "Get a list of providers",
description = "Retrieve a list of content providers",
tags = {"Provider"},
operationId = "getProviders")
@GetMapping
public DefaultResponseDTO<ProviderListDTO> getMangas(
@RequestParam(name = "manualImport", required = false) Boolean manualImport) {
return DefaultResponseDTO.ok(providerService.getProviders(manualImport));
}
}

View File

@ -2,4 +2,4 @@ package com.magamochi.mangamochi.model.dto;
import jakarta.validation.constraints.NotNull;
public record ImportRequestDTO(@NotNull String id) {}
public record ImportRequestDTO(String metadataId, @NotNull String id) {}

View File

@ -0,0 +1,17 @@
package com.magamochi.mangamochi.model.dto;
import com.magamochi.mangamochi.model.entity.*;
import jakarta.validation.constraints.NotNull;
import java.util.List;
public record ProviderListDTO(@NotNull List<ProviderDTO> providers) {
public static ProviderListDTO from(List<Provider> providers) {
return new ProviderListDTO(providers.stream().map(ProviderDTO::from).toList());
}
record ProviderDTO(@NotNull Long id, @NotNull String name) {
public static ProviderDTO from(Provider provider) {
return new ProviderDTO(provider.getId(), provider.getName());
}
}
}

View File

@ -33,4 +33,6 @@ public class Provider {
private List<MangaProvider> mangaProviders;
@Builder.Default private Boolean supportsChapterFetch = true;
@Builder.Default private Boolean manualImport = false;
}

View File

@ -7,7 +7,7 @@ import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
public interface MangaProviderRepository extends JpaRepository<MangaProvider, Long> {
Optional<MangaProvider> findByMangaAndProvider(Manga manga, Provider provider);
boolean existsByMangaAndProviderAndUrlIgnoreCase(Manga manga, Provider provider, String url);
Optional<MangaProvider> findByMangaTitleIgnoreCaseAndProvider(
String mangaTitle, Provider provider);

View File

@ -11,4 +11,6 @@ public interface MangaRepository
Optional<Manga> findByTitleIgnoreCase(String title);
List<Manga> findByFollowTrue();
Optional<Manga> findByMalId(Long malId);
}

View File

@ -78,19 +78,28 @@ public class MangaCreationService {
}
var result = resultOptional.get();
return getOrCreateManga(result.mal_id(), result.title());
}
existingManga = mangaRepository.findByTitleIgnoreCase(result.title());
return existingManga.orElseGet(
() -> {
var manga =
mangaRepository.save(
Manga.builder().title(result.title()).malId(result.mal_id()).build());
public Manga getOrCreateManga(Long malId) {
jikanRateLimiter.acquire();
var data = jikanClient.getMangaById(malId);
updateMangaDataProducer.sendUpdateMangaDataCommand(
new UpdateMangaDataCommand(manga.getId()));
return getOrCreateManga(data.data().mal_id(), data.data().title());
}
return manga;
});
private Manga getOrCreateManga(Long malId, String title) {
return mangaRepository
.findByMalId(malId)
.orElseGet(
() -> {
var manga = mangaRepository.save(Manga.builder().title(title).malId(malId).build());
updateMangaDataProducer.sendUpdateMangaDataCommand(
new UpdateMangaDataCommand(manga.getId()));
return manga;
});
}
private void createMangaImportReview(String title, String url, Provider provider) {

View File

@ -62,13 +62,16 @@ public class MangaImportReviewService {
.malId(Long.parseLong(malId))
.build()));
mangaProviderRepository.save(
MangaProvider.builder()
.manga(manga)
.mangaTitle(importReview.getTitle())
.provider(importReview.getProvider())
.url(importReview.getUrl())
.build());
if (!mangaProviderRepository.existsByMangaAndProviderAndUrlIgnoreCase(
manga, importReview.getProvider(), importReview.getUrl())) {
mangaProviderRepository.save(
MangaProvider.builder()
.manga(manga)
.mangaTitle(importReview.getTitle())
.provider(importReview.getProvider())
.url(importReview.getUrl())
.build());
}
mangaImportReviewRepository.delete(importReview);

View File

@ -43,7 +43,8 @@ public class MangaListService {
return;
}
try {
if (!mangaProviderRepository.existsByMangaAndProviderAndUrlIgnoreCase(
manga, provider, mangaResponse.url())) {
mangaProviderRepository.save(
MangaProvider.builder()
.manga(manga)
@ -51,8 +52,6 @@ public class MangaListService {
.provider(provider)
.url(mangaResponse.url())
.build());
} catch (Exception e) {
log.error(e.getMessage());
}
});
}

View File

@ -0,0 +1,73 @@
package com.magamochi.mangamochi.service;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import com.magamochi.mangamochi.exception.NotFoundException;
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
import com.magamochi.mangamochi.model.dto.ImportRequestDTO;
import com.magamochi.mangamochi.model.entity.*;
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
import com.magamochi.mangamochi.model.repository.*;
import com.magamochi.mangamochi.service.providers.ManualImportContentProviderFactory;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class ProviderManualMangaImportService {
private final MangaCreationService mangaCreationService;
private final ManualImportContentProviderFactory contentProviderFactory;
private final ProviderRepository providerRepository;
private final MangaProviderRepository mangaProviderRepository;
public ImportMangaResponseDTO importFromProvider(Long providerId, ImportRequestDTO requestDTO) {
var provider = getProvider(providerId);
var contentProvider = contentProviderFactory.getManualImportContentProvider(provider.getName());
var title = contentProvider.getMangaTitle(requestDTO.id());
var manga =
(nonNull(requestDTO.metadataId()) && !requestDTO.metadataId().isBlank())
? mangaCreationService.getOrCreateManga(Long.parseLong(requestDTO.metadataId()))
: mangaCreationService.getOrCreateManga(title, requestDTO.id(), provider);
if (isNull(manga)) {
throw new NotFoundException("Manga could not be found or created for ID: " + requestDTO.id());
}
if (!mangaProviderRepository.existsByMangaAndProviderAndUrlIgnoreCase(
manga, provider, requestDTO.id())) {
mangaProviderRepository.save(
MangaProvider.builder()
.manga(manga)
.mangaTitle(title)
.provider(provider)
.url(requestDTO.id())
.build());
}
return new ImportMangaResponseDTO(manga.getId());
}
public Provider getProvider(Long providerId) {
var provider =
providerRepository
.findById(providerId)
.orElseThrow(() -> new NotFoundException("Provider not found"));
if (!provider.getStatus().equals(ProviderStatus.ACTIVE)) {
throw new IllegalStateException("Provider is not active");
}
if (!provider.getManualImport()) {
throw new IllegalArgumentException("Manual import not supported");
}
return provider;
}
}

View File

@ -1,5 +1,8 @@
package com.magamochi.mangamochi.service;
import static java.util.Objects.nonNull;
import com.magamochi.mangamochi.model.dto.ProviderListDTO;
import com.magamochi.mangamochi.model.entity.Provider;
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
import com.magamochi.mangamochi.model.repository.ProviderRepository;
@ -11,6 +14,16 @@ import org.springframework.stereotype.Service;
public class ProviderService {
private final ProviderRepository providerRepository;
public ProviderListDTO getProviders(Boolean manualImport) {
var providers = providerRepository.findAll();
if (nonNull(manualImport) && manualImport) {
providers = providers.stream().filter(Provider::getManualImport).toList();
}
return ProviderListDTO.from(providers);
}
public Provider getOrCreateProvider(String providerName) {
return getOrCreateProvider(providerName, true);
}

View File

@ -0,0 +1,5 @@
package com.magamochi.mangamochi.service.providers;
public interface ManualImportContentProvider {
String getMangaTitle(String value);
}

View File

@ -0,0 +1,24 @@
package com.magamochi.mangamochi.service.providers;
import java.util.Map;
import java.util.Objects;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
@Getter
@Component
@RequiredArgsConstructor
public class ManualImportContentProviderFactory {
private final Map<String, ManualImportContentProvider> contentProviders;
public ManualImportContentProvider getManualImportContentProvider(String providerName) {
var provider = contentProviders.get(providerName);
if (Objects.isNull(provider)) {
throw new IllegalArgumentException("No such provider " + providerName);
}
return provider;
}
}

View File

@ -2,19 +2,13 @@ package com.magamochi.mangamochi.service.providers.impl;
import static java.util.Objects.isNull;
import com.magamochi.mangamochi.exception.NotFoundException;
import com.magamochi.mangamochi.exception.UnprocessableException;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
import com.magamochi.mangamochi.model.entity.MangaProvider;
import com.magamochi.mangamochi.model.entity.Provider;
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
import com.magamochi.mangamochi.model.repository.ProviderRepository;
import com.magamochi.mangamochi.service.FlareService;
import com.magamochi.mangamochi.service.MangaCreationService;
import com.magamochi.mangamochi.service.providers.ContentProvider;
import com.magamochi.mangamochi.service.providers.ContentProviders;
import com.magamochi.mangamochi.service.providers.ManualImportContentProvider;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@ -25,13 +19,10 @@ import org.springframework.stereotype.Service;
@Log4j2
@Service(ContentProviders.BATO)
@RequiredArgsConstructor
public class BatoProvider implements ContentProvider {
public class BatoProvider implements ContentProvider, ManualImportContentProvider {
private static final String URL = "https://battwo.com";
private final FlareService flareService;
private final MangaCreationService mangaCreationService;
private final ProviderRepository providerRepository;
private final MangaProviderRepository mangaProviderRepository;
@Override
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
@ -88,39 +79,19 @@ public class BatoProvider implements ContentProvider {
}
}
public ImportMangaResponseDTO importManga(String url) {
var document = flareService.getContentAsJsoupDocument(url, ContentProviders.BATO);
@Override
public String getMangaTitle(String value) {
var document = flareService.getContentAsJsoupDocument(value, ContentProviders.BATO);
// Method 1: Look for the main title in the manga info section
var titleElement = document.selectFirst("h3 a[href*=/title/]");
if (isNull(titleElement)) {
throw new UnprocessableException("Manga title not found for url: " + url);
titleElement = document.selectFirst("h3.item-title > a");
if (isNull(titleElement)) {
throw new UnprocessableException("Manga title not found for url: " + value);
}
}
var mangaTitle = titleElement.text();
var provider =
providerRepository
.findByNameIgnoreCase("Bato")
.orElseGet(
() ->
providerRepository.save(
Provider.builder().name("Bato").status(ProviderStatus.ACTIVE).build()));
var manga = mangaCreationService.getOrCreateManga(mangaTitle, url, provider);
if (isNull(manga)) {
throw new NotFoundException("Manga could not be found or created for url: " + url);
}
mangaProviderRepository.save(
MangaProvider.builder()
.manga(manga)
.mangaTitle(mangaTitle)
.provider(provider)
.url(url)
.build());
return new ImportMangaResponseDTO(manga.getId());
return titleElement.text();
}
}

View File

@ -1,21 +1,13 @@
package com.magamochi.mangamochi.service.providers.impl;
import static java.util.Objects.isNull;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.mangamochi.client.MangaDexClient;
import com.magamochi.mangamochi.exception.NotFoundException;
import com.magamochi.mangamochi.exception.UnprocessableException;
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.mangamochi.model.dto.ImportMangaResponseDTO;
import com.magamochi.mangamochi.model.entity.MangaProvider;
import com.magamochi.mangamochi.model.entity.Provider;
import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
import com.magamochi.mangamochi.model.repository.ProviderRepository;
import com.magamochi.mangamochi.service.MangaCreationService;
import com.magamochi.mangamochi.service.providers.ContentProvider;
import com.magamochi.mangamochi.service.providers.ContentProviders;
import com.magamochi.mangamochi.service.providers.ManualImportContentProvider;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@ -26,11 +18,8 @@ import org.springframework.stereotype.Service;
@Log4j2
@Service(ContentProviders.MANGA_DEX)
@RequiredArgsConstructor
public class MangaDexProvider implements ContentProvider {
public class MangaDexProvider implements ContentProvider, ManualImportContentProvider {
private final MangaDexClient mangaDexClient;
private final MangaCreationService mangaCreationService;
private final ProviderRepository providerRepository;
private final MangaProviderRepository mangaProviderRepository;
private final RateLimiter mangaDexRateLimiter;
@ -112,46 +101,22 @@ public class MangaDexProvider implements ContentProvider {
LinkedHashMap::new));
}
public ImportMangaResponseDTO importManga(String id) {
@Override
public String getMangaTitle(String value) {
mangaDexRateLimiter.acquire();
var resultData = mangaDexClient.getManga(UUID.fromString(id)).data();
var resultData = mangaDexClient.getManga(UUID.fromString(value)).data();
if (resultData.attributes().title().isEmpty()) {
throw new UnprocessableException("Manga title not found for ID: " + id);
throw new UnprocessableException("Manga title not found for ID: " + value);
}
var mangaTitle =
resultData
.attributes()
.title()
.getOrDefault(
"en",
resultData.attributes().title().values().stream()
.findFirst()
.orElseThrow(() -> new UnprocessableException("No title available")));
var provider =
providerRepository
.findByNameIgnoreCase("MangaDex")
.orElseGet(
() ->
providerRepository.save(
Provider.builder().name("MangaDex").status(ProviderStatus.ACTIVE).build()));
var manga = mangaCreationService.getOrCreateManga(mangaTitle, id.toString(), provider);
if (isNull(manga)) {
throw new NotFoundException("Manga could not be found or created for ID: " + id);
}
mangaProviderRepository.save(
MangaProvider.builder()
.manga(manga)
.mangaTitle(mangaTitle)
.provider(provider)
.url(id.toString())
.build());
return new ImportMangaResponseDTO(manga.getId());
return resultData
.attributes()
.title()
.getOrDefault(
"en",
resultData.attributes().title().values().stream()
.findFirst()
.orElseThrow(() -> new UnprocessableException("No title available")));
}
}

View File

@ -0,0 +1,6 @@
ALTER TABLE providers
ADD COLUMN manual_import BOOLEAN DEFAULT FALSE;
UPDATE providers
SET manual_import = TRUE
WHERE name IN ('MangaDex', 'Bato');