mangamochi/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: MangaDex import

Browse Source
This commit is contained in:
Rodrigo Verdiani 2026-03-28 15:42:58 -03:00
parent a9b416a9a2
commit 420b522edf
17 changed files with 177 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/com/magamochi/controller/MangaImportController.java → src/main/java/com/magamochi/catalog/controller/MangaImportController.java
View File

@ -1,10 +1,9 @@
package com.magamochi.controller;
package com.magamochi.catalog.controller;
import com.magamochi.catalog.model.dto.ImportMangaResponseDTO;
import com.magamochi.catalog.model.dto.ImportRequestDTO;
import com.magamochi.catalog.service.MangaManualImportService;
import com.magamochi.common.model.dto.DefaultResponseDTO;
import com.magamochi.model.dto.ImportMangaResponseDTO;
import com.magamochi.model.dto.ImportRequestDTO;
// import com.magamochi.service.MangaImportService;
import com.magamochi.service.ProviderManualMangaImportService;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@ -13,8 +12,7 @@ import org.springframework.web.bind.annotation.*;
@RequestMapping("/manga/import")
@RequiredArgsConstructor
public class MangaImportController {
// private final MangaImportService mangaImportService;
private final ProviderManualMangaImportService providerManualMangaImportService;
private final MangaManualImportService mangaManualImportService;
@Operation(
summary = "Import manga from content provider",
@ -25,6 +23,6 @@ public class MangaImportController {
public DefaultResponseDTO<ImportMangaResponseDTO> importFromProvider(
@PathVariable Long providerId, @RequestBody ImportRequestDTO requestDTO) {
return DefaultResponseDTO.ok(
providerManualMangaImportService.importFromProvider(providerId, requestDTO));
mangaManualImportService.importFromProvider(providerId, requestDTO));
}
}

2
src/main/java/com/magamochi/model/dto/ImportMangaResponseDTO.java → src/main/java/com/magamochi/catalog/model/dto/ImportMangaResponseDTO.java
View File

@ -1,4 +1,4 @@
package com.magamochi.model.dto;
package com.magamochi.catalog.model.dto;
import jakarta.validation.constraints.NotNull;

5
src/main/java/com/magamochi/catalog/model/dto/ImportRequestDTO.java Normal file
View File

@ -0,0 +1,5 @@
package com.magamochi.catalog.model.dto;
import jakarta.validation.constraints.NotNull;
public record ImportRequestDTO(String malId, String aniListId, @NotNull String id) {}

2
src/main/java/com/magamochi/model/specification/MangaImportJobSpecification.java → src/main/java/com/magamochi/catalog/model/specification/MangaImportJobSpecification.java
View File

@ -1,4 +1,4 @@
package com.magamochi.model.specification;
package com.magamochi.catalog.model.specification;
import static java.util.Objects.nonNull;

70
src/main/java/com/magamochi/catalog/service/MangaManualImportService.java Normal file
View File

@ -0,0 +1,70 @@
package com.magamochi.catalog.service;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import com.magamochi.catalog.model.dto.ImportMangaResponseDTO;
import com.magamochi.catalog.model.dto.ImportRequestDTO;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.service.ContentProviderService;
import com.magamochi.ingestion.service.IngestionService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class MangaManualImportService {
private final ContentProviderService contentProviderService;
private final IngestionService ingestionService;
private final MangaResolutionService mangaResolutionService;
private final MangaContentProviderRepository mangaContentProviderRepository;
public ImportMangaResponseDTO importFromProvider(Long providerId, ImportRequestDTO requestDTO) {
var provider = contentProviderService.getManualImportProvider(providerId);
var metadata =
ingestionService.getMangaMetadataFromProvider(provider.getName(), requestDTO.id());
var malId = nonNull(requestDTO.malId()) ? Long.parseLong(requestDTO.malId()) : metadata.malId();
var aniListId =
nonNull(requestDTO.aniListId())
? Long.parseLong(requestDTO.aniListId())
: metadata.aniListId();
var manga =
nonNull(malId) || nonNull(aniListId)
? mangaResolutionService.findOrCreateManga(aniListId, malId)
: mangaResolutionService.findOrCreateManga(metadata.title());
if (isNull(manga)) {
throw new NotFoundException("Manga could not be found or created for ID: " + requestDTO.id());
}
var mangaContentProviderOpt =
mangaContentProviderRepository.findByManga_IdAndContentProvider_Id(
manga.getId(), provider.getId());
if (mangaContentProviderOpt.isEmpty()) {
mangaContentProviderRepository.save(
MangaContentProvider.builder()
.manga(manga)
.mangaTitle(metadata.title())
.contentProvider(provider)
.url(requestDTO.id())
.build());
} else {
var mangaContentProvider = mangaContentProviderOpt.get();
if (isNull(mangaContentProvider.getUrl())) {
mangaContentProvider.setUrl(requestDTO.id());
mangaContentProviderRepository.save(mangaContentProvider);
}
}
return new ImportMangaResponseDTO(manga.getId());
}
}

2
src/main/java/com/magamochi/content/service/ContentImportService.java
View File

@ -4,6 +4,7 @@ import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static org.apache.commons.lang3.StringUtils.isBlank;
import com.magamochi.catalog.model.specification.MangaImportJobSpecification;
import com.magamochi.catalog.service.MangaContentProviderService;
import com.magamochi.catalog.service.MangaResolutionService;
import com.magamochi.common.exception.UnprocessableException;
@ -26,7 +27,6 @@ import com.magamochi.image.service.ImageFetchService;
import com.magamochi.image.service.ImageService;
import com.magamochi.image.service.S3Service;
import com.magamochi.ingestion.service.ContentProviderService;
import com.magamochi.model.specification.MangaImportJobSpecification;
import jakarta.validation.constraints.NotNull;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;

9
src/main/java/com/magamochi/content/service/ContentService.java
View File

@ -11,6 +11,7 @@ import jakarta.validation.constraints.NotNull;
import java.util.Comparator;
import java.util.List;
import lombok.RequiredArgsConstructor;
import net.greypanther.natsort.CaseInsensitiveSimpleNaturalComparator;
import org.springframework.stereotype.Service;
@Service
@ -25,7 +26,9 @@ public class ContentService {
var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId);
return mangaContentProvider.getMangaContents().stream()
.sorted(Comparator.comparing(MangaContent::getTitle))
.sorted(
Comparator.comparing(
MangaContent::getTitle, CaseInsensitiveSimpleNaturalComparator.getInstance()))
.map(
mangaContent -> {
var isRead = userMangaContentReadService.isRead(mangaContent.getId());
@ -45,7 +48,9 @@ public class ContentService {
var chapters =
mangaContent.getMangaContentProvider().getMangaContents().stream()
.sorted(Comparator.comparing(MangaContent::getId))
.sorted(
Comparator.comparing(
MangaContent::getTitle, CaseInsensitiveSimpleNaturalComparator.getInstance()))
.toList();
Long prevId = null;
Long nextId = null;

4
src/main/java/com/magamochi/client/MangaDexClient.java → src/main/java/com/magamochi/ingestion/client/MangaDexClient.java
View File

@ -1,6 +1,6 @@
package com.magamochi.client;
package com.magamochi.ingestion.client;
import com.magamochi.model.dto.MangaDexMangaDTO;
import com.magamochi.ingestion.model.dto.MangaDexMangaDTO;
import java.util.List;
import java.util.UUID;
import org.springframework.cloud.openfeign.FeignClient;

14
src/main/java/com/magamochi/ingestion/model/dto/MangaDexMangaDTO.java Normal file
View File

@ -0,0 +1,14 @@
package com.magamochi.ingestion.model.dto;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public record MangaDexMangaDTO(MangaData data) {
public record MangaData(UUID id, AttributesData attributes) {
public record AttributesData(
Map<String, String> title,
List<Map<String, String>> altTitles,
Map<String, String> links) {}
}
}

6
src/main/java/com/magamochi/ingestion/model/dto/ProviderMangaMetadataDTO.java Normal file
View File

@ -0,0 +1,6 @@
package com.magamochi.ingestion.model.dto;
import lombok.Builder;
@Builder
public record ProviderMangaMetadataDTO(String title, Long malId, Long aniListId) {}

4
src/main/java/com/magamochi/ingestion/providers/ManualImportContentProvider.java
View File

@ -1,5 +1,7 @@
package com.magamochi.ingestion.providers;
import com.magamochi.ingestion.model.dto.ProviderMangaMetadataDTO;
public interface ManualImportContentProvider {
String getMangaTitle(String value);
ProviderMangaMetadataDTO getMangaMetadata(String value);
}

51
src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java
View File

@ -1,14 +1,16 @@
package com.magamochi.ingestion.providers.impl;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.client.MangaDexClient;
import com.magamochi.common.ContentProviders;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.ingestion.client.MangaDexClient;
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
import com.magamochi.ingestion.model.dto.ProviderMangaMetadataDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ManualImportContentProvider;
import java.util.*;
@ -58,8 +60,7 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
}
// NOTE: this is getting only pt-br and en chapters for now, we may want to make this
// configurable
// later
// configurable later
var languagesToImport = Map.of("pt-br", "pt-BR", "en", "en-US");
return mangas.stream()
@ -85,7 +86,9 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
.map(
c ->
new ContentInfoDTO(
c.attributes().chapter() + " - " + c.attributes().title(),
nonNull(c.attributes().title())
? c.attributes().chapter() + " - " + c.attributes().title()
: c.attributes().chapter(),
c.id().toString(),
languagesToImport.get(c.attributes().translatedLanguage())))
.toList();
@ -112,7 +115,7 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
}
@Override
public String getMangaTitle(String value) {
public ProviderMangaMetadataDTO getMangaMetadata(String value) {
mangaDexRateLimiter.acquire();
var resultData = mangaDexClient.getManga(UUID.fromString(value)).data();
@ -120,13 +123,35 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
throw new UnprocessableException("Manga title not found for ID: " + value);
}
return resultData
.attributes()
.title()
.getOrDefault(
"en",
resultData.attributes().title().values().stream()
.findFirst()
.orElseThrow(() -> new UnprocessableException("No title available")));
var title =
resultData
.attributes()
.title()
.getOrDefault(
"en",
resultData.attributes().title().values().stream()
.findFirst()
.orElseThrow(() -> new UnprocessableException("No title available")));
Long malId = null;
Long aniListId = null;
if (nonNull(resultData.attributes().links())) {
var links = resultData.attributes().links();
try {
malId = links.containsKey("mal") ? Long.parseLong(links.get("mal")) : null;
} catch (NumberFormatException ignored) {
}
try {
aniListId = links.containsKey("al") ? Long.parseLong(links.get("al")) : null;
} catch (NumberFormatException ignored) {
}
}
return ProviderMangaMetadataDTO.builder()
.title(title)
.malId(malId)
.aniListId(aniListId)
.build();
}
}

14
src/main/java/com/magamochi/ingestion/service/ContentProviderService.java
View File

@ -34,6 +34,20 @@ public class ContentProviderService {
"Content Provider not found (ID: " + contentProviderId + ")."));
}
public ContentProvider getManualImportProvider(Long providerId) {
var provider = find(providerId);
if (!provider.isActive()) {
throw new IllegalStateException("Provider is not active");
}
if (!provider.getManualImport()) {
throw new IllegalArgumentException("Manual import not supported");
}
return provider;
}
public ContentProvider findManualImportContentProvider() {
return contentProviderRepository
.findByNameIgnoreCase(ContentProviders.MANUAL_IMPORT)

9
src/main/java/com/magamochi/ingestion/service/IngestionService.java
View File

@ -5,7 +5,9 @@ import com.magamochi.common.queue.command.MangaContentImageIngestCommand;
import com.magamochi.common.queue.command.MangaContentIngestCommand;
import com.magamochi.common.queue.command.MangaIngestCommand;
import com.magamochi.content.service.ContentService;
import com.magamochi.ingestion.model.dto.ProviderMangaMetadataDTO;
import com.magamochi.ingestion.providers.ContentProviderFactory;
import com.magamochi.ingestion.providers.ManualImportContentProviderFactory;
import com.magamochi.ingestion.providers.PagedContentProviderFactory;
import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
import com.magamochi.ingestion.queue.producer.MangaContentImageIngestProducer;
@ -24,6 +26,7 @@ public class IngestionService {
private final MangaContentProviderService mangaContentProviderService;
private final ContentProviderFactory contentProviderFactory;
private final ManualImportContentProviderFactory manualImportContentProviderFactory;
private final PagedContentProviderFactory pagedContentProviderFactory;
private final ProviderPageIngestProducer providerPageIngestProducer;
@ -103,4 +106,10 @@ public class IngestionService {
mangaContent.getId(), item.url(), item.position(), isLast));
});
}
public ProviderMangaMetadataDTO getMangaMetadataFromProvider(String providerName, String url) {
var contentProvider =
manualImportContentProviderFactory.getManualImportContentProvider(providerName);
return contentProvider.getMangaMetadata(url);
}
}

5
src/main/java/com/magamochi/model/dto/ImportRequestDTO.java
View File

@ -1,5 +0,0 @@
package com.magamochi.model.dto;
import jakarta.validation.constraints.NotNull;
public record ImportRequestDTO(String metadataId, String aniListId, @NotNull String id) {}

11
src/main/java/com/magamochi/model/dto/MangaDexMangaDTO.java
View File

@ -1,11 +0,0 @@
package com.magamochi.model.dto;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public record MangaDexMangaDTO(MangaData data) {
public record MangaData(UUID id, AttributesData attributes) {
public record AttributesData(Map<String, String> title, List<Map<String, String>> altTitles) {}
}
}

78
src/main/java/com/magamochi/service/ProviderManualMangaImportService.java
View File

@ -1,78 +0,0 @@
package com.magamochi.service;
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.model.entity.ContentProvider;
import com.magamochi.ingestion.model.repository.ContentProviderRepository;
import com.magamochi.ingestion.providers.ManualImportContentProviderFactory;
import com.magamochi.model.dto.ImportMangaResponseDTO;
import com.magamochi.model.dto.ImportRequestDTO;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.NotImplementedException;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class ProviderManualMangaImportService {
private final ManualImportContentProviderFactory contentProviderFactory;
private final ContentProviderRepository contentProviderRepository;
private final MangaContentProviderRepository mangaContentProviderRepository;
public ImportMangaResponseDTO importFromProvider(Long providerId, ImportRequestDTO requestDTO) {
throw new NotImplementedException();
// var provider = getProvider(providerId);
// var contentProvider =
// contentProviderFactory.getManualImportContentProvider(provider.getName());
//
// var title = contentProvider.getMangaTitle(requestDTO.id());
//
// var malId = nonNull(requestDTO.metadataId()) ? Long.parseLong(requestDTO.metadataId()) :
// null;
// var aniListId = nonNull(requestDTO.aniListId()) ? Long.parseLong(requestDTO.aniListId()) :
// null;
//
// var manga =
// nonNull(malId) || nonNull(aniListId)
// ? mangaCreationService.getOrCreateManga(malId, aniListId)
// : mangaCreationService.getOrCreateManga(title, requestDTO.id(), provider);
//
// if (isNull(manga)) {
// throw new NotFoundException("Manga could not be found or created for ID: " +
// requestDTO.id());
// }
//
// if (!mangaContentProviderRepository.existsByMangaAndContentProviderAndUrlIgnoreCase(
// manga, provider, requestDTO.id())) {
// mangaContentProviderRepository.save(
// MangaContentProvider.builder()
// .manga(manga)
// .mangaTitle(title)
// .contentProvider(provider)
// .url(requestDTO.id())
// .build());
// }
//
// return new ImportMangaResponseDTO(manga.getId());
}
public ContentProvider getProvider(Long providerId) {
var provider =
contentProviderRepository
.findById(providerId)
.orElseThrow(() -> new NotFoundException("Provider not found"));
if (!provider.isActive()) {
throw new IllegalStateException("Provider is not active");
}
if (!provider.getManualImport()) {
throw new IllegalArgumentException("Manual import not supported");
}
return provider;
}
}