mangamochi/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: manga file import

Browse Source
This commit is contained in:
Rodrigo Verdiani 2026-03-19 15:42:59 -03:00
parent 1d55420d64
commit 3fbd9d3085
27 changed files with 334 additions and 515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/magamochi/catalog/model/repository/MangaContentProviderRepository.java
View File

@ -2,9 +2,13 @@ package com.magamochi.catalog.model.repository;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import jakarta.validation.constraints.NotBlank;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
public interface MangaContentProviderRepository extends JpaRepository<MangaContentProvider, Long> {
boolean existsByMangaTitleIgnoreCaseAndContentProvider_Id(
@NotBlank String mangaTitle, long contentProviderId);
Optional<MangaContentProvider> findByManga_IdAndContentProvider_Id(
long mangaId, long contentProviderId);
}

15
src/main/java/com/magamochi/catalog/service/MangaContentProviderService.java
View File

@ -1,8 +1,10 @@
package com.magamochi.catalog.service;
import com.magamochi.catalog.model.entity.Manga;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.model.entity.ContentProvider;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@ -19,4 +21,17 @@ public class MangaContentProviderService {
new NotFoundException(
"MangaContentProvider not found - ID: " + mangaContentProviderId));
}
public MangaContentProvider findOrCreate(Manga manga, ContentProvider contentProvider) {
return mangaContentProviderRepository
.findByManga_IdAndContentProvider_Id(manga.getId(), contentProvider.getId())
.orElseGet(
() ->
mangaContentProviderRepository.save(
MangaContentProvider.builder()
.manga(manga)
.mangaTitle(manga.getTitle())
.contentProvider(contentProvider)
.build()));
}
}

30
src/main/java/com/magamochi/catalog/service/MangaResolutionService.java
View File

@ -98,6 +98,10 @@ public class MangaResolutionService {
return Optional.of(new ProviderResult(bestTitle, malId));
}
public Manga findOrCreateManga(Long aniListId, Long malId) {
return findOrCreateManga(null, aniListId, malId);
}
private Manga findOrCreateManga(String canonicalTitle, Long aniListId, Long malId) {
if (nonNull(aniListId)) {
var existingByAniList = mangaRepository.findByAniListId(aniListId);
@ -113,20 +117,24 @@ public class MangaResolutionService {
}
}
return mangaRepository
.findByTitleIgnoreCase(canonicalTitle)
.orElseGet(
() -> {
var newManga =
Manga.builder().title(canonicalTitle).malId(malId).aniListId(aniListId).build();
if (nonNull(canonicalTitle)) {
var existingByTitle = mangaRepository.findByTitleIgnoreCase(canonicalTitle);
if (existingByTitle.isPresent()) {
return existingByTitle.get();
}
}
var savedManga = mangaRepository.save(newManga);
return createAndNotifyManga(canonicalTitle, aniListId, malId);
}
mangaUpdateProducer.sendMangaUpdateCommand(
new MangaUpdateCommand(savedManga.getId()));
private Manga createAndNotifyManga(String title, Long aniListId, Long malId) {
var manga =
mangaRepository.save(
Manga.builder().title(title).aniListId(aniListId).malId(malId).build());
return savedManga;
});
mangaUpdateProducer.sendMangaUpdateCommand(new MangaUpdateCommand(manga.getId()));
return manga;
}
private record ProviderResult(String title, Long externalId) {}

2
src/main/java/com/magamochi/catalog/service/MyAnimeListService.java
View File

@ -77,7 +77,7 @@ public class MyAnimeListService {
}
private MangaStatus mapStatus(String malStatus) {
return switch (malStatus) {
return switch (malStatus.toLowerCase()) {
case "finished" -> MangaStatus.COMPLETED;
case "publishing" -> MangaStatus.ONGOING;
case "on hiatus" -> MangaStatus.HIATUS;

3
src/main/java/com/magamochi/ingestion/providers/ContentProviders.java → src/main/java/com/magamochi/common/ContentProviders.java
View File

@ -1,8 +1,9 @@
package com.magamochi.ingestion.providers;
package com.magamochi.common;
public class ContentProviders {
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
public static final String MANGA_DEX = "MangaDex";
public static final String MANUAL_IMPORT = "Manual Import";
}

8
src/main/java/com/magamochi/common/config/RabbitConfig.java
View File

@ -37,6 +37,9 @@ public class RabbitConfig {
@Value("${queues.image-fetch}")
private String imageFetchQueue;
@Value("${queues.file-import}")
private String fileImportQueue;
@Value("${topics.image-updates}")
private String imageUpdatesTopic;
@ -68,6 +71,11 @@ public class RabbitConfig {
return new Queue(mangaCoverUpdateQueue, false);
}
@Bean
public Queue fileImportQueue() {
return new Queue(fileImportQueue, false);
}
@Bean
public Binding bindingMangaCoverUpdateQueue(
Queue mangaCoverUpdateQueue, TopicExchange imageUpdatesExchange) {

17
src/main/java/com/magamochi/content/controller/ContentController.java
View File

@ -1,10 +1,12 @@
package com.magamochi.content.controller;
import com.magamochi.common.model.dto.DefaultResponseDTO;
import com.magamochi.content.model.dto.FileImportRequestDTO;
import com.magamochi.content.model.dto.MangaContentDTO;
import com.magamochi.content.model.dto.MangaContentImagesDTO;
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
import com.magamochi.content.service.ContentDownloadService;
import com.magamochi.content.service.ContentImportService;
import com.magamochi.content.service.ContentService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
@ -25,6 +27,7 @@ import org.springframework.web.bind.annotation.*;
public class ContentController {
private final ContentService contentService;
private final ContentDownloadService contentDownloadService;
private final ContentImportService contentImportService;
@Operation(
summary = "Get the content for a specific manga/content provider combination",
@ -76,4 +79,18 @@ public class ContentController {
.header("Content-Disposition", "attachment; filename=\"" + response.filename() + "\"")
.body(response.content());
}
@Operation(
summary = "Import multiple files",
description = "Accepts multiple content files via multipart/form-data and processes them.",
tags = {"Content"},
operationId = "importContentFiles")
@PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public DefaultResponseDTO<Void> importContentFiles(
@ModelAttribute FileImportRequestDTO requestDTO) {
contentImportService.importFiles(
requestDTO.malId(), requestDTO.aniListId(), requestDTO.files());
return DefaultResponseDTO.ok().build();
}
}

6
src/main/java/com/magamochi/content/model/dto/FileImportRequestDTO.java Normal file
View File

@ -0,0 +1,6 @@
package com.magamochi.content.model.dto;
import java.util.List;
import org.springframework.web.multipart.MultipartFile;
public record FileImportRequestDTO(String malId, String aniListId, List<MultipartFile> files) {}

3
src/main/java/com/magamochi/content/queue/command/FileImportCommand.java Normal file
View File

@ -0,0 +1,3 @@
package com.magamochi.content.queue.command;
public record FileImportCommand(long mangaContentProviderId, String filename) {}

21
src/main/java/com/magamochi/content/queue/consumer/FileImportConsumer.java Normal file
View File

@ -0,0 +1,21 @@
package com.magamochi.content.queue.consumer;
import com.magamochi.content.queue.command.FileImportCommand;
import com.magamochi.content.service.ContentImportService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class FileImportConsumer {
private final ContentImportService contentImportService;
@RabbitListener(queues = "${queues.file-import}")
public void receiveFileImportCommand(FileImportCommand command) {
log.info("Received file import command: {}", command);
contentImportService.importFile(command.mangaContentProviderId(), command.filename());
}
}

23
src/main/java/com/magamochi/content/queue/producer/FileImportProducer.java Normal file
View File

@ -0,0 +1,23 @@
package com.magamochi.content.queue.producer;
import com.magamochi.content.queue.command.FileImportCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class FileImportProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${queues.file-import}")
private String fileImportQueue;
public void sendFileImportCommand(FileImportCommand command) {
rabbitTemplate.convertAndSend(fileImportQueue, command);
log.info("Sent file import command: {}", command);
}
}

151
src/main/java/com/magamochi/content/service/ContentImportService.java Normal file
View File

@ -0,0 +1,151 @@
package com.magamochi.content.service;
import static java.util.Objects.isNull;
import static org.apache.commons.lang3.StringUtils.isBlank;
import com.magamochi.catalog.service.MangaContentProviderService;
import com.magamochi.catalog.service.MangaResolutionService;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.common.model.enumeration.ContentType;
import com.magamochi.content.model.entity.MangaContentImage;
import com.magamochi.content.model.repository.MangaContentImageRepository;
import com.magamochi.content.queue.command.FileImportCommand;
import com.magamochi.content.queue.producer.FileImportProducer;
import com.magamochi.image.service.ImageFetchService;
import com.magamochi.image.service.ImageService;
import com.magamochi.image.service.S3Service;
import com.magamochi.ingestion.service.ContentProviderService;
import jakarta.validation.constraints.NotNull;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
@Log4j2
@Service
@RequiredArgsConstructor
public class ContentImportService {
private final ContentProviderService contentProviderService;
private final MangaResolutionService mangaResolutionService;
private final MangaContentProviderService mangaContentProviderService;
private final ContentIngestService contentIngestService;
private final ImageFetchService imageFetchService;
private final S3Service s3Service;
private final FileImportProducer fileImportProducer;
private final MangaContentImageRepository mangaContentImageRepository;
private final ImageService imageService;
public void importFiles(String malId, String aniListId, @NotNull List<MultipartFile> files) {
if (isBlank(malId) && isBlank(aniListId)) {
throw new UnprocessableException("Either MyAnimeList or AniList IDs are required.");
}
if (files.isEmpty()) {
return;
}
var manga =
mangaResolutionService.findOrCreateManga(
isBlank(aniListId) ? null : Long.parseLong(aniListId),
isBlank(malId) ? null : Long.parseLong(malId));
var contentProvider = contentProviderService.findManualImportContentProvider();
var mangaContentProvider = mangaContentProviderService.findOrCreate(manga, contentProvider);
var sortedFiles = files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList();
sortedFiles.forEach(
file -> {
try {
var filename =
s3Service.uploadFile(
file.getBytes(),
file.getContentType(),
"temp/import/" + file.getOriginalFilename());
log.info("Temp file uploaded to S3: {}", filename);
fileImportProducer.sendFileImportCommand(
new FileImportCommand(mangaContentProvider.getId(), filename));
} catch (IOException e) {
throw new UnprocessableException("Failed to upload file to S3");
}
});
}
@Transactional
public void importFile(Long mangaContentProviderId, String filename) {
var mangaContent =
contentIngestService.ingest(
mangaContentProviderId,
removeImportPrefix(removeFileExtension(filename)),
null,
"en-US");
try {
var is = s3Service.getFileStream(filename);
var zis = new ZipInputStream(is);
ZipEntry entry;
var position = 0;
while ((entry = zis.getNextEntry()) != null) {
if (entry.isDirectory()) {
continue;
}
var os = new ByteArrayOutputStream();
zis.transferTo(os);
var bytes = os.toByteArray();
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
var image = imageService.find(imageId);
mangaContentImageRepository.save(
MangaContentImage.builder()
.image(image)
.mangaContent(mangaContent)
.position(position)
.build());
zis.closeEntry();
}
} catch (Exception e) {
throw new RuntimeException(e);
}
mangaContent.setDownloaded(true);
s3Service.deleteObjects(Set.of(filename));
}
private String removeFileExtension(String filename) {
if (isBlank(filename)) {
return filename;
}
int lastDotIndex = filename.lastIndexOf('.');
// No dot, or dot is the first character (like .gitignore)
if (lastDotIndex <= 0) {
return filename;
}
return filename.substring(0, lastDotIndex);
}
private String removeImportPrefix(String path) {
if (isNull(path)) {
return null;
}
return path.replace("temp/import/", "");
}
}

23
src/main/java/com/magamochi/content/service/ContentIngestService.java
View File

@ -1,6 +1,7 @@
package com.magamochi.content.service;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import com.magamochi.catalog.service.LanguageService;
import com.magamochi.catalog.service.MangaContentProviderService;
@ -34,22 +35,24 @@ public class ContentIngestService {
private final ImageFetchProducer imageFetchProducer;
private final ImageService imageService;
public void ingest(
public MangaContent ingest(
long mangaContentProviderId,
@NotBlank String title,
@NotBlank String url,
String url,
@NotBlank String languageCode) {
log.info("Ingesting Manga Content ({}) for provider {}", title, mangaContentProviderId);
var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId);
if (mangaContentRepository.existsByMangaContentProvider_IdAndUrlIgnoreCase(
mangaContentProvider.getId(), url)) {
log.info(
"Manga Content ({}) for provider {} already exists. Skipped.",
title,
mangaContentProviderId);
return;
if (nonNull(url)) {
if (mangaContentRepository.existsByMangaContentProvider_IdAndUrlIgnoreCase(
mangaContentProvider.getId(), url)) {
log.info(
"Manga Content ({}) for provider {} already exists. Skipped.",
title,
mangaContentProviderId);
return null;
}
}
var language = languageService.find(languageCode);
@ -68,6 +71,8 @@ public class ContentIngestService {
title,
mangaContentProviderId,
mangaContent.getId());
return mangaContent;
}
@Transactional

32
src/main/java/com/magamochi/controller/MangaImportController.java
View File

@ -6,15 +6,8 @@ import com.magamochi.model.dto.ImportRequestDTO;
// import com.magamochi.service.MangaImportService;
import com.magamochi.service.ProviderManualMangaImportService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
@RestController
@RequestMapping("/manga/import")
@ -34,29 +27,4 @@ public class MangaImportController {
return DefaultResponseDTO.ok(
providerManualMangaImportService.importFromProvider(providerId, requestDTO));
}
@Operation(
summary = "Upload multiple files",
description = "Accepts multiple files via multipart/form-data and processes them.",
tags = {"Manga Import"},
operationId = "importMultipleFiles")
@PostMapping(
value = "/upload",
consumes = {"multipart/form-data"})
public DefaultResponseDTO<Void> uploadMultipleFiles(
@RequestPart("malId") @NotBlank String malId,
@Parameter(
description = "List of files to upload",
required = true,
content =
@Content(
mediaType = "multipart/form-data",
schema = @Schema(type = "array", format = "binary")))
@RequestPart("files")
@NotNull
List<MultipartFile> files) {
// mangaImportService.importMangaFiles(malId, files);
return DefaultResponseDTO.ok().build();
}
}

38
src/main/java/com/magamochi/image/service/ImageFetchService.java
View File

@ -37,28 +37,36 @@ public class ImageFetchService {
var imageBytes = response.body();
var fileContentType = resolveContentType(response, imageBytes);
var fileHash = computeHash(imageBytes);
return imageManagerService.upload(
imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash);
return uploadImage(imageBytes, response, contentType);
} catch (Exception e) {
log.error("Failed to fetch image from URL: {}", imageUrl, e);
return null;
}
}
private String resolveContentType(HttpResponse<byte[]> response, byte[] fileBytes) {
var headerType =
response
.headers()
.firstValue("Content-Type")
.map(val -> val.split(";")[0].trim().toLowerCase())
.orElse(null);
public UUID uploadImage(
byte[] imageBytes, HttpResponse<byte[]> httpResponse, ContentType contentType)
throws NoSuchAlgorithmException {
var fileContentType = resolveContentType(httpResponse, imageBytes);
if (nonNull(headerType) && headerType.startsWith("image/")) {
return headerType;
var fileHash = computeHash(imageBytes);
return imageManagerService.upload(
imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash);
}
private String resolveContentType(HttpResponse<byte[]> response, byte[] fileBytes) {
if (nonNull(response)) {
var headerType =
response
.headers()
.firstValue("Content-Type")
.map(val -> val.split(";")[0].trim().toLowerCase())
.orElse(null);
if (nonNull(headerType) && headerType.startsWith("image/")) {
return headerType;
}
}
return tika.detect(fileBytes);

5
src/main/java/com/magamochi/ingestion/model/repository/ContentProviderRepository.java
View File

@ -1,6 +1,9 @@
package com.magamochi.ingestion.model.repository;
import com.magamochi.ingestion.model.entity.ContentProvider;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
public interface ContentProviderRepository extends JpaRepository<ContentProvider, Long> {}
public interface ContentProviderRepository extends JpaRepository<ContentProvider, Long> {
Optional<ContentProvider> findByNameIgnoreCase(String name);
}

2
src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java
View File

@ -5,11 +5,11 @@ import static java.util.Objects.isNull;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.client.MangaDexClient;
import com.magamochi.common.ContentProviders;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.ManualImportContentProvider;
import java.util.*;
import java.util.stream.IntStream;

2
src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java
View File

@ -1,11 +1,11 @@
package com.magamochi.ingestion.providers.impl;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.common.ContentProviders;
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.PagedContentProvider;
import java.io.IOException;
import java.util.*;

2
src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java
View File

@ -3,11 +3,11 @@ package com.magamochi.ingestion.providers.impl;
import static java.util.Objects.nonNull;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.common.ContentProviders;
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.PagedContentProvider;
import com.magamochi.ingestion.service.FlareService;
import java.util.*;

2
src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java
View File

@ -4,11 +4,11 @@ import static java.util.Objects.isNull;
import static org.apache.commons.lang3.StringUtils.isBlank;
import com.magamochi.catalog.model.entity.MangaContentProvider;
import com.magamochi.common.ContentProviders;
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.PagedContentProvider;
import com.magamochi.ingestion.service.FlareService;
import java.util.*;

7
src/main/java/com/magamochi/ingestion/service/ContentProviderService.java
View File

@ -2,6 +2,7 @@ package com.magamochi.ingestion.service;
import static java.util.Objects.nonNull;
import com.magamochi.common.ContentProviders;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
import com.magamochi.ingestion.model.entity.ContentProvider;
@ -32,4 +33,10 @@ public class ContentProviderService {
new NotFoundException(
"Content Provider not found (ID: " + contentProviderId + ")."));
}
public ContentProvider findManualImportContentProvider() {
return contentProviderRepository
.findByNameIgnoreCase(ContentProviders.MANUAL_IMPORT)
.orElseThrow(() -> new NotFoundException("Manual Import Content Provider not found"));
}
}

438
src/main/java/com/magamochi/service/MangaImportService.java
View File

@ -1,438 +0,0 @@
// package com.magamochi.service;
//
// import static java.util.Objects.isNull;
// import static java.util.Objects.nonNull;
//
// import com.google.common.util.concurrent.RateLimiter;
// import com.magamochi.catalog.model.entity.Genre;
// import com.magamochi.catalog.model.repository.GenreRepository;
// import com.magamochi.catalog.client.AniListClient;
// import com.magamochi.catalog.client.JikanClient;
// import com.magamochi.common.exception.NotFoundException;
// import com.magamochi.ingestion.model.entity.ContentProvider;
// import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
// import com.magamochi.model.entity.*;
// import com.magamochi.model.repository.*;
// import com.magamochi.catalog.util.DoubleUtil;
// import java.io.*;
// import java.net.URI;
// import java.net.URISyntaxException;
// import java.net.URL;
// import java.time.OffsetDateTime;
// import java.time.ZoneOffset;
// import java.util.ArrayList;
// import java.util.Comparator;
// import java.util.List;
// import java.util.stream.IntStream;
// import java.util.zip.ZipEntry;
// import java.util.zip.ZipInputStream;
// import lombok.RequiredArgsConstructor;
// import lombok.extern.log4j.Log4j2;
// import org.apache.commons.lang3.StringUtils;
// import org.springframework.stereotype.Service;
// import org.springframework.web.multipart.MultipartFile;
//
// @Log4j2
//// @Service
// @RequiredArgsConstructor
// public class MangaImportService {
// private final ProviderService providerService;
// private final MangaCreationService mangaCreationService;
// private final ImageService imageService;
// private final LanguageService languageService;
//
// private final GenreRepository genreRepository;
// private final MangaGenreRepository mangaGenreRepository;
// private final MangaContentProviderRepository mangaContentProviderRepository;
// private final AuthorRepository authorRepository;
// private final MangaAuthorRepository mangaAuthorRepository;
// private final MangaChapterRepository mangaChapterRepository;
// private final MangaRepository mangaRepository;
//
// private final JikanClient jikanClient;
// private final AniListClient aniListClient;
// private final MangaChapterImageRepository mangaChapterImageRepository;
// private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository;
//
// private final RateLimiter jikanRateLimiter;
//
// public void importMangaFiles(String malId, List<MultipartFile> files) {
// log.info("Importing manga files for MAL ID {}", malId);
// var provider = providerService.getOrCreateProvider("Manual Import", false);
//
// jikanRateLimiter.acquire();
// var mangaData = jikanClient.getMangaById(Long.parseLong(malId));
//
// var mangaProvider = getOrCreateMangaProvider(mangaData.data().title(), provider);
//
// var sortedFiles =
// files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList();
//
// IntStream.rangeClosed(1, sortedFiles.size())
// .forEach(
// fileIndex -> {
// var file = sortedFiles.get(fileIndex - 1);
// log.info(
// "Importing file {}/{}: {}, for Mangá {}",
// fileIndex,
// sortedFiles.size(),
// file.getOriginalFilename(),
// mangaProvider.getManga().getTitle());
//
// var chapter =
// persistMangaChapter(
// mangaProvider,
// new ContentProviderMangaChapterResponseDTO(
// removeFileExtension(file.getOriginalFilename()),
// "manual_" + file.getOriginalFilename(),
// file.getOriginalFilename(),
// "en-US"));
//
// List<MangaChapterImage> allChapterImages = new ArrayList<>();
// try (InputStream is = file.getInputStream();
// ZipInputStream zis = new ZipInputStream(is)) {
// ZipEntry entry;
// var position = 0;
//
// while ((entry = zis.getNextEntry()) != null) {
// if (entry.isDirectory()) {
// continue;
// }
//
// var os = new ByteArrayOutputStream();
// zis.transferTo(os);
// var bytes = os.toByteArray();
//
// var image =
// imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapter.getId());
//
// var chapterImage =
// MangaChapterImage.builder()
// .position(position++)
// .image(image)
// .mangaChapter(chapter)
// .build();
//
// allChapterImages.add(chapterImage);
// zis.closeEntry();
// }
//
// log.info("Chapter images added for chapter {}", chapter.getTitle());
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
//
// mangaChapterImageRepository.saveAll(allChapterImages);
// chapter.setDownloaded(true);
// mangaChapterRepository.save(chapter);
// });
//
// log.info("Import manga files for MAL ID {} completed.", malId);
// }
//
// public void updateMangaData(Long mangaId) {
// var manga =
// mangaRepository
// .findById(mangaId)
// .orElseThrow(() -> new NotFoundException("Manga not found for ID: " + mangaId));
//
// updateMangaData(manga);
// }
//
// public void updateMangaData(Manga manga) {
// log.info("Updating manga {}", manga.getTitle());
//
// if (nonNull(manga.getMalId())) {
// try {
// updateFromJikan(manga);
// return;
// } catch (Exception e) {
// log.warn(
// "Error updating manga data from Jikan for manga {}. Trying AniList... Error: {}",
// manga.getTitle(),
// e.getMessage());
// }
// }
//
// if (nonNull(manga.getAniListId())) {
// try {
// updateFromAniList(manga);
// return;
// } catch (Exception e) {
// log.warn(
// "Error updating manga data from AniList for manga {}. Error: {}",
// manga.getTitle(),
// e.getMessage());
// }
// }
//
// log.warn(
// "Could not update manga data for {}. No provider data available/found.",
// manga.getTitle());
// }
//
// private void updateFromJikan(Manga manga) throws IOException, URISyntaxException {
// jikanRateLimiter.acquire();
// var mangaData = jikanClient.getMangaById(manga.getMalId());
//
// manga.setSynopsis(mangaData.data().synopsis());
// manga.setStatus(mangaData.data().status());
// manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2));
// manga.setPublishedFrom(mangaData.data().published().from());
// manga.setPublishedTo(mangaData.data().published().to());
// manga.setChapterCount(mangaData.data().chapters());
//
// var authors =
// mangaData.data().authors().stream()
// .map(
// authorData ->
// authorRepository
// .findByMalId(authorData.mal_id())
// .orElseGet(
// () ->
// authorRepository.save(
// Author.builder()
// .malId(authorData.mal_id())
// .name(authorData.name())
// .build())))
// .toList();
//
// updateMangaAuthors(manga, authors);
//
// var genres =
// mangaData.data().genres().stream()
// .map(
// genreData ->
// genreRepository
// .findByMalId(genreData.mal_id())
// .orElseGet(
// () ->
// genreRepository.save(
// Genre.builder()
// .malId(genreData.mal_id())
// .name(genreData.name())
// .build())))
// .toList();
//
// updateMangaGenres(manga, genres);
//
// if (isNull(manga.getCoverImage())) {
// downloadCoverImage(manga, mangaData.data().images().jpg().large_image_url());
// }
//
// var mangaEntity = mangaRepository.save(manga);
// var alternativeTitles =
// mangaData.data().title_synonyms().stream()
// .map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build())
// .toList();
// mangaAlternativeTitlesRepository.saveAll(alternativeTitles);
// }
//
// private void updateFromAniList(Manga manga) throws IOException, URISyntaxException {
// var query =
// """
// query ($id: Int) {
// Media (id: $id, type: MANGA) {
// startDate { year month day }
// endDate { year month day }
// description
// status
// averageScore
// chapters
// coverImage { large }
// genres
// staff {
// edges {
// role
// node {
// name {
// full
// }
// }
// }
// }
// }
// }
// """;
// var request =
// new AniListClient.GraphQLRequest(
// query, new AniListClient.GraphQLRequest.Variables(manga.getAniListId()));
// var media = aniListClient.getManga(request).data().Media();
//
// manga.setSynopsis(media.description());
// manga.setStatus(mapAniListStatus(media.status()));
// manga.setScore(DoubleUtil.round((double) media.averageScore() / 10, 2)); // 0-100 -> 0-10
// manga.setPublishedFrom(convertFuzzyDate(media.startDate()));
// manga.setPublishedTo(convertFuzzyDate(media.endDate()));
// manga.setChapterCount(media.chapters());
//
// var authors =
// media.staff().edges().stream()
// .filter(edge -> isAuthorRole(edge.role()))
// .map(edge -> edge.node().name().full())
// .distinct()
// .map(
// name ->
// authorRepository
// .findByName(name)
// .orElseGet(
// () -> authorRepository.save(Author.builder().name(name).build())))
// .toList();
//
// updateMangaAuthors(manga, authors);
//
// var genres =
// media.genres().stream()
// .map(
// name ->
// genreRepository
// .findByName(name)
// .orElseGet(() ->
// genreRepository.save(Genre.builder().name(name).build())))
// .toList();
//
// updateMangaGenres(manga, genres);
//
// if (isNull(manga.getCoverImage())) {
// downloadCoverImage(manga, media.coverImage().large());
// }
//
// mangaRepository.save(manga);
// }
//
// private boolean isAuthorRole(String role) {
// return role.equalsIgnoreCase("Story & Art")
// || role.equalsIgnoreCase("Story")
// || role.equalsIgnoreCase("Art");
// }
//
// private String mapAniListStatus(String status) {
// return switch (status) {
// case "RELEASING" -> "Publishing";
// case "FINISHED" -> "Finished";
// case "NOT_YET_RELEASED" -> "Not yet published";
// default -> "Unknown";
// };
// }
//
// private OffsetDateTime convertFuzzyDate(AniListClient.MangaResponse.Manga.FuzzyDate date) {
// if (isNull(date) || isNull(date.year())) {
// return null;
// }
// return OffsetDateTime.of(
// date.year(),
// isNull(date.month()) ? 1 : date.month(),
// isNull(date.day()) ? 1 : date.day(),
// 0,
// 0,
// 0,
// 0,
// ZoneOffset.UTC);
// }
//
// private void updateMangaAuthors(Manga manga, List<Author> authors) {
// var mangaAuthors =
// authors.stream()
// .map(
// author ->
// mangaAuthorRepository
// .findByMangaAndAuthor(manga, author)
// .orElseGet(
// () ->
// mangaAuthorRepository.save(
// MangaAuthor.builder().manga(manga).author(author).build())))
// .toList();
// manga.setMangaAuthors(mangaAuthors);
// }
//
// private void updateMangaGenres(Manga manga, List<Genre> genres) {
// var mangaGenres =
// genres.stream()
// .map(
// genre ->
// mangaGenreRepository
// .findByMangaAndGenre(manga, genre)
// .orElseGet(
// () ->
// mangaGenreRepository.save(
// MangaGenre.builder().manga(manga).genre(genre).build())))
// .toList();
// manga.setMangaGenres(mangaGenres);
// }
//
// private void downloadCoverImage(Manga manga, String imageUrl)
// throws IOException, URISyntaxException {
// var inputStream =
// new BufferedInputStream(new URL(new URI(imageUrl).toASCIIString()).openStream());
//
// var bytes = inputStream.readAllBytes();
//
// inputStream.close();
// var image = imageService.uploadImage(bytes, "image/jpeg", "cover");
//
// manga.setCoverImage(image);
// }
//
// public MangaChapter persistMangaChapter(
// MangaContentProvider mangaContentProvider, ContentProviderMangaChapterResponseDTO chapter) {
// var mangaChapter =
// mangaChapterRepository
// .findByMangaContentProviderAndUrlIgnoreCase(mangaContentProvider,
// chapter.url())
// .orElseGet(MangaChapter::new);
//
// mangaChapter.setMangaContentProvider(mangaContentProvider);
// mangaChapter.setTitle(chapter.title());
// mangaChapter.setUrl(chapter.url());
//
// var language = languageService.getOrThrow(chapter.languageCode());
// mangaChapter.setLanguage(language);
//
// if (nonNull(chapter.chapter())) {
// try {
// mangaChapter.setChapterNumber(Integer.parseInt(chapter.chapter()));
// } catch (NumberFormatException e) {
// log.warn(
// "Could not parse chapter number {} from manga {}",
// chapter.chapter(),
// mangaContentProvider.getManga().getTitle());
// }
// }
//
// return mangaChapterRepository.save(mangaChapter);
// }
//
// private MangaContentProvider getOrCreateMangaProvider(
// String title, ContentProvider contentProvider) {
// return mangaContentProviderRepository
// .findByMangaTitleIgnoreCaseAndContentProvider(title, contentProvider)
// .orElseGet(
// () -> {
// jikanRateLimiter.acquire();
// var manga = mangaCreationService.getOrCreateManga(title, "manual", contentProvider);
//
// return mangaContentProviderRepository.save(
// MangaContentProvider.builder()
// .manga(manga)
// .mangaTitle(manga.getTitle())
// .contentProvider(contentProvider)
// .url("manual")
// .build());
// });
// }
//
// private String removeFileExtension(String filename) {
// if (StringUtils.isBlank(filename)) {
// return filename;
// }
//
// int lastDotIndex = filename.lastIndexOf('.');
//
// // No dot, or dot is the first character (like .gitignore)
// if (lastDotIndex <= 0) {
// return filename;
// }
//
// return filename.substring(0, lastDotIndex);
// }
// }

3
src/main/java/com/magamochi/service/OldMangaService.java
View File

@ -5,7 +5,6 @@ import com.magamochi.common.exception.NotFoundException;
import com.magamochi.content.model.entity.MangaContent;
import com.magamochi.model.dto.*;
import com.magamochi.queue.MangaChapterDownloadProducer;
import com.magamochi.user.service.UserService;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
@ -15,8 +14,6 @@ import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class OldMangaService {
private final UserService userService;
private final MangaContentProviderRepository mangaContentProviderRepository;
private final MangaChapterDownloadProducer mangaChapterDownloadProducer;

1
src/main/resources/application.yml
View File

@ -68,6 +68,7 @@ queues:
provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:mangamochi.provider.page.ingest}
image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch}
manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update}
file-import: ${FILE_IMPORT_QUEUE:mangamochi.file.import}
routing-key:
image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update}

6
src/main/resources/db/migration/V0004__CONTENT_PROVIDERS.sql Normal file
View File

@ -0,0 +1,6 @@
ALTER TABLE content_providers
ALTER COLUMN url DROP NOT NULL;
INSERT INTO content_providers(name, url, active, supports_content_fetch, manual_import)
VALUES ('MangaDex', NULL, TRUE, TRUE, TRUE),
('Manual Import', NULL, TRUE, FALSE, FALSE);

2
src/main/resources/db/migration/V0005__MANGA_CONTENT.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE manga_contents
ALTER COLUMN url DROP NOT NULL;

3
src/main/resources/db/migration/V0006__MANGA_CONTENT.sql Normal file
View File

@ -0,0 +1,3 @@
ALTER TABLE manga_content_provider
ALTER COLUMN url DROP NOT NULL,
ALTER COLUMN manga_title DROP NOT NULL;