diff --git a/.env b/.env index 715a91d..060d5d8 100644 --- a/.env +++ b/.env @@ -6,4 +6,9 @@ MINIO_ENDPOINT=http://omv.badger-pirarucu.ts.net:9000 MINIO_USER=admin MINIO_PASS=!E9v4i0v3 -WEBSCRAPPER_ENDPOINT=http://localhost:8090/url \ No newline at end of file +WEBSCRAPPER_ENDPOINT=http://localhost:8090/url + +MANGADEX_USER=rocverde +MANGADEX_PASS=!A3u8e4s0 +MANGADEX_CLIENT_ID=personal-client-3c21667a-6de3-4273-94c4-e6014690f128-68830913 +MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS \ No newline at end of file diff --git a/src/main/java/com/magamochi/mangamochi/client/JikanClient.java b/src/main/java/com/magamochi/mangamochi/client/JikanClient.java index b805d4e..6ed37b6 100644 --- a/src/main/java/com/magamochi/mangamochi/client/JikanClient.java +++ b/src/main/java/com/magamochi/mangamochi/client/JikanClient.java @@ -25,6 +25,7 @@ public interface JikanClient { public record MangaData( Long mal_id, ImageData images, + String title, List title_synonyms, String status, boolean publishing, diff --git a/src/main/java/com/magamochi/mangamochi/client/MangaDexAuthenticationClient.java b/src/main/java/com/magamochi/mangamochi/client/MangaDexAuthenticationClient.java new file mode 100644 index 0000000..0f93025 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/client/MangaDexAuthenticationClient.java @@ -0,0 +1,33 @@ +package com.magamochi.mangamochi.client; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.http.MediaType; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.*; + +@FeignClient(name = "mangaDexAuthentication", url = "https://auth.mangadex.org") +public interface MangaDexAuthenticationClient { + + // @Headers("Content-Type: application/x-www-form-urlencoded") + @PostMapping( + value = "/realms/mangadex/protocol/openid-connect/token", + consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE) + AuthTokenResponse authenticate(@RequestBody MultiValueMap form); + + public record AuthTokenResponse( + @JsonProperty("access_token") String accessToken, + @JsonProperty("refresh_token") String refreshToken) {} + + public static MultiValueMap build( + String username, String password, String clientId, String clientSecret) { + MultiValueMap form = new LinkedMultiValueMap<>(); + form.add("grant_type", "password"); + form.add("username", username); + form.add("password", password); + form.add("client_id", clientId); + form.add("client_secret", clientSecret); + return form; + } +} diff --git a/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java b/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java new file mode 100644 index 0000000..eefc22d --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java @@ -0,0 +1,45 @@ +package com.magamochi.mangamochi.client; + +import com.magamochi.mangamochi.model.dto.MangaDexMangaDTO; +import java.util.List; +import java.util.UUID; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.*; + +@FeignClient(name = "mangaDex", url = "https://api.mangadex.org") +public interface MangaDexClient { + @GetMapping("/manga/{id}") + MangaDexMangaDTO getManga( + @PathVariable UUID id, @RequestHeader("Authorization") String authorization); + + @GetMapping("/manga/{id}/feed") + MangaDexMangaFeedDTO getMangaFeed( + @PathVariable UUID id, @RequestHeader("Authorization") String authorization); + + @GetMapping("/manga/{id}/feed") + MangaDexMangaFeedDTO getMangaFeed( + @PathVariable UUID id, + @RequestParam int limit, + @RequestParam int offset, + @RequestHeader("Authorization") String authorization); + + @GetMapping("/at-home/server/{chapterId}") + MangaChapterDataDTO getMangaChapter( + @PathVariable UUID chapterId, @RequestHeader("Authorization") String authorization); + + record MangaDexMangaFeedDTO( + List data, Integer limit, Integer offset, Integer total) { + public record MangaFeedData(UUID id, String type, MangaFeedAttributes attributes) { + public record MangaFeedAttributes( + String volume, + String chapter, + String title, + String translatedLanguage, + Boolean isUnavailable) {} + } + } + + record MangaChapterDataDTO(String baseUrl, ChapterData chapter) { + public record ChapterData(String hash, List data) {} + } +} diff --git a/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java b/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java index 3cf970b..12b239d 100644 --- a/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java +++ b/src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java @@ -5,7 +5,7 @@ import org.springframework.cloud.openfeign.FeignClient; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -@FeignClient(name = "rapidFuzz", url = "http://127.0.0.1:9000/match-title") +@FeignClient(name = "rapidFuzz", url = "http://127.0.0.1:8000/match-title") public interface RapidFuzzClient { @PostMapping Response mangaSearch(@RequestBody Request dto); diff --git a/src/main/java/com/magamochi/mangamochi/controller/MangaImportController.java b/src/main/java/com/magamochi/mangamochi/controller/MangaImportController.java new file mode 100644 index 0000000..ab6c467 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/controller/MangaImportController.java @@ -0,0 +1,57 @@ +package com.magamochi.mangamochi.controller; + +import com.magamochi.mangamochi.model.dto.*; +import com.magamochi.mangamochi.service.MangaImportService; +import com.magamochi.mangamochi.service.providers.impl.MangaDexProvider; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequestMapping("/manga/import") +@RequiredArgsConstructor +public class MangaImportController { + private final MangaDexProvider mangaDexProvider; + private final MangaImportService mangaImportService; + + @Operation( + summary = "Import manga from MangaDex", + description = "Imports manga data from MangaDex into the local database.", + tags = {"Manga Import"}, + operationId = "importFromMangaDex") + @PostMapping("/manga-dex") + public ImportMangaDexResponseDTO importFromMangaDex( + @RequestBody ImportMangaDexRequestDTO requestDTO) { + return mangaDexProvider.importManga(requestDTO.id()); + } + + @Operation( + summary = "Upload multiple files", + description = "Accepts multiple files via multipart/form-data and processes them.", + tags = {"Manga Import"}, + operationId = "importMultipleFiles") + @PostMapping( + value = "/upload", + consumes = {"multipart/form-data"}) + public void uploadMultipleFiles( + @RequestPart("malId") @NotBlank String malId, + @Parameter( + description = "List of files to upload", + required = true, + content = + @Content( + mediaType = "multipart/form-data", + schema = @Schema(type = "array", format = "binary"))) + @RequestPart("files") + @NotNull + List files) { + mangaImportService.importMangaFiles(malId, files); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/ContentProviderMangaChapterResponseDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/ContentProviderMangaChapterResponseDTO.java index e2fcebf..950dcd3 100644 --- a/src/main/java/com/magamochi/mangamochi/model/dto/ContentProviderMangaChapterResponseDTO.java +++ b/src/main/java/com/magamochi/mangamochi/model/dto/ContentProviderMangaChapterResponseDTO.java @@ -3,4 +3,4 @@ package com.magamochi.mangamochi.model.dto; import jakarta.validation.constraints.NotBlank; public record ContentProviderMangaChapterResponseDTO( - @NotBlank String chapterTitle, @NotBlank String chapterUrl) {} + @NotBlank String chapterTitle, @NotBlank String chapterUrl, String chapter, String language) {} diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexRequestDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexRequestDTO.java new file mode 100644 index 0000000..7c3ddf5 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexRequestDTO.java @@ -0,0 +1,6 @@ +package com.magamochi.mangamochi.model.dto; + +import jakarta.validation.constraints.NotNull; +import java.util.UUID; + +public record ImportMangaDexRequestDTO(@NotNull UUID id) {} diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexResponseDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexResponseDTO.java new file mode 100644 index 0000000..3f70358 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexResponseDTO.java @@ -0,0 +1,5 @@ +package com.magamochi.mangamochi.model.dto; + +import jakarta.validation.constraints.NotNull; + +public record ImportMangaDexResponseDTO(@NotNull Long id) {} diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/MangaDexMangaDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/MangaDexMangaDTO.java new file mode 100644 index 0000000..38c6753 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/model/dto/MangaDexMangaDTO.java @@ -0,0 +1,11 @@ +package com.magamochi.mangamochi.model.dto; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public record MangaDexMangaDTO(MangaData data) { + public record MangaData(UUID id, AttributesData attributes) { + public record AttributesData(Map title, List> altTitles) {} + } +} diff --git a/src/main/java/com/magamochi/mangamochi/model/entity/MangaChapter.java b/src/main/java/com/magamochi/mangamochi/model/entity/MangaChapter.java index 2df8f4f..a60b849 100644 --- a/src/main/java/com/magamochi/mangamochi/model/entity/MangaChapter.java +++ b/src/main/java/com/magamochi/mangamochi/model/entity/MangaChapter.java @@ -37,4 +37,8 @@ public class MangaChapter { @OneToMany(mappedBy = "mangaChapter") private List mangaChapterImages; + + private String language; + + private Integer chapterNumber; } diff --git a/src/main/java/com/magamochi/mangamochi/service/ImageService.java b/src/main/java/com/magamochi/mangamochi/service/ImageService.java index b9ed188..8162b06 100644 --- a/src/main/java/com/magamochi/mangamochi/service/ImageService.java +++ b/src/main/java/com/magamochi/mangamochi/service/ImageService.java @@ -4,8 +4,10 @@ import com.magamochi.mangamochi.model.entity.Image; import com.magamochi.mangamochi.model.repository.ImageRepository; import java.io.InputStream; import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; +@Log4j2 @Service @RequiredArgsConstructor public class ImageService { @@ -13,6 +15,7 @@ public class ImageService { private final ImageRepository imageRepository; public Image uploadImage(byte[] data, String contentType, String path) { + log.info("Uploading image {} to S3", path); var fileKey = s3Service.uploadFile(data, contentType, path); return imageRepository.save(Image.builder().fileKey(fileKey).build()); diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java new file mode 100644 index 0000000..6b4b011 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java @@ -0,0 +1,82 @@ +package com.magamochi.mangamochi.service; + +import com.magamochi.mangamochi.client.JikanClient; +import com.magamochi.mangamochi.client.RapidFuzzClient; +import com.magamochi.mangamochi.model.entity.Manga; +import com.magamochi.mangamochi.model.entity.MangaImportReview; +import com.magamochi.mangamochi.model.entity.Provider; +import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository; +import com.magamochi.mangamochi.model.repository.MangaRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class MangaCreationService { + private final MangaRepository mangaRepository; + private final MangaImportReviewRepository mangaImportReviewRepository; + + private final JikanClient jikanClient; + private final RapidFuzzClient rapidFuzzClient; + + public Manga getOrCreateManga(String title, String url, Provider provider) { + var existingManga = mangaRepository.findByTitleIgnoreCase(title); + if (existingManga.isPresent()) { + return existingManga.get(); + } + + var jikanResults = jikanClient.mangaSearch(title).data(); + if (jikanResults.isEmpty()) { + createMangaImportReview(title, url, provider); + log.warn("No manga found with title {}", title); + return null; + } + + var request = + new RapidFuzzClient.Request( + title, + jikanResults.stream() + .flatMap( + results -> + results.titles().stream() + .map(JikanClient.SearchResponse.MangaData.TitleData::title)) + .toList()); + + var fuzzResults = rapidFuzzClient.mangaSearch(request); + if (!fuzzResults.match_found()) { + createMangaImportReview(title, url, provider); + log.warn("No match found for manga with title {}", title); + return null; + } + + var resultOptional = + jikanResults.stream() + .filter( + results -> + results.titles().stream() + .map(JikanClient.SearchResponse.MangaData.TitleData::title) + .toList() + .contains(fuzzResults.best_match())) + .findFirst(); + if (resultOptional.isEmpty()) { + createMangaImportReview(title, url, provider); + log.warn("No match found for manga with title {}", title); + return null; + } + + var result = resultOptional.get(); + + existingManga = mangaRepository.findByTitleIgnoreCase(result.title()); + return existingManga.orElseGet( + () -> + mangaRepository.save( + Manga.builder().title(result.title()).malId(result.mal_id()).build())); + } + + private void createMangaImportReview(String title, String url, Provider provider) { + mangaImportReviewRepository.save( + MangaImportReview.builder().title(title).url(url).provider(provider).build()); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java b/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java new file mode 100644 index 0000000..c6f66b0 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java @@ -0,0 +1,276 @@ +package com.magamochi.mangamochi.service; + +import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; + +import com.google.common.util.concurrent.RateLimiter; +import com.magamochi.mangamochi.client.JikanClient; +import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; +import com.magamochi.mangamochi.model.entity.*; +import com.magamochi.mangamochi.model.repository.*; +import com.magamochi.mangamochi.util.DoubleUtil; +import java.io.*; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.IntStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class MangaImportService { + private final ProviderService providerService; + private final MangaCreationService mangaCreationService; + private final ImageService imageService; + + private final GenreRepository genreRepository; + private final MangaGenreRepository mangaGenreRepository; + private final MangaProviderRepository mangaProviderRepository; + private final AuthorRepository authorRepository; + private final MangaAuthorRepository mangaAuthorRepository; + private final MangaChapterRepository mangaChapterRepository; + private final MangaRepository mangaRepository; + + private final JikanClient jikanClient; + private final MangaChapterImageRepository mangaChapterImageRepository; + + RateLimiter rateLimiter = RateLimiter.create(1); + + public void importMangaFiles(String malId, List files) { + var provider = providerService.getOrCreateProvider("Manual Import"); + + rateLimiter.acquire(); + var mangaData = jikanClient.getMangaById(Long.parseLong(malId)); + + var mangaProvider = getOrCreateMangaProvider(mangaData.data().title(), provider); + + var sortedFiles = files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList(); + + IntStream.rangeClosed(1, sortedFiles.size()) + .forEach( + fileIndex -> { + var file = sortedFiles.get(fileIndex - 1); + log.info( + "Importing file {}/{}: {}, for Mangá {}", + fileIndex, + sortedFiles.size(), + file.getOriginalFilename(), + mangaProvider.getManga().getTitle()); + + var chapter = + persistMangaChapter( + mangaProvider, + new ContentProviderMangaChapterResponseDTO( + removeFileExtension(file.getOriginalFilename()), + "manual_" + file.getOriginalFilename(), + file.getOriginalFilename(), + "pt-br")); + + List allChapterImages = new ArrayList<>(); + try (InputStream is = file.getInputStream(); + ZipInputStream zis = new ZipInputStream(is)) { + ZipEntry entry; + var position = 0; + + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + + var os = new ByteArrayOutputStream(); + zis.transferTo(os); + var bytes = os.toByteArray(); + + var image = + imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapter.getId()); + + var chapterImage = + MangaChapterImage.builder() + .position(position++) + .image(image) + .mangaChapter(chapter) + .build(); + + allChapterImages.add(chapterImage); + zis.closeEntry(); + } + + log.info("Chapter images added for chapter {}", chapter.getTitle()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + mangaChapterImageRepository.saveAll(allChapterImages); + chapter.setDownloaded(true); + mangaChapterRepository.save(chapter); + }); + + log.warn("test"); + } + + public void updateMangaData(Manga manga) { + log.info("Updating manga {}", manga.getTitle()); + + try { + rateLimiter.acquire(); + var mangaData = jikanClient.getMangaById(manga.getMalId()); + + manga.setAlternativeTitles(mangaData.data().title_synonyms()); + manga.setSynopsis(mangaData.data().synopsis()); + manga.setStatus(mangaData.data().status()); + manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2)); + manga.setPublishedFrom(mangaData.data().published().from()); + manga.setPublishedTo(mangaData.data().published().to()); + manga.setChapterCount(mangaData.data().chapters()); + + var authors = + mangaData.data().authors().stream() + .map( + authorData -> { + return authorRepository + .findByMalId(authorData.mal_id()) + .orElseGet( + () -> + authorRepository.save( + Author.builder() + .malId(authorData.mal_id()) + .name(authorData.name()) + .build())); + }) + .toList(); + + var mangaAuthors = + authors.stream() + .map( + author -> { + return mangaAuthorRepository + .findByMangaAndAuthor(manga, author) + .orElseGet( + () -> + mangaAuthorRepository.save( + MangaAuthor.builder().manga(manga).author(author).build())); + }) + .toList(); + + manga.setMangaAuthors(mangaAuthors); + + var genres = + mangaData.data().genres().stream() + .map( + genreData -> { + return genreRepository + .findByMalId(genreData.mal_id()) + .orElseGet( + () -> + genreRepository.save( + Genre.builder() + .malId(genreData.mal_id()) + .name(genreData.name()) + .build())); + }) + .toList(); + + var mangaGenres = + genres.stream() + .map( + genre -> { + return mangaGenreRepository + .findByMangaAndGenre(manga, genre) + .orElseGet( + () -> + mangaGenreRepository.save( + MangaGenre.builder().manga(manga).genre(genre).build())); + }) + .toList(); + + manga.setMangaGenres(mangaGenres); + + if (isNull(manga.getCoverImage())) { + var inputStream = + new BufferedInputStream( + new URL(new URI(mangaData.data().images().jpg().large_image_url()).toASCIIString()) + .openStream()); + + var bytes = inputStream.readAllBytes(); + + inputStream.close(); + var image = imageService.uploadImage(bytes, "image/jpeg", "cover"); + + manga.setCoverImage(image); + } + + mangaRepository.save(manga); + + } catch (Exception e) { + log.warn("Error updating manga data for manga {}. {}", manga.getTitle(), e); + } + } + + public MangaChapter persistMangaChapter( + MangaProvider mangaProvider, ContentProviderMangaChapterResponseDTO chapter) { + var mangaChapter = + mangaChapterRepository + .findByMangaProviderAndUrlIgnoreCase(mangaProvider, chapter.chapterUrl()) + .orElseGet(MangaChapter::new); + + mangaChapter.setMangaProvider(mangaProvider); + mangaChapter.setTitle(chapter.chapterTitle()); + mangaChapter.setUrl(chapter.chapterUrl()); + mangaChapter.setLanguage(chapter.language()); + + if (nonNull(chapter.chapter())) { + try { + mangaChapter.setChapterNumber(Integer.parseInt(chapter.chapter())); + } catch (NumberFormatException e) { + log.warn( + "Could not parse chapter number {} from manga {}", + chapter.chapter(), + mangaProvider.getManga().getTitle()); + } + } + + return mangaChapterRepository.save(mangaChapter); + } + + private MangaProvider getOrCreateMangaProvider(String title, Provider provider) { + return mangaProviderRepository + .findByMangaTitleIgnoreCaseAndProvider(title, provider) + .orElseGet( + () -> { + rateLimiter.acquire(); + var manga = mangaCreationService.getOrCreateManga(title, "manual", provider); + + return mangaProviderRepository.save( + MangaProvider.builder() + .manga(manga) + .mangaTitle(manga.getTitle()) + .provider(provider) + .url("manual") + .build()); + }); + } + + private String removeFileExtension(String filename) { + if (StringUtils.isBlank(filename)) { + return filename; + } + + int lastDotIndex = filename.lastIndexOf('.'); + + // No dot, or dot is the first character (like .gitignore) + if (lastDotIndex <= 0) { + return filename; + } + + return filename.substring(0, lastDotIndex); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaListService.java b/src/main/java/com/magamochi/mangamochi/service/MangaListService.java index 2327a32..d2961c8 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaListService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaListService.java @@ -3,18 +3,9 @@ package com.magamochi.mangamochi.service; import static java.util.Objects.isNull; import com.google.common.util.concurrent.RateLimiter; -import com.magamochi.mangamochi.client.JikanClient; -import com.magamochi.mangamochi.client.RapidFuzzClient; import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; -import com.magamochi.mangamochi.model.entity.Manga; -import com.magamochi.mangamochi.model.entity.MangaImportReview; import com.magamochi.mangamochi.model.entity.MangaProvider; -import com.magamochi.mangamochi.model.entity.Provider; -import com.magamochi.mangamochi.model.enumeration.ProviderStatus; -import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository; import com.magamochi.mangamochi.model.repository.MangaProviderRepository; -import com.magamochi.mangamochi.model.repository.MangaRepository; -import com.magamochi.mangamochi.model.repository.ProviderRepository; import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -24,28 +15,16 @@ import org.springframework.stereotype.Service; @Service @RequiredArgsConstructor public class MangaListService { - private final ProviderRepository providerRepository; - private final MangaRepository mangaRepository; - private final MangaProviderRepository mangaProviderRepository; - private final MangaImportReviewRepository mangaImportReviewRepository; + private final ProviderService providerService; + private final MangaCreationService mangaCreationService; - private final JikanClient jikanClient; - private final RapidFuzzClient rapidFuzzClient; + private final MangaProviderRepository mangaProviderRepository; public void updateMangaList( String contentProviderName, List mangaInfoResponseDTOs) { var rateLimiter = RateLimiter.create(1); - var provider = - providerRepository - .findByNameIgnoreCase(contentProviderName) - .orElseGet( - () -> - providerRepository.save( - Provider.builder() - .name(contentProviderName) - .status(ProviderStatus.ACTIVE) - .build())); + var provider = providerService.getOrCreateProvider(contentProviderName); mangaInfoResponseDTOs.forEach( mangaResponse -> { @@ -58,14 +37,15 @@ public class MangaListService { } rateLimiter.acquire(); - var manga = getOrCreateManga(mangaResponse.title(), mangaResponse.url(), provider); + var manga = + mangaCreationService.getOrCreateManga( + mangaResponse.title(), mangaResponse.url(), provider); if (isNull(manga)) { return; } try { - mangaProviderRepository.save( MangaProvider.builder() .manga(manga) @@ -78,63 +58,4 @@ public class MangaListService { } }); } - - private Manga getOrCreateManga(String title, String url, Provider provider) { - var existingManga = mangaRepository.findByTitleIgnoreCase(title); - if (existingManga.isPresent()) { - return existingManga.get(); - } - - var jikanResults = jikanClient.mangaSearch(title).data(); - if (jikanResults.isEmpty()) { - createMangaImportReview(title, url, provider); - log.warn("No manga found with title {}", title); - return null; - } - - var request = - new RapidFuzzClient.Request( - title, - jikanResults.stream() - .flatMap( - results -> - results.titles().stream() - .map(JikanClient.SearchResponse.MangaData.TitleData::title)) - .toList()); - - var fuzzResults = rapidFuzzClient.mangaSearch(request); - if (!fuzzResults.match_found()) { - createMangaImportReview(title, url, provider); - log.warn("No match found for manga with title {}", title); - return null; - } - - var resultOptional = - jikanResults.stream() - .filter( - results -> - results.titles().stream() - .map(JikanClient.SearchResponse.MangaData.TitleData::title) - .toList() - .contains(fuzzResults.best_match())) - .findFirst(); - if (resultOptional.isEmpty()) { - createMangaImportReview(title, url, provider); - log.warn("No match found for manga with title {}", title); - return null; - } - - var result = resultOptional.get(); - - existingManga = mangaRepository.findByTitleIgnoreCase(result.title()); - return existingManga.orElseGet( - () -> - mangaRepository.save( - Manga.builder().title(result.title()).malId(result.mal_id()).build())); - } - - private void createMangaImportReview(String title, String url, Provider provider) { - mangaImportReviewRepository.save( - MangaImportReview.builder().title(title).url(url).provider(provider).build()); - } } diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaService.java b/src/main/java/com/magamochi/mangamochi/service/MangaService.java index 75cfef0..4871a89 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaService.java @@ -7,7 +7,6 @@ import com.magamochi.mangamochi.model.dto.*; import com.magamochi.mangamochi.model.entity.Manga; import com.magamochi.mangamochi.model.entity.MangaChapter; import com.magamochi.mangamochi.model.entity.MangaChapterImage; -import com.magamochi.mangamochi.model.entity.MangaProvider; import com.magamochi.mangamochi.model.enumeration.ArchiveFileType; import com.magamochi.mangamochi.model.repository.*; import com.magamochi.mangamochi.model.specification.MangaSpecification; @@ -25,14 +24,17 @@ import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.apache.tomcat.util.http.fileupload.IOUtils; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; +@Log4j2 @Service @RequiredArgsConstructor public class MangaService { + private final MangaImportService mangaImportService; private final UserService userService; private final MangaChapterRepository mangaChapterRepository; private final MangaRepository mangaRepository; @@ -72,10 +74,13 @@ public class MangaService { .findById(mangaProviderId) .orElseThrow(() -> new RuntimeException("manga provider not found")); - return mangaProvider.getMangaChapters().stream() - .sorted(Comparator.comparing(MangaChapter::getId)) - .map(MangaChapterDTO::from) - .toList(); + var chapters = + mangaProvider.getMangaChapters().stream() + .sorted(Comparator.comparing(MangaChapter::getId)) + .map(MangaChapterDTO::from) + .toList(); + + return chapters; } public void fetchChapter(Long chapterId) { @@ -109,6 +114,14 @@ public class MangaService { var image = imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapterId); + log.info( + "Downloaded image {}/{} for manga {} chapter {}: {}", + entry.getKey() + 1, + chapterImagesUrls.size(), + chapter.getMangaProvider().getManga().getTitle(), + chapterId, + entry.getValue()); + return MangaChapterImage.builder() .mangaChapter(chapter) .position(entry.getKey()) @@ -173,20 +186,6 @@ public class MangaService { return byteArrayOutputStream; } - private void persistMangaChapters( - MangaProvider mangaProvider, ContentProviderMangaChapterResponseDTO chapter) { - var mangaChapter = - mangaChapterRepository - .findByMangaProviderAndUrlIgnoreCase(mangaProvider, chapter.chapterUrl()) - .orElseGet(MangaChapter::new); - - mangaChapter.setMangaProvider(mangaProvider); - mangaChapter.setTitle(chapter.chapterTitle()); - mangaChapter.setUrl(chapter.chapterUrl()); - - mangaChapterRepository.save(mangaChapter); - } - public void downloadAllChapters(Long mangaProviderId) { var mangaProvider = mangaProviderRepository @@ -230,7 +229,8 @@ public class MangaService { contentProviderFactory.getContentProvider(mangaProvider.getProvider().getName()); var availableChapters = contentProvider.getAvailableChapters(mangaProvider); - availableChapters.forEach(chapter -> persistMangaChapters(mangaProvider, chapter)); + availableChapters.forEach( + chapter -> mangaImportService.persistMangaChapter(mangaProvider, chapter)); } public MangaChapterImagesDTO getMangaChapterImages(Long chapterId) { diff --git a/src/main/java/com/magamochi/mangamochi/service/ProviderService.java b/src/main/java/com/magamochi/mangamochi/service/ProviderService.java new file mode 100644 index 0000000..2bc8ebe --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/ProviderService.java @@ -0,0 +1,22 @@ +package com.magamochi.mangamochi.service; + +import com.magamochi.mangamochi.model.entity.Provider; +import com.magamochi.mangamochi.model.enumeration.ProviderStatus; +import com.magamochi.mangamochi.model.repository.ProviderRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class ProviderService { + private final ProviderRepository providerRepository; + + public Provider getOrCreateProvider(String providerName) { + return providerRepository + .findByNameIgnoreCase(providerName) + .orElseGet( + () -> + providerRepository.save( + Provider.builder().name(providerName).status(ProviderStatus.ACTIVE).build())); + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java b/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java index af315c5..e24a5dc 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java @@ -3,4 +3,5 @@ package com.magamochi.mangamochi.service.providers; public class ContentProviders { public static final String MANGA_LIVRE = "Manga Livre"; public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog"; + public static final String MANGA_DEX = "MangaDex"; } diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java new file mode 100644 index 0000000..6c1db13 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java @@ -0,0 +1,193 @@ +package com.magamochi.mangamochi.service.providers.impl; + +import static java.util.Objects.isNull; + +import com.google.common.util.concurrent.RateLimiter; +import com.magamochi.mangamochi.client.MangaDexAuthenticationClient; +import com.magamochi.mangamochi.client.MangaDexClient; +import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; +import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO; +import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO; +import com.magamochi.mangamochi.model.entity.MangaProvider; +import com.magamochi.mangamochi.model.entity.Provider; +import com.magamochi.mangamochi.model.enumeration.ProviderStatus; +import com.magamochi.mangamochi.model.repository.MangaProviderRepository; +import com.magamochi.mangamochi.model.repository.ProviderRepository; +import com.magamochi.mangamochi.service.MangaCreationService; +import com.magamochi.mangamochi.service.providers.ContentProvider; +import com.magamochi.mangamochi.service.providers.ContentProviders; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service(ContentProviders.MANGA_DEX) +@RequiredArgsConstructor +public class MangaDexProvider implements ContentProvider { + @Value("${manga-dex.username}") + private String mangaDexUsername; + + @Value("${manga-dex.password}") + private String mangaDexPassword; + + @Value("${manga-dex.client-id}") + private String mangaDexClientId; + + @Value("${manga-dex.client-secret}") + private String mangaDexClientSecret; + + private final MangaDexClient mangaDexClient; + private final MangaDexAuthenticationClient mangaDexAuthenticationClient; + private final MangaCreationService mangaCreationService; + private final ProviderRepository providerRepository; + private final MangaProviderRepository mangaProviderRepository; + + RateLimiter rateLimiter = RateLimiter.create(1); + + private String authorizationToken; + + @Override + public List getAvailableMangas() { + // MangaDex API does not provide an endpoint to list all mangas directly. + // As there is lots and lots of mangas, this is not feasible to implement here. + // The frontend has a function to import mangas by their IDs instead. + return List.of(); + } + + @Override + public List getAvailableChapters(MangaProvider provider) { + rateLimiter.acquire(); + var response = + mangaDexClient.getMangaFeed(UUID.fromString(provider.getUrl()), getAuthorizationToken()); + + var mangas = new ArrayList<>(response.data()); + var totalPages = (int) Math.ceil((double) response.total() / 100); // Default page size is 100 + + IntStream.range(1, totalPages) + .forEach( + i -> { + rateLimiter.acquire(); + + var pagedResponse = + mangaDexClient.getMangaFeed( + UUID.fromString(provider.getUrl()), 100, i * 100, getAuthorizationToken()); + + mangas.addAll(pagedResponse.data()); + }); + + // TODO this is filtering only pt-br chapters for now, we may want to make this configurable + // later + return mangas.stream() + .filter( + c -> + c.type().equals("chapter") + && c.attributes().isUnavailable().equals(Boolean.FALSE) + && c.attributes().translatedLanguage().equals("pt-br")) + .sorted( + (o1, o2) -> { + try { + Float chapter1 = Float.parseFloat(o1.attributes().chapter()); + Float chapter2 = Float.parseFloat(o2.attributes().chapter()); + return chapter2.compareTo(chapter1); + } catch (NumberFormatException e) { + return o2.attributes().chapter().compareTo(o1.attributes().chapter()); + } + }) + .map( + c -> + new ContentProviderMangaChapterResponseDTO( + c.attributes().chapter() + " - " + c.attributes().title(), + c.id().toString(), + c.attributes().chapter(), + c.attributes().translatedLanguage())) + .toList(); + } + + @Override + public Map getChapterImagesUrls(String chapterUrl) { + rateLimiter.acquire(); + var chapter = + mangaDexClient.getMangaChapter(UUID.fromString(chapterUrl), getAuthorizationToken()); + + var chapterImageHashes = + chapter.chapter().data().stream() + .map(s -> chapter.baseUrl() + "/data/" + chapter.chapter().hash() + "/" + s) + .toList(); + + var map = + IntStream.range(0, chapterImageHashes.size()) + .boxed() + .collect( + Collectors.toMap( + i -> i, + chapterImageHashes::get, + (existing, replacement) -> existing, + LinkedHashMap::new)); + + return map; + } + + public ImportMangaDexResponseDTO importManga(UUID id) { + var token = getAuthorizationToken(); + + rateLimiter.acquire(); + var resultData = mangaDexClient.getManga(id, token).data(); + + if (resultData.attributes().title().isEmpty()) { + throw new NoSuchElementException("Manga title not found for ID: " + id); + } + + var mangaTitle = + resultData + .attributes() + .title() + .getOrDefault( + "en", + resultData.attributes().title().values().stream() + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No title available"))); + + var provider = + providerRepository + .findByNameIgnoreCase("MangaDex") + .orElseGet( + () -> + providerRepository.save( + Provider.builder().name("MangaDex").status(ProviderStatus.ACTIVE).build())); + + var manga = mangaCreationService.getOrCreateManga(mangaTitle, id.toString(), provider); + + if (isNull(manga)) { + throw new NoSuchElementException("Manga not found for ID: " + id); + } + + mangaProviderRepository.save( + MangaProvider.builder() + .manga(manga) + .mangaTitle(mangaTitle) + .provider(provider) + .url(id.toString()) + .build()); + + return new ImportMangaDexResponseDTO(manga.getId()); + } + + private String getAuthorizationToken() { + if (isNull(authorizationToken)) { + rateLimiter.acquire(); + + var authResponse = + mangaDexAuthenticationClient.authenticate( + MangaDexAuthenticationClient.build( + mangaDexUsername, mangaDexPassword, mangaDexClientId, mangaDexClientSecret)); + + authorizationToken = "Bearer " + authResponse.accessToken(); + } + + return authorizationToken; + } +} diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java index 63f184f..49eeeec 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java @@ -62,7 +62,7 @@ public class MangaLivreBlogProvider implements ContentProvider { linkElement.getElementsByClass("chapter-number").getFirst(); return new ContentProviderMangaChapterResponseDTO( - chapterNumberElement.text(), linkElement.attr("href")); + chapterNumberElement.text(), linkElement.attr("href"), null, null); }) .toList(); } catch (IOException | NoSuchElementException e) { diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java index 8027434..8b144c2 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java @@ -51,7 +51,7 @@ public class MangaLivreProvider implements ContentProvider { var linkElement = chapterItemElement.getElementsByTag("a").getFirst(); return new ContentProviderMangaChapterResponseDTO( - linkElement.text(), linkElement.attr("href")); + linkElement.text(), linkElement.attr("href"), null, null); }) .toList(); } catch (NoSuchElementException | IOException e) { diff --git a/src/main/java/com/magamochi/mangamochi/task/UpdateMangaDataTask.java b/src/main/java/com/magamochi/mangamochi/task/UpdateMangaDataTask.java index c2adc1a..abed99b 100644 --- a/src/main/java/com/magamochi/mangamochi/task/UpdateMangaDataTask.java +++ b/src/main/java/com/magamochi/mangamochi/task/UpdateMangaDataTask.java @@ -3,144 +3,27 @@ package com.magamochi.mangamochi.task; import static java.util.Objects.isNull; import com.google.common.util.concurrent.RateLimiter; -import com.magamochi.mangamochi.client.JikanClient; -import com.magamochi.mangamochi.model.entity.Author; -import com.magamochi.mangamochi.model.entity.Genre; -import com.magamochi.mangamochi.model.entity.MangaAuthor; -import com.magamochi.mangamochi.model.entity.MangaGenre; import com.magamochi.mangamochi.model.repository.*; -import com.magamochi.mangamochi.service.ImageService; -import com.magamochi.mangamochi.util.DoubleUtil; -import java.io.BufferedInputStream; -import java.net.URI; -import java.net.URL; +import com.magamochi.mangamochi.service.MangaImportService; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @Log4j2 @Component @RequiredArgsConstructor public class UpdateMangaDataTask { - private final AuthorRepository authorRepository; - private final MangaAuthorRepository mangaAuthorRepository; private final MangaRepository mangaRepository; + private final MangaImportService mangaImportService; - private final JikanClient jikanClient; - - private final ImageService imageService; - private final GenreRepository genreRepository; - private final MangaGenreRepository mangaGenreRepository; - - // @Scheduled(fixedDelayString = "1d") + @Scheduled(fixedDelayString = "1d") public void updateMangaData() { var rateLimiter = RateLimiter.create(1); var mangas = mangaRepository.findAll().stream().filter(manga -> isNull(manga.getScore())).toList(); - mangas.forEach( - manga -> { - log.info("Updating manga {}", manga.getTitle()); - - try { - rateLimiter.acquire(); - var mangaData = jikanClient.getMangaById(manga.getMalId()); - - manga.setAlternativeTitles(mangaData.data().title_synonyms()); - manga.setSynopsis(mangaData.data().synopsis()); - manga.setStatus(mangaData.data().status()); - manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2)); - manga.setPublishedFrom(mangaData.data().published().from()); - manga.setPublishedTo(mangaData.data().published().to()); - manga.setChapterCount(mangaData.data().chapters()); - - var authors = - mangaData.data().authors().stream() - .map( - authorData -> { - return authorRepository - .findByMalId(authorData.mal_id()) - .orElseGet( - () -> - authorRepository.save( - Author.builder() - .malId(authorData.mal_id()) - .name(authorData.name()) - .build())); - }) - .toList(); - - var mangaAuthors = - authors.stream() - .map( - author -> { - return mangaAuthorRepository - .findByMangaAndAuthor(manga, author) - .orElseGet( - () -> - mangaAuthorRepository.save( - MangaAuthor.builder() - .manga(manga) - .author(author) - .build())); - }) - .toList(); - - manga.setMangaAuthors(mangaAuthors); - - var genres = - mangaData.data().genres().stream() - .map( - genreData -> { - return genreRepository - .findByMalId(genreData.mal_id()) - .orElseGet( - () -> - genreRepository.save( - Genre.builder() - .malId(genreData.mal_id()) - .name(genreData.name()) - .build())); - }) - .toList(); - - var mangaGenres = - genres.stream() - .map( - genre -> { - return mangaGenreRepository - .findByMangaAndGenre(manga, genre) - .orElseGet( - () -> - mangaGenreRepository.save( - MangaGenre.builder().manga(manga).genre(genre).build())); - }) - .toList(); - - manga.setMangaGenres(mangaGenres); - - if (isNull(manga.getCoverImage())) { - var inputStream = - new BufferedInputStream( - new URL( - new URI(mangaData.data().images().jpg().large_image_url()) - .toASCIIString()) - .openStream()); - - var bytes = inputStream.readAllBytes(); - - inputStream.close(); - var image = imageService.uploadImage(bytes, "image/jpeg", "cover"); - - manga.setCoverImage(image); - } - - mangaRepository.save(manga); - - } catch (Exception e) { - log.warn("Error updating manga data for manga {}. {}", manga.getTitle(), e); - } - }); + mangas.forEach(mangaImportService::updateMangaData); } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e7e8224..2bbe97c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -14,6 +14,10 @@ spring: schemas: - mangamochi default-schema: mangamochi + servlet: + multipart: + max-file-size: 250MB + max-request-size: 2GB springdoc: api-docs: @@ -32,3 +36,9 @@ minio: jwt: secret: mySecretKeymySecretKeymySecretKeymySecretKeymySecretKeymySecretKey expiration: 86400000 # 24 hours in milliseconds + +manga-dex: + username: ${MANGADEX_USER} + password: ${MANGADEX_PASS} + client-id: ${MANGADEX_CLIENT_ID} + client-secret: ${MANGADEX_CLIENT_SECRET} diff --git a/src/main/resources/db/migration/V0013__MANGA_CHAPTER_LANGUAGE.sql b/src/main/resources/db/migration/V0013__MANGA_CHAPTER_LANGUAGE.sql new file mode 100644 index 0000000..f7ac5df --- /dev/null +++ b/src/main/resources/db/migration/V0013__MANGA_CHAPTER_LANGUAGE.sql @@ -0,0 +1,3 @@ +ALTER TABLE manga_chapters + ADD COLUMN language VARCHAR(10) DEFAULT 'pt-br', + ADD COLUMN chapter_number INTEGER; \ No newline at end of file