diff --git a/src/main/java/com/magamochi/mangamochi/client/AniListClient.java b/src/main/java/com/magamochi/mangamochi/client/AniListClient.java new file mode 100644 index 0000000..b5c5b86 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/client/AniListClient.java @@ -0,0 +1,51 @@ +package com.magamochi.mangamochi.client; + +import io.github.resilience4j.retry.annotation.Retry; +import java.util.List; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +@FeignClient(name = "aniList", url = "https://graphql.anilist.co") +@Retry(name = "AniListRetry") +public interface AniListClient { + + @PostMapping + MangaResponse getManga(@RequestBody GraphQLRequest request); + + record GraphQLRequest(String query, Variables variables) { + public record Variables(Long id) {} + } + + record MangaResponse(Data data) { + public record Data(Manga Media) {} + + public record Manga( + Long id, + Long idMal, + Title title, + String status, + String description, // synopsis + int chapters, + int averageScore, // score (0-100) + CoverImage coverImage, + List genres, + FuzzyDate startDate, + FuzzyDate endDate, + StaffConnection staff) { + public record Title(String romaji, String english, String nativeTitle) {} + + public record CoverImage(String large) {} + + public record FuzzyDate(Integer year, Integer month, Integer day) {} + + public record StaffConnection(List edges) { + public record StaffEdge(String role, Staff node) { + public record Staff(Name name) { + public record Name(String full) {} + } + } + } + } + } +} diff --git a/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java b/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java index a1bc614..e4da1e5 100644 --- a/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java +++ b/src/main/java/com/magamochi/mangamochi/controller/ManagementController.java @@ -2,7 +2,9 @@ package com.magamochi.mangamochi.controller; import com.magamochi.mangamochi.client.NtfyClient; import com.magamochi.mangamochi.model.dto.DefaultResponseDTO; +import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand; import com.magamochi.mangamochi.model.repository.UserRepository; +import com.magamochi.mangamochi.queue.UpdateMangaDataProducer; import com.magamochi.mangamochi.task.ImageCleanupTask; import com.magamochi.mangamochi.task.MangaFollowUpdateTask; import com.magamochi.mangamochi.task.UpdateMangaListTask; @@ -18,8 +20,20 @@ public class ManagementController { private final ImageCleanupTask imageCleanupTask; private final MangaFollowUpdateTask mangaFollowUpdateTask; private final UserRepository userRepository; - private final NtfyClient ntfyClient; + private final UpdateMangaDataProducer updateMangaDataProducer; + + @Operation( + summary = "Trigger manga data update", + description = "Triggers the update of the metadata for a manga by its ID", + tags = {"Management"}, + operationId = "triggerUpdateMangaData") + @PostMapping("update-manga-data/{mangaId}") + public DefaultResponseDTO triggerUpdateMangaData(@PathVariable Long mangaId) { + updateMangaDataProducer.sendUpdateMangaDataCommand(new UpdateMangaDataCommand(mangaId)); + + return DefaultResponseDTO.ok().build(); + } @Operation( summary = "Queue update manga list", diff --git a/src/main/java/com/magamochi/mangamochi/model/dto/ImportRequestDTO.java b/src/main/java/com/magamochi/mangamochi/model/dto/ImportRequestDTO.java index 7354589..2b0d9f9 100644 --- a/src/main/java/com/magamochi/mangamochi/model/dto/ImportRequestDTO.java +++ b/src/main/java/com/magamochi/mangamochi/model/dto/ImportRequestDTO.java @@ -2,4 +2,4 @@ package com.magamochi.mangamochi.model.dto; import jakarta.validation.constraints.NotNull; -public record ImportRequestDTO(String metadataId, @NotNull String id) {} +public record ImportRequestDTO(String metadataId, String aniListId, @NotNull String id) {} diff --git a/src/main/java/com/magamochi/mangamochi/model/entity/Manga.java b/src/main/java/com/magamochi/mangamochi/model/entity/Manga.java index c126fd7..6c75e62 100644 --- a/src/main/java/com/magamochi/mangamochi/model/entity/Manga.java +++ b/src/main/java/com/magamochi/mangamochi/model/entity/Manga.java @@ -22,6 +22,8 @@ public class Manga { private Long malId; + private Long aniListId; + private String title; private String status; diff --git a/src/main/java/com/magamochi/mangamochi/model/repository/AuthorRepository.java b/src/main/java/com/magamochi/mangamochi/model/repository/AuthorRepository.java index 4fb3499..4ab61ff 100644 --- a/src/main/java/com/magamochi/mangamochi/model/repository/AuthorRepository.java +++ b/src/main/java/com/magamochi/mangamochi/model/repository/AuthorRepository.java @@ -6,4 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository; public interface AuthorRepository extends JpaRepository { Optional findByMalId(Long aLong); + + Optional findByName(String name); } diff --git a/src/main/java/com/magamochi/mangamochi/model/repository/GenreRepository.java b/src/main/java/com/magamochi/mangamochi/model/repository/GenreRepository.java index 57732c7..438148e 100644 --- a/src/main/java/com/magamochi/mangamochi/model/repository/GenreRepository.java +++ b/src/main/java/com/magamochi/mangamochi/model/repository/GenreRepository.java @@ -6,4 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository; public interface GenreRepository extends JpaRepository { Optional findByMalId(Long malId); + + Optional findByName(String name); } diff --git a/src/main/java/com/magamochi/mangamochi/model/repository/MangaRepository.java b/src/main/java/com/magamochi/mangamochi/model/repository/MangaRepository.java index a65077f..8e981e9 100644 --- a/src/main/java/com/magamochi/mangamochi/model/repository/MangaRepository.java +++ b/src/main/java/com/magamochi/mangamochi/model/repository/MangaRepository.java @@ -13,4 +13,6 @@ public interface MangaRepository List findByFollowTrue(); Optional findByMalId(Long malId); + + Optional findByAniListId(Long aniListId); } diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java index 4a884fd..6e22b1e 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java @@ -1,6 +1,9 @@ package com.magamochi.mangamochi.service; +import static java.util.Objects.nonNull; + import com.google.common.util.concurrent.RateLimiter; +import com.magamochi.mangamochi.client.AniListClient; import com.magamochi.mangamochi.client.JikanClient; import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO; import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand; @@ -24,6 +27,7 @@ public class MangaCreationService { private final TitleMatcherService titleMatcherService; private final JikanClient jikanClient; + private final AniListClient aniListClient; private final RateLimiter jikanRateLimiter; @@ -78,28 +82,74 @@ public class MangaCreationService { } var result = resultOptional.get(); - return getOrCreateManga(result.mal_id(), result.title()); + return getOrCreateManga(result.mal_id(), null, result.title()); } - public Manga getOrCreateManga(Long malId) { - jikanRateLimiter.acquire(); - var data = jikanClient.getMangaById(malId); + public Manga getOrCreateManga(Long malId, Long aniListId) { + if (nonNull(malId)) { + try { + jikanRateLimiter.acquire(); + var data = jikanClient.getMangaById(malId); + return getOrCreateManga(data.data().mal_id(), aniListId, data.data().title()); + } catch (feign.FeignException.NotFound e) { + log.warn("Manga not found on MyAnimeList for ID: {}", malId); + } + } - return getOrCreateManga(data.data().mal_id(), data.data().title()); + if (nonNull(aniListId)) { + try { + var query = + """ + query ($id: Int) { + Media (id: $id, type: MANGA) { + id + idMal + title { + romaji + english + native + } + } + } + """; + var request = + new AniListClient.GraphQLRequest( + query, new AniListClient.GraphQLRequest.Variables(aniListId)); + var data = aniListClient.getManga(request).data().Media(); + + String title = + nonNull(data.title().english()) ? data.title().english() : data.title().romaji(); + return getOrCreateManga(data.idMal(), data.id(), title); + } catch (feign.FeignException.NotFound e) { + log.warn("Manga not found on AniList for ID: {}", aniListId); + } + } + + throw new RuntimeException("Could not find manga on any provider"); } - private Manga getOrCreateManga(Long malId, String title) { - return mangaRepository - .findByMalId(malId) - .orElseGet( - () -> { - var manga = mangaRepository.save(Manga.builder().title(title).malId(malId).build()); + private Manga getOrCreateManga(Long malId, Long aniListId, String title) { + var mangaOptional = java.util.Optional.empty(); - updateMangaDataProducer.sendUpdateMangaDataCommand( - new UpdateMangaDataCommand(manga.getId())); + if (nonNull(malId)) { + mangaOptional = mangaRepository.findByMalId(malId); + } - return manga; - }); + if (mangaOptional.isEmpty() && nonNull(aniListId)) { + mangaOptional = mangaRepository.findByAniListId(aniListId); + } + + return mangaOptional.orElseGet( + () -> { + var manga = + mangaRepository.save( + Manga.builder().title(title).malId(malId).aniListId(aniListId).build()); + + updateMangaDataProducer.sendUpdateMangaDataCommand( + new UpdateMangaDataCommand(manga.getId())); + + return manga; + }); } private void createMangaImportReview(String title, String url, Provider provider) { diff --git a/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java b/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java index 92911c3..1948bfa 100644 --- a/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java +++ b/src/main/java/com/magamochi/mangamochi/service/MangaImportService.java @@ -4,6 +4,7 @@ import static java.util.Objects.isNull; import static java.util.Objects.nonNull; import com.google.common.util.concurrent.RateLimiter; +import com.magamochi.mangamochi.client.AniListClient; import com.magamochi.mangamochi.client.JikanClient; import com.magamochi.mangamochi.exception.NotFoundException; import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO; @@ -12,7 +13,10 @@ import com.magamochi.mangamochi.model.repository.*; import com.magamochi.mangamochi.util.DoubleUtil; import java.io.*; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -43,6 +47,7 @@ public class MangaImportService { private final MangaRepository mangaRepository; private final JikanClient jikanClient; + private final AniListClient aniListClient; private final MangaChapterImageRepository mangaChapterImageRepository; private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository; @@ -133,99 +138,233 @@ public class MangaImportService { public void updateMangaData(Manga manga) { log.info("Updating manga {}", manga.getTitle()); - try { - jikanRateLimiter.acquire(); - var mangaData = jikanClient.getMangaById(manga.getMalId()); - - manga.setSynopsis(mangaData.data().synopsis()); - manga.setStatus(mangaData.data().status()); - manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2)); - manga.setPublishedFrom(mangaData.data().published().from()); - manga.setPublishedTo(mangaData.data().published().to()); - manga.setChapterCount(mangaData.data().chapters()); - - var authors = - mangaData.data().authors().stream() - .map( - authorData -> - authorRepository - .findByMalId(authorData.mal_id()) - .orElseGet( - () -> - authorRepository.save( - Author.builder() - .malId(authorData.mal_id()) - .name(authorData.name()) - .build()))) - .toList(); - - var mangaAuthors = - authors.stream() - .map( - author -> - mangaAuthorRepository - .findByMangaAndAuthor(manga, author) - .orElseGet( - () -> - mangaAuthorRepository.save( - MangaAuthor.builder().manga(manga).author(author).build()))) - .toList(); - - manga.setMangaAuthors(mangaAuthors); - - var genres = - mangaData.data().genres().stream() - .map( - genreData -> - genreRepository - .findByMalId(genreData.mal_id()) - .orElseGet( - () -> - genreRepository.save( - Genre.builder() - .malId(genreData.mal_id()) - .name(genreData.name()) - .build()))) - .toList(); - - var mangaGenres = - genres.stream() - .map( - genre -> - mangaGenreRepository - .findByMangaAndGenre(manga, genre) - .orElseGet( - () -> - mangaGenreRepository.save( - MangaGenre.builder().manga(manga).genre(genre).build()))) - .toList(); - - manga.setMangaGenres(mangaGenres); - - if (isNull(manga.getCoverImage())) { - var inputStream = - new BufferedInputStream( - new URL(new URI(mangaData.data().images().jpg().large_image_url()).toASCIIString()) - .openStream()); - - var bytes = inputStream.readAllBytes(); - - inputStream.close(); - var image = imageService.uploadImage(bytes, "image/jpeg", "cover"); - - manga.setCoverImage(image); + if (nonNull(manga.getMalId())) { + try { + updateFromJikan(manga); + return; + } catch (Exception e) { + log.warn( + "Error updating manga data from Jikan for manga {}. Trying AniList... Error: {}", + manga.getTitle(), + e.getMessage()); } - - var mangaEntity = mangaRepository.save(manga); - var alternativeTitles = - mangaData.data().title_synonyms().stream() - .map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build()) - .toList(); - mangaAlternativeTitlesRepository.saveAll(alternativeTitles); - - } catch (Exception e) { - log.warn("Error updating manga data for manga {}. {}", manga.getTitle(), e); } + + if (nonNull(manga.getAniListId())) { + try { + updateFromAniList(manga); + return; + } catch (Exception e) { + log.warn( + "Error updating manga data from AniList for manga {}. Error: {}", + manga.getTitle(), + e.getMessage()); + } + } + + log.warn( + "Could not update manga data for {}. No provider data available/found.", manga.getTitle()); + } + + private void updateFromJikan(Manga manga) throws IOException, URISyntaxException { + jikanRateLimiter.acquire(); + var mangaData = jikanClient.getMangaById(manga.getMalId()); + + manga.setSynopsis(mangaData.data().synopsis()); + manga.setStatus(mangaData.data().status()); + manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2)); + manga.setPublishedFrom(mangaData.data().published().from()); + manga.setPublishedTo(mangaData.data().published().to()); + manga.setChapterCount(mangaData.data().chapters()); + + var authors = + mangaData.data().authors().stream() + .map( + authorData -> + authorRepository + .findByMalId(authorData.mal_id()) + .orElseGet( + () -> + authorRepository.save( + Author.builder() + .malId(authorData.mal_id()) + .name(authorData.name()) + .build()))) + .toList(); + + updateMangaAuthors(manga, authors); + + var genres = + mangaData.data().genres().stream() + .map( + genreData -> + genreRepository + .findByMalId(genreData.mal_id()) + .orElseGet( + () -> + genreRepository.save( + Genre.builder() + .malId(genreData.mal_id()) + .name(genreData.name()) + .build()))) + .toList(); + + updateMangaGenres(manga, genres); + + if (isNull(manga.getCoverImage())) { + downloadCoverImage(manga, mangaData.data().images().jpg().large_image_url()); + } + + var mangaEntity = mangaRepository.save(manga); + var alternativeTitles = + mangaData.data().title_synonyms().stream() + .map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build()) + .toList(); + mangaAlternativeTitlesRepository.saveAll(alternativeTitles); + } + + private void updateFromAniList(Manga manga) throws IOException, URISyntaxException { + var query = + """ + query ($id: Int) { + Media (id: $id, type: MANGA) { + startDate { year month day } + endDate { year month day } + description + status + averageScore + chapters + coverImage { large } + genres + staff { + edges { + role + node { + name { + full + } + } + } + } + } + } + """; + var request = + new AniListClient.GraphQLRequest( + query, new AniListClient.GraphQLRequest.Variables(manga.getAniListId())); + var media = aniListClient.getManga(request).data().Media(); + + manga.setSynopsis(media.description()); + manga.setStatus(mapAniListStatus(media.status())); + manga.setScore(DoubleUtil.round((double) media.averageScore() / 10, 2)); // 0-100 -> 0-10 + manga.setPublishedFrom(convertFuzzyDate(media.startDate())); + manga.setPublishedTo(convertFuzzyDate(media.endDate())); + manga.setChapterCount(media.chapters()); + + var authors = + media.staff().edges().stream() + .filter(edge -> isAuthorRole(edge.role())) + .map(edge -> edge.node().name().full()) + .distinct() + .map( + name -> + authorRepository + .findByName(name) + .orElseGet( + () -> authorRepository.save(Author.builder().name(name).build()))) + .toList(); + + updateMangaAuthors(manga, authors); + + var genres = + media.genres().stream() + .map( + name -> + genreRepository + .findByName(name) + .orElseGet(() -> genreRepository.save(Genre.builder().name(name).build()))) + .toList(); + + updateMangaGenres(manga, genres); + + if (isNull(manga.getCoverImage())) { + downloadCoverImage(manga, media.coverImage().large()); + } + + mangaRepository.save(manga); + } + + private boolean isAuthorRole(String role) { + return role.equalsIgnoreCase("Story & Art") + || role.equalsIgnoreCase("Story") + || role.equalsIgnoreCase("Art"); + } + + private String mapAniListStatus(String status) { + return switch (status) { + case "RELEASING" -> "Publishing"; + case "FINISHED" -> "Finished"; + case "NOT_YET_RELEASED" -> "Not yet published"; + default -> "Unknown"; + }; + } + + private OffsetDateTime convertFuzzyDate(AniListClient.MangaResponse.Manga.FuzzyDate date) { + if (isNull(date) || isNull(date.year())) { + return null; + } + return OffsetDateTime.of( + date.year(), + isNull(date.month()) ? 1 : date.month(), + isNull(date.day()) ? 1 : date.day(), + 0, + 0, + 0, + 0, + ZoneOffset.UTC); + } + + private void updateMangaAuthors(Manga manga, List authors) { + var mangaAuthors = + authors.stream() + .map( + author -> + mangaAuthorRepository + .findByMangaAndAuthor(manga, author) + .orElseGet( + () -> + mangaAuthorRepository.save( + MangaAuthor.builder().manga(manga).author(author).build()))) + .toList(); + manga.setMangaAuthors(mangaAuthors); + } + + private void updateMangaGenres(Manga manga, List genres) { + var mangaGenres = + genres.stream() + .map( + genre -> + mangaGenreRepository + .findByMangaAndGenre(manga, genre) + .orElseGet( + () -> + mangaGenreRepository.save( + MangaGenre.builder().manga(manga).genre(genre).build()))) + .toList(); + manga.setMangaGenres(mangaGenres); + } + + private void downloadCoverImage(Manga manga, String imageUrl) + throws IOException, URISyntaxException { + var inputStream = + new BufferedInputStream(new URL(new URI(imageUrl).toASCIIString()).openStream()); + + var bytes = inputStream.readAllBytes(); + + inputStream.close(); + var image = imageService.uploadImage(bytes, "image/jpeg", "cover"); + + manga.setCoverImage(image); } public MangaChapter persistMangaChapter( diff --git a/src/main/java/com/magamochi/mangamochi/service/ProviderManualMangaImportService.java b/src/main/java/com/magamochi/mangamochi/service/ProviderManualMangaImportService.java index 0549944..db9413f 100644 --- a/src/main/java/com/magamochi/mangamochi/service/ProviderManualMangaImportService.java +++ b/src/main/java/com/magamochi/mangamochi/service/ProviderManualMangaImportService.java @@ -31,9 +31,12 @@ public class ProviderManualMangaImportService { var title = contentProvider.getMangaTitle(requestDTO.id()); + var malId = nonNull(requestDTO.metadataId()) ? Long.parseLong(requestDTO.metadataId()) : null; + var aniListId = nonNull(requestDTO.aniListId()) ? Long.parseLong(requestDTO.aniListId()) : null; + var manga = - (nonNull(requestDTO.metadataId()) && !requestDTO.metadataId().isBlank()) - ? mangaCreationService.getOrCreateManga(Long.parseLong(requestDTO.metadataId())) + nonNull(malId) || nonNull(aniListId) + ? mangaCreationService.getOrCreateManga(malId, aniListId) : mangaCreationService.getOrCreateManga(title, requestDTO.id(), provider); if (isNull(manga)) { diff --git a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java index 61deb1c..2980be5 100644 --- a/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java +++ b/src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java @@ -35,7 +35,7 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro List.of("safe", "suggestive", "erotica", "pornographic")); var mangas = new ArrayList<>(response.data()); - var totalPages = (int) Math.ceil((double) response.total() / 500); + var totalPages = (int) Math.ceil((double) response.total() / 100); try { IntStream.range(1, totalPages) @@ -47,8 +47,8 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro var pagedResponse = mangaDexClient.getMangaFeed( UUID.fromString(provider.getUrl()), - 500, - i * 500, + 100, + i * 100, List.of("safe", "suggestive", "erotica", "pornographic")); mangas.addAll(pagedResponse.data()); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 260235a..2ab855a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -74,6 +74,12 @@ resilience4j: seconds: 5 retry-exceptions: - feign.FeignException + AniListRetry: + max-attempts: 5 + wait-duration: + seconds: 5 + retry-exceptions: + - feign.FeignException ImageDownloadRetry: max-attempts: 3 wait-duration: diff --git a/src/main/resources/db/migration/V0020__ADD_ANILIST_ID.sql b/src/main/resources/db/migration/V0020__ADD_ANILIST_ID.sql new file mode 100644 index 0000000..0b9a4f7 --- /dev/null +++ b/src/main/resources/db/migration/V0020__ADD_ANILIST_ID.sql @@ -0,0 +1 @@ +ALTER TABLE mangas ADD COLUMN ani_list_id BIGINT;