Merge pull request 'improvements' (#26) from improvements into main
Reviewed-on: #26
This commit is contained in:
commit
76c71dbd81
@ -0,0 +1,51 @@
|
|||||||
|
package com.magamochi.mangamochi.client;
|
||||||
|
|
||||||
|
import io.github.resilience4j.retry.annotation.Retry;
|
||||||
|
import java.util.List;
|
||||||
|
import org.springframework.cloud.openfeign.FeignClient;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
|
||||||
|
@FeignClient(name = "aniList", url = "https://graphql.anilist.co")
|
||||||
|
@Retry(name = "AniListRetry")
|
||||||
|
public interface AniListClient {
|
||||||
|
|
||||||
|
@PostMapping
|
||||||
|
MangaResponse getManga(@RequestBody GraphQLRequest request);
|
||||||
|
|
||||||
|
record GraphQLRequest(String query, Variables variables) {
|
||||||
|
public record Variables(Long id) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
record MangaResponse(Data data) {
|
||||||
|
public record Data(Manga Media) {}
|
||||||
|
|
||||||
|
public record Manga(
|
||||||
|
Long id,
|
||||||
|
Long idMal,
|
||||||
|
Title title,
|
||||||
|
String status,
|
||||||
|
String description, // synopsis
|
||||||
|
int chapters,
|
||||||
|
int averageScore, // score (0-100)
|
||||||
|
CoverImage coverImage,
|
||||||
|
List<String> genres,
|
||||||
|
FuzzyDate startDate,
|
||||||
|
FuzzyDate endDate,
|
||||||
|
StaffConnection staff) {
|
||||||
|
public record Title(String romaji, String english, String nativeTitle) {}
|
||||||
|
|
||||||
|
public record CoverImage(String large) {}
|
||||||
|
|
||||||
|
public record FuzzyDate(Integer year, Integer month, Integer day) {}
|
||||||
|
|
||||||
|
public record StaffConnection(List<StaffEdge> edges) {
|
||||||
|
public record StaffEdge(String role, Staff node) {
|
||||||
|
public record Staff(Name name) {
|
||||||
|
public record Name(String full) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -14,11 +14,15 @@ public interface MangaDexClient {
|
|||||||
MangaDexMangaDTO getManga(@PathVariable UUID id);
|
MangaDexMangaDTO getManga(@PathVariable UUID id);
|
||||||
|
|
||||||
@GetMapping("/manga/{id}/feed")
|
@GetMapping("/manga/{id}/feed")
|
||||||
MangaDexMangaFeedDTO getMangaFeed(@PathVariable UUID id, @RequestParam("contentRating[]") List<String> contentRating);
|
MangaDexMangaFeedDTO getMangaFeed(
|
||||||
|
@PathVariable UUID id, @RequestParam("contentRating[]") List<String> contentRating);
|
||||||
|
|
||||||
@GetMapping("/manga/{id}/feed")
|
@GetMapping("/manga/{id}/feed")
|
||||||
MangaDexMangaFeedDTO getMangaFeed(
|
MangaDexMangaFeedDTO getMangaFeed(
|
||||||
@PathVariable UUID id, @RequestParam int limit, @RequestParam int offset, @RequestParam("contentRating[]") List<String> contentRating);
|
@PathVariable UUID id,
|
||||||
|
@RequestParam int limit,
|
||||||
|
@RequestParam int offset,
|
||||||
|
@RequestParam("contentRating[]") List<String> contentRating);
|
||||||
|
|
||||||
@GetMapping("/at-home/server/{chapterId}")
|
@GetMapping("/at-home/server/{chapterId}")
|
||||||
MangaChapterDataDTO getMangaChapter(@PathVariable UUID chapterId);
|
MangaChapterDataDTO getMangaChapter(@PathVariable UUID chapterId);
|
||||||
|
|||||||
@ -2,7 +2,9 @@ package com.magamochi.mangamochi.controller;
|
|||||||
|
|
||||||
import com.magamochi.mangamochi.client.NtfyClient;
|
import com.magamochi.mangamochi.client.NtfyClient;
|
||||||
import com.magamochi.mangamochi.model.dto.DefaultResponseDTO;
|
import com.magamochi.mangamochi.model.dto.DefaultResponseDTO;
|
||||||
|
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||||
import com.magamochi.mangamochi.model.repository.UserRepository;
|
import com.magamochi.mangamochi.model.repository.UserRepository;
|
||||||
|
import com.magamochi.mangamochi.queue.UpdateMangaDataProducer;
|
||||||
import com.magamochi.mangamochi.task.ImageCleanupTask;
|
import com.magamochi.mangamochi.task.ImageCleanupTask;
|
||||||
import com.magamochi.mangamochi.task.MangaFollowUpdateTask;
|
import com.magamochi.mangamochi.task.MangaFollowUpdateTask;
|
||||||
import com.magamochi.mangamochi.task.UpdateMangaListTask;
|
import com.magamochi.mangamochi.task.UpdateMangaListTask;
|
||||||
@ -18,8 +20,20 @@ public class ManagementController {
|
|||||||
private final ImageCleanupTask imageCleanupTask;
|
private final ImageCleanupTask imageCleanupTask;
|
||||||
private final MangaFollowUpdateTask mangaFollowUpdateTask;
|
private final MangaFollowUpdateTask mangaFollowUpdateTask;
|
||||||
private final UserRepository userRepository;
|
private final UserRepository userRepository;
|
||||||
|
|
||||||
private final NtfyClient ntfyClient;
|
private final NtfyClient ntfyClient;
|
||||||
|
private final UpdateMangaDataProducer updateMangaDataProducer;
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Trigger manga data update",
|
||||||
|
description = "Triggers the update of the metadata for a manga by its ID",
|
||||||
|
tags = {"Management"},
|
||||||
|
operationId = "triggerUpdateMangaData")
|
||||||
|
@PostMapping("update-manga-data/{mangaId}")
|
||||||
|
public DefaultResponseDTO<Void> triggerUpdateMangaData(@PathVariable Long mangaId) {
|
||||||
|
updateMangaDataProducer.sendUpdateMangaDataCommand(new UpdateMangaDataCommand(mangaId));
|
||||||
|
|
||||||
|
return DefaultResponseDTO.ok().build();
|
||||||
|
}
|
||||||
|
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "Queue update manga list",
|
summary = "Queue update manga list",
|
||||||
|
|||||||
@ -4,4 +4,8 @@ public class UnprocessableException extends RuntimeException {
|
|||||||
public UnprocessableException(String message) {
|
public UnprocessableException(String message) {
|
||||||
super(message);
|
super(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public UnprocessableException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,4 +2,4 @@ package com.magamochi.mangamochi.model.dto;
|
|||||||
|
|
||||||
import jakarta.validation.constraints.NotNull;
|
import jakarta.validation.constraints.NotNull;
|
||||||
|
|
||||||
public record ImportRequestDTO(String metadataId, @NotNull String id) {}
|
public record ImportRequestDTO(String metadataId, String aniListId, @NotNull String id) {}
|
||||||
|
|||||||
@ -22,6 +22,8 @@ public class Manga {
|
|||||||
|
|
||||||
private Long malId;
|
private Long malId;
|
||||||
|
|
||||||
|
private Long aniListId;
|
||||||
|
|
||||||
private String title;
|
private String title;
|
||||||
|
|
||||||
private String status;
|
private String status;
|
||||||
|
|||||||
@ -6,4 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
|
|||||||
|
|
||||||
public interface AuthorRepository extends JpaRepository<Author, Long> {
|
public interface AuthorRepository extends JpaRepository<Author, Long> {
|
||||||
Optional<Author> findByMalId(Long aLong);
|
Optional<Author> findByMalId(Long aLong);
|
||||||
|
|
||||||
|
Optional<Author> findByName(String name);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,4 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
|
|||||||
|
|
||||||
public interface GenreRepository extends JpaRepository<Genre, Long> {
|
public interface GenreRepository extends JpaRepository<Genre, Long> {
|
||||||
Optional<Genre> findByMalId(Long malId);
|
Optional<Genre> findByMalId(Long malId);
|
||||||
|
|
||||||
|
Optional<Genre> findByName(String name);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -13,4 +13,6 @@ public interface MangaRepository
|
|||||||
List<Manga> findByFollowTrue();
|
List<Manga> findByFollowTrue();
|
||||||
|
|
||||||
Optional<Manga> findByMalId(Long malId);
|
Optional<Manga> findByMalId(Long malId);
|
||||||
|
|
||||||
|
Optional<Manga> findByAniListId(Long aniListId);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,12 +10,13 @@ import com.magamochi.mangamochi.model.enumeration.ArchiveFileType;
|
|||||||
import com.magamochi.mangamochi.model.repository.MangaChapterImageRepository;
|
import com.magamochi.mangamochi.model.repository.MangaChapterImageRepository;
|
||||||
import com.magamochi.mangamochi.model.repository.MangaChapterRepository;
|
import com.magamochi.mangamochi.model.repository.MangaChapterRepository;
|
||||||
import com.magamochi.mangamochi.service.providers.ContentProviderFactory;
|
import com.magamochi.mangamochi.service.providers.ContentProviderFactory;
|
||||||
|
import io.github.resilience4j.retry.Retry;
|
||||||
|
import io.github.resilience4j.retry.RetryRegistry;
|
||||||
import java.io.BufferedInputStream;
|
import java.io.BufferedInputStream;
|
||||||
import java.io.BufferedOutputStream;
|
import java.io.BufferedOutputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -39,6 +40,7 @@ public class MangaChapterService {
|
|||||||
private final ContentProviderFactory contentProviderFactory;
|
private final ContentProviderFactory contentProviderFactory;
|
||||||
|
|
||||||
private final RateLimiter imageDownloadRateLimiter;
|
private final RateLimiter imageDownloadRateLimiter;
|
||||||
|
private final RetryRegistry retryRegistry;
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public void fetchChapter(Long chapterId) {
|
public void fetchChapter(Long chapterId) {
|
||||||
@ -53,6 +55,8 @@ public class MangaChapterService {
|
|||||||
"No images found on provider for Manga Chapter ID: " + chapterId);
|
"No images found on provider for Manga Chapter ID: " + chapterId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var retryConfig = retryRegistry.retry("ImageDownloadRetry").getRetryConfig();
|
||||||
|
|
||||||
var chapterImages =
|
var chapterImages =
|
||||||
chapterImagesUrls.entrySet().parallelStream()
|
chapterImagesUrls.entrySet().parallelStream()
|
||||||
.map(
|
.map(
|
||||||
@ -60,33 +64,55 @@ public class MangaChapterService {
|
|||||||
imageDownloadRateLimiter.acquire();
|
imageDownloadRateLimiter.acquire();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
var inputStream =
|
var finalUrl = new URI(entry.getValue().trim()).toASCIIString().trim();
|
||||||
new BufferedInputStream(
|
var retry =
|
||||||
new URL(new URI(entry.getValue().trim()).toASCIIString().trim())
|
Retry.of("image-download-" + chapterId + "-" + entry.getKey(), retryConfig);
|
||||||
.openStream());
|
|
||||||
|
|
||||||
var bytes = inputStream.readAllBytes();
|
retry
|
||||||
|
.getEventPublisher()
|
||||||
|
.onRetry(
|
||||||
|
event ->
|
||||||
|
log.warn(
|
||||||
|
"Retrying image download {}/{} for chapter {}. Attempt #{}. Error: {}",
|
||||||
|
entry.getKey() + 1,
|
||||||
|
chapterImagesUrls.size(),
|
||||||
|
chapterId,
|
||||||
|
event.getNumberOfRetryAttempts(),
|
||||||
|
event.getLastThrowable().getMessage()));
|
||||||
|
|
||||||
inputStream.close();
|
return retry.executeCheckedSupplier(
|
||||||
var image =
|
() -> {
|
||||||
imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapterId);
|
var url = new URL(finalUrl);
|
||||||
|
var connection = url.openConnection();
|
||||||
|
connection.setConnectTimeout(5000);
|
||||||
|
connection.setReadTimeout(5000);
|
||||||
|
|
||||||
log.info(
|
try (var inputStream =
|
||||||
"Downloaded image {}/{} for manga {} chapter {}: {}",
|
new BufferedInputStream(connection.getInputStream())) {
|
||||||
entry.getKey() + 1,
|
var bytes = inputStream.readAllBytes();
|
||||||
chapterImagesUrls.size(),
|
|
||||||
chapter.getMangaProvider().getManga().getTitle(),
|
|
||||||
chapterId,
|
|
||||||
entry.getValue());
|
|
||||||
|
|
||||||
return MangaChapterImage.builder()
|
var image =
|
||||||
.mangaChapter(chapter)
|
imageService.uploadImage(
|
||||||
.position(entry.getKey())
|
bytes, "image/jpeg", "chapter/" + chapterId);
|
||||||
.image(image)
|
|
||||||
.build();
|
log.info(
|
||||||
} catch (IOException | URISyntaxException e) {
|
"Downloaded image {}/{} for manga {} chapter {}: {}",
|
||||||
|
entry.getKey() + 1,
|
||||||
|
chapterImagesUrls.size(),
|
||||||
|
chapter.getMangaProvider().getManga().getTitle(),
|
||||||
|
chapterId,
|
||||||
|
entry.getValue());
|
||||||
|
|
||||||
|
return MangaChapterImage.builder()
|
||||||
|
.mangaChapter(chapter)
|
||||||
|
.position(entry.getKey())
|
||||||
|
.image(image)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (Throwable e) {
|
||||||
throw new UnprocessableException(
|
throw new UnprocessableException(
|
||||||
"Could not download image for chapter ID: " + chapterId);
|
"Could not download image for chapter ID: " + chapterId, e);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.toList();
|
.toList();
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
package com.magamochi.mangamochi.service;
|
package com.magamochi.mangamochi.service;
|
||||||
|
|
||||||
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
import com.google.common.util.concurrent.RateLimiter;
|
import com.google.common.util.concurrent.RateLimiter;
|
||||||
|
import com.magamochi.mangamochi.client.AniListClient;
|
||||||
import com.magamochi.mangamochi.client.JikanClient;
|
import com.magamochi.mangamochi.client.JikanClient;
|
||||||
import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO;
|
import com.magamochi.mangamochi.model.dto.TitleMatchRequestDTO;
|
||||||
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
import com.magamochi.mangamochi.model.dto.UpdateMangaDataCommand;
|
||||||
@ -24,6 +27,7 @@ public class MangaCreationService {
|
|||||||
private final TitleMatcherService titleMatcherService;
|
private final TitleMatcherService titleMatcherService;
|
||||||
|
|
||||||
private final JikanClient jikanClient;
|
private final JikanClient jikanClient;
|
||||||
|
private final AniListClient aniListClient;
|
||||||
|
|
||||||
private final RateLimiter jikanRateLimiter;
|
private final RateLimiter jikanRateLimiter;
|
||||||
|
|
||||||
@ -78,28 +82,74 @@ public class MangaCreationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var result = resultOptional.get();
|
var result = resultOptional.get();
|
||||||
return getOrCreateManga(result.mal_id(), result.title());
|
return getOrCreateManga(result.mal_id(), null, result.title());
|
||||||
}
|
}
|
||||||
|
|
||||||
public Manga getOrCreateManga(Long malId) {
|
public Manga getOrCreateManga(Long malId, Long aniListId) {
|
||||||
jikanRateLimiter.acquire();
|
if (nonNull(malId)) {
|
||||||
var data = jikanClient.getMangaById(malId);
|
try {
|
||||||
|
jikanRateLimiter.acquire();
|
||||||
|
var data = jikanClient.getMangaById(malId);
|
||||||
|
return getOrCreateManga(data.data().mal_id(), aniListId, data.data().title());
|
||||||
|
} catch (feign.FeignException.NotFound e) {
|
||||||
|
log.warn("Manga not found on MyAnimeList for ID: {}", malId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return getOrCreateManga(data.data().mal_id(), data.data().title());
|
if (nonNull(aniListId)) {
|
||||||
|
try {
|
||||||
|
var query =
|
||||||
|
"""
|
||||||
|
query ($id: Int) {
|
||||||
|
Media (id: $id, type: MANGA) {
|
||||||
|
id
|
||||||
|
idMal
|
||||||
|
title {
|
||||||
|
romaji
|
||||||
|
english
|
||||||
|
native
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
var request =
|
||||||
|
new AniListClient.GraphQLRequest(
|
||||||
|
query, new AniListClient.GraphQLRequest.Variables(aniListId));
|
||||||
|
var data = aniListClient.getManga(request).data().Media();
|
||||||
|
|
||||||
|
String title =
|
||||||
|
nonNull(data.title().english()) ? data.title().english() : data.title().romaji();
|
||||||
|
return getOrCreateManga(data.idMal(), data.id(), title);
|
||||||
|
} catch (feign.FeignException.NotFound e) {
|
||||||
|
log.warn("Manga not found on AniList for ID: {}", aniListId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new RuntimeException("Could not find manga on any provider");
|
||||||
}
|
}
|
||||||
|
|
||||||
private Manga getOrCreateManga(Long malId, String title) {
|
private Manga getOrCreateManga(Long malId, Long aniListId, String title) {
|
||||||
return mangaRepository
|
var mangaOptional = java.util.Optional.<Manga>empty();
|
||||||
.findByMalId(malId)
|
|
||||||
.orElseGet(
|
|
||||||
() -> {
|
|
||||||
var manga = mangaRepository.save(Manga.builder().title(title).malId(malId).build());
|
|
||||||
|
|
||||||
updateMangaDataProducer.sendUpdateMangaDataCommand(
|
if (nonNull(malId)) {
|
||||||
new UpdateMangaDataCommand(manga.getId()));
|
mangaOptional = mangaRepository.findByMalId(malId);
|
||||||
|
}
|
||||||
|
|
||||||
return manga;
|
if (mangaOptional.isEmpty() && nonNull(aniListId)) {
|
||||||
});
|
mangaOptional = mangaRepository.findByAniListId(aniListId);
|
||||||
|
}
|
||||||
|
|
||||||
|
return mangaOptional.orElseGet(
|
||||||
|
() -> {
|
||||||
|
var manga =
|
||||||
|
mangaRepository.save(
|
||||||
|
Manga.builder().title(title).malId(malId).aniListId(aniListId).build());
|
||||||
|
|
||||||
|
updateMangaDataProducer.sendUpdateMangaDataCommand(
|
||||||
|
new UpdateMangaDataCommand(manga.getId()));
|
||||||
|
|
||||||
|
return manga;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createMangaImportReview(String title, String url, Provider provider) {
|
private void createMangaImportReview(String title, String url, Provider provider) {
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import static java.util.Objects.isNull;
|
|||||||
import static java.util.Objects.nonNull;
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
import com.google.common.util.concurrent.RateLimiter;
|
import com.google.common.util.concurrent.RateLimiter;
|
||||||
|
import com.magamochi.mangamochi.client.AniListClient;
|
||||||
import com.magamochi.mangamochi.client.JikanClient;
|
import com.magamochi.mangamochi.client.JikanClient;
|
||||||
import com.magamochi.mangamochi.exception.NotFoundException;
|
import com.magamochi.mangamochi.exception.NotFoundException;
|
||||||
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
||||||
@ -12,7 +13,10 @@ import com.magamochi.mangamochi.model.repository.*;
|
|||||||
import com.magamochi.mangamochi.util.DoubleUtil;
|
import com.magamochi.mangamochi.util.DoubleUtil;
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -43,6 +47,7 @@ public class MangaImportService {
|
|||||||
private final MangaRepository mangaRepository;
|
private final MangaRepository mangaRepository;
|
||||||
|
|
||||||
private final JikanClient jikanClient;
|
private final JikanClient jikanClient;
|
||||||
|
private final AniListClient aniListClient;
|
||||||
private final MangaChapterImageRepository mangaChapterImageRepository;
|
private final MangaChapterImageRepository mangaChapterImageRepository;
|
||||||
private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository;
|
private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository;
|
||||||
|
|
||||||
@ -133,99 +138,233 @@ public class MangaImportService {
|
|||||||
public void updateMangaData(Manga manga) {
|
public void updateMangaData(Manga manga) {
|
||||||
log.info("Updating manga {}", manga.getTitle());
|
log.info("Updating manga {}", manga.getTitle());
|
||||||
|
|
||||||
try {
|
if (nonNull(manga.getMalId())) {
|
||||||
jikanRateLimiter.acquire();
|
try {
|
||||||
var mangaData = jikanClient.getMangaById(manga.getMalId());
|
updateFromJikan(manga);
|
||||||
|
return;
|
||||||
manga.setSynopsis(mangaData.data().synopsis());
|
} catch (Exception e) {
|
||||||
manga.setStatus(mangaData.data().status());
|
log.warn(
|
||||||
manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2));
|
"Error updating manga data from Jikan for manga {}. Trying AniList... Error: {}",
|
||||||
manga.setPublishedFrom(mangaData.data().published().from());
|
manga.getTitle(),
|
||||||
manga.setPublishedTo(mangaData.data().published().to());
|
e.getMessage());
|
||||||
manga.setChapterCount(mangaData.data().chapters());
|
|
||||||
|
|
||||||
var authors =
|
|
||||||
mangaData.data().authors().stream()
|
|
||||||
.map(
|
|
||||||
authorData ->
|
|
||||||
authorRepository
|
|
||||||
.findByMalId(authorData.mal_id())
|
|
||||||
.orElseGet(
|
|
||||||
() ->
|
|
||||||
authorRepository.save(
|
|
||||||
Author.builder()
|
|
||||||
.malId(authorData.mal_id())
|
|
||||||
.name(authorData.name())
|
|
||||||
.build())))
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
var mangaAuthors =
|
|
||||||
authors.stream()
|
|
||||||
.map(
|
|
||||||
author ->
|
|
||||||
mangaAuthorRepository
|
|
||||||
.findByMangaAndAuthor(manga, author)
|
|
||||||
.orElseGet(
|
|
||||||
() ->
|
|
||||||
mangaAuthorRepository.save(
|
|
||||||
MangaAuthor.builder().manga(manga).author(author).build())))
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
manga.setMangaAuthors(mangaAuthors);
|
|
||||||
|
|
||||||
var genres =
|
|
||||||
mangaData.data().genres().stream()
|
|
||||||
.map(
|
|
||||||
genreData ->
|
|
||||||
genreRepository
|
|
||||||
.findByMalId(genreData.mal_id())
|
|
||||||
.orElseGet(
|
|
||||||
() ->
|
|
||||||
genreRepository.save(
|
|
||||||
Genre.builder()
|
|
||||||
.malId(genreData.mal_id())
|
|
||||||
.name(genreData.name())
|
|
||||||
.build())))
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
var mangaGenres =
|
|
||||||
genres.stream()
|
|
||||||
.map(
|
|
||||||
genre ->
|
|
||||||
mangaGenreRepository
|
|
||||||
.findByMangaAndGenre(manga, genre)
|
|
||||||
.orElseGet(
|
|
||||||
() ->
|
|
||||||
mangaGenreRepository.save(
|
|
||||||
MangaGenre.builder().manga(manga).genre(genre).build())))
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
manga.setMangaGenres(mangaGenres);
|
|
||||||
|
|
||||||
if (isNull(manga.getCoverImage())) {
|
|
||||||
var inputStream =
|
|
||||||
new BufferedInputStream(
|
|
||||||
new URL(new URI(mangaData.data().images().jpg().large_image_url()).toASCIIString())
|
|
||||||
.openStream());
|
|
||||||
|
|
||||||
var bytes = inputStream.readAllBytes();
|
|
||||||
|
|
||||||
inputStream.close();
|
|
||||||
var image = imageService.uploadImage(bytes, "image/jpeg", "cover");
|
|
||||||
|
|
||||||
manga.setCoverImage(image);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var mangaEntity = mangaRepository.save(manga);
|
|
||||||
var alternativeTitles =
|
|
||||||
mangaData.data().title_synonyms().stream()
|
|
||||||
.map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build())
|
|
||||||
.toList();
|
|
||||||
mangaAlternativeTitlesRepository.saveAll(alternativeTitles);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("Error updating manga data for manga {}. {}", manga.getTitle(), e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (nonNull(manga.getAniListId())) {
|
||||||
|
try {
|
||||||
|
updateFromAniList(manga);
|
||||||
|
return;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn(
|
||||||
|
"Error updating manga data from AniList for manga {}. Error: {}",
|
||||||
|
manga.getTitle(),
|
||||||
|
e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.warn(
|
||||||
|
"Could not update manga data for {}. No provider data available/found.", manga.getTitle());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateFromJikan(Manga manga) throws IOException, URISyntaxException {
|
||||||
|
jikanRateLimiter.acquire();
|
||||||
|
var mangaData = jikanClient.getMangaById(manga.getMalId());
|
||||||
|
|
||||||
|
manga.setSynopsis(mangaData.data().synopsis());
|
||||||
|
manga.setStatus(mangaData.data().status());
|
||||||
|
manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2));
|
||||||
|
manga.setPublishedFrom(mangaData.data().published().from());
|
||||||
|
manga.setPublishedTo(mangaData.data().published().to());
|
||||||
|
manga.setChapterCount(mangaData.data().chapters());
|
||||||
|
|
||||||
|
var authors =
|
||||||
|
mangaData.data().authors().stream()
|
||||||
|
.map(
|
||||||
|
authorData ->
|
||||||
|
authorRepository
|
||||||
|
.findByMalId(authorData.mal_id())
|
||||||
|
.orElseGet(
|
||||||
|
() ->
|
||||||
|
authorRepository.save(
|
||||||
|
Author.builder()
|
||||||
|
.malId(authorData.mal_id())
|
||||||
|
.name(authorData.name())
|
||||||
|
.build())))
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
updateMangaAuthors(manga, authors);
|
||||||
|
|
||||||
|
var genres =
|
||||||
|
mangaData.data().genres().stream()
|
||||||
|
.map(
|
||||||
|
genreData ->
|
||||||
|
genreRepository
|
||||||
|
.findByMalId(genreData.mal_id())
|
||||||
|
.orElseGet(
|
||||||
|
() ->
|
||||||
|
genreRepository.save(
|
||||||
|
Genre.builder()
|
||||||
|
.malId(genreData.mal_id())
|
||||||
|
.name(genreData.name())
|
||||||
|
.build())))
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
updateMangaGenres(manga, genres);
|
||||||
|
|
||||||
|
if (isNull(manga.getCoverImage())) {
|
||||||
|
downloadCoverImage(manga, mangaData.data().images().jpg().large_image_url());
|
||||||
|
}
|
||||||
|
|
||||||
|
var mangaEntity = mangaRepository.save(manga);
|
||||||
|
var alternativeTitles =
|
||||||
|
mangaData.data().title_synonyms().stream()
|
||||||
|
.map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build())
|
||||||
|
.toList();
|
||||||
|
mangaAlternativeTitlesRepository.saveAll(alternativeTitles);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateFromAniList(Manga manga) throws IOException, URISyntaxException {
|
||||||
|
var query =
|
||||||
|
"""
|
||||||
|
query ($id: Int) {
|
||||||
|
Media (id: $id, type: MANGA) {
|
||||||
|
startDate { year month day }
|
||||||
|
endDate { year month day }
|
||||||
|
description
|
||||||
|
status
|
||||||
|
averageScore
|
||||||
|
chapters
|
||||||
|
coverImage { large }
|
||||||
|
genres
|
||||||
|
staff {
|
||||||
|
edges {
|
||||||
|
role
|
||||||
|
node {
|
||||||
|
name {
|
||||||
|
full
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
var request =
|
||||||
|
new AniListClient.GraphQLRequest(
|
||||||
|
query, new AniListClient.GraphQLRequest.Variables(manga.getAniListId()));
|
||||||
|
var media = aniListClient.getManga(request).data().Media();
|
||||||
|
|
||||||
|
manga.setSynopsis(media.description());
|
||||||
|
manga.setStatus(mapAniListStatus(media.status()));
|
||||||
|
manga.setScore(DoubleUtil.round((double) media.averageScore() / 10, 2)); // 0-100 -> 0-10
|
||||||
|
manga.setPublishedFrom(convertFuzzyDate(media.startDate()));
|
||||||
|
manga.setPublishedTo(convertFuzzyDate(media.endDate()));
|
||||||
|
manga.setChapterCount(media.chapters());
|
||||||
|
|
||||||
|
var authors =
|
||||||
|
media.staff().edges().stream()
|
||||||
|
.filter(edge -> isAuthorRole(edge.role()))
|
||||||
|
.map(edge -> edge.node().name().full())
|
||||||
|
.distinct()
|
||||||
|
.map(
|
||||||
|
name ->
|
||||||
|
authorRepository
|
||||||
|
.findByName(name)
|
||||||
|
.orElseGet(
|
||||||
|
() -> authorRepository.save(Author.builder().name(name).build())))
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
updateMangaAuthors(manga, authors);
|
||||||
|
|
||||||
|
var genres =
|
||||||
|
media.genres().stream()
|
||||||
|
.map(
|
||||||
|
name ->
|
||||||
|
genreRepository
|
||||||
|
.findByName(name)
|
||||||
|
.orElseGet(() -> genreRepository.save(Genre.builder().name(name).build())))
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
updateMangaGenres(manga, genres);
|
||||||
|
|
||||||
|
if (isNull(manga.getCoverImage())) {
|
||||||
|
downloadCoverImage(manga, media.coverImage().large());
|
||||||
|
}
|
||||||
|
|
||||||
|
mangaRepository.save(manga);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isAuthorRole(String role) {
|
||||||
|
return role.equalsIgnoreCase("Story & Art")
|
||||||
|
|| role.equalsIgnoreCase("Story")
|
||||||
|
|| role.equalsIgnoreCase("Art");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String mapAniListStatus(String status) {
|
||||||
|
return switch (status) {
|
||||||
|
case "RELEASING" -> "Publishing";
|
||||||
|
case "FINISHED" -> "Finished";
|
||||||
|
case "NOT_YET_RELEASED" -> "Not yet published";
|
||||||
|
default -> "Unknown";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private OffsetDateTime convertFuzzyDate(AniListClient.MangaResponse.Manga.FuzzyDate date) {
|
||||||
|
if (isNull(date) || isNull(date.year())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return OffsetDateTime.of(
|
||||||
|
date.year(),
|
||||||
|
isNull(date.month()) ? 1 : date.month(),
|
||||||
|
isNull(date.day()) ? 1 : date.day(),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateMangaAuthors(Manga manga, List<Author> authors) {
|
||||||
|
var mangaAuthors =
|
||||||
|
authors.stream()
|
||||||
|
.map(
|
||||||
|
author ->
|
||||||
|
mangaAuthorRepository
|
||||||
|
.findByMangaAndAuthor(manga, author)
|
||||||
|
.orElseGet(
|
||||||
|
() ->
|
||||||
|
mangaAuthorRepository.save(
|
||||||
|
MangaAuthor.builder().manga(manga).author(author).build())))
|
||||||
|
.toList();
|
||||||
|
manga.setMangaAuthors(mangaAuthors);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateMangaGenres(Manga manga, List<Genre> genres) {
|
||||||
|
var mangaGenres =
|
||||||
|
genres.stream()
|
||||||
|
.map(
|
||||||
|
genre ->
|
||||||
|
mangaGenreRepository
|
||||||
|
.findByMangaAndGenre(manga, genre)
|
||||||
|
.orElseGet(
|
||||||
|
() ->
|
||||||
|
mangaGenreRepository.save(
|
||||||
|
MangaGenre.builder().manga(manga).genre(genre).build())))
|
||||||
|
.toList();
|
||||||
|
manga.setMangaGenres(mangaGenres);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void downloadCoverImage(Manga manga, String imageUrl)
|
||||||
|
throws IOException, URISyntaxException {
|
||||||
|
var inputStream =
|
||||||
|
new BufferedInputStream(new URL(new URI(imageUrl).toASCIIString()).openStream());
|
||||||
|
|
||||||
|
var bytes = inputStream.readAllBytes();
|
||||||
|
|
||||||
|
inputStream.close();
|
||||||
|
var image = imageService.uploadImage(bytes, "image/jpeg", "cover");
|
||||||
|
|
||||||
|
manga.setCoverImage(image);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MangaChapter persistMangaChapter(
|
public MangaChapter persistMangaChapter(
|
||||||
|
|||||||
@ -31,9 +31,12 @@ public class ProviderManualMangaImportService {
|
|||||||
|
|
||||||
var title = contentProvider.getMangaTitle(requestDTO.id());
|
var title = contentProvider.getMangaTitle(requestDTO.id());
|
||||||
|
|
||||||
|
var malId = nonNull(requestDTO.metadataId()) ? Long.parseLong(requestDTO.metadataId()) : null;
|
||||||
|
var aniListId = nonNull(requestDTO.aniListId()) ? Long.parseLong(requestDTO.aniListId()) : null;
|
||||||
|
|
||||||
var manga =
|
var manga =
|
||||||
(nonNull(requestDTO.metadataId()) && !requestDTO.metadataId().isBlank())
|
nonNull(malId) || nonNull(aniListId)
|
||||||
? mangaCreationService.getOrCreateManga(Long.parseLong(requestDTO.metadataId()))
|
? mangaCreationService.getOrCreateManga(malId, aniListId)
|
||||||
: mangaCreationService.getOrCreateManga(title, requestDTO.id(), provider);
|
: mangaCreationService.getOrCreateManga(title, requestDTO.id(), provider);
|
||||||
|
|
||||||
if (isNull(manga)) {
|
if (isNull(manga)) {
|
||||||
|
|||||||
@ -1,5 +1,7 @@
|
|||||||
package com.magamochi.mangamochi.service.providers.impl;
|
package com.magamochi.mangamochi.service.providers.impl;
|
||||||
|
|
||||||
|
import static java.util.Objects.isNull;
|
||||||
|
|
||||||
import com.google.common.util.concurrent.RateLimiter;
|
import com.google.common.util.concurrent.RateLimiter;
|
||||||
import com.magamochi.mangamochi.client.MangaDexClient;
|
import com.magamochi.mangamochi.client.MangaDexClient;
|
||||||
import com.magamochi.mangamochi.exception.UnprocessableException;
|
import com.magamochi.mangamochi.exception.UnprocessableException;
|
||||||
@ -15,8 +17,6 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import static java.util.Objects.isNull;
|
|
||||||
|
|
||||||
@Log4j2
|
@Log4j2
|
||||||
@Service(ContentProviders.MANGA_DEX)
|
@Service(ContentProviders.MANGA_DEX)
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@ -29,10 +29,13 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
|
|||||||
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
|
||||||
try {
|
try {
|
||||||
mangaDexRateLimiter.acquire();
|
mangaDexRateLimiter.acquire();
|
||||||
var response = mangaDexClient.getMangaFeed(UUID.fromString(provider.getUrl()), List.of("safe", "suggestive", "erotica", "pornographic"));
|
var response =
|
||||||
|
mangaDexClient.getMangaFeed(
|
||||||
|
UUID.fromString(provider.getUrl()),
|
||||||
|
List.of("safe", "suggestive", "erotica", "pornographic"));
|
||||||
|
|
||||||
var mangas = new ArrayList<>(response.data());
|
var mangas = new ArrayList<>(response.data());
|
||||||
var totalPages = (int) Math.ceil((double) response.total() / 500);
|
var totalPages = (int) Math.ceil((double) response.total() / 100);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
IntStream.range(1, totalPages)
|
IntStream.range(1, totalPages)
|
||||||
@ -42,7 +45,11 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
|
|||||||
mangaDexRateLimiter.acquire();
|
mangaDexRateLimiter.acquire();
|
||||||
|
|
||||||
var pagedResponse =
|
var pagedResponse =
|
||||||
mangaDexClient.getMangaFeed(UUID.fromString(provider.getUrl()), 500, i * 500, List.of("safe", "suggestive", "erotica", "pornographic"));
|
mangaDexClient.getMangaFeed(
|
||||||
|
UUID.fromString(provider.getUrl()),
|
||||||
|
100,
|
||||||
|
i * 100,
|
||||||
|
List.of("safe", "suggestive", "erotica", "pornographic"));
|
||||||
|
|
||||||
mangas.addAll(pagedResponse.data());
|
mangas.addAll(pagedResponse.data());
|
||||||
});
|
});
|
||||||
@ -50,7 +57,8 @@ public class MangaDexProvider implements ContentProvider, ManualImportContentPro
|
|||||||
log.warn(e.getMessage());
|
log.warn(e.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
// NOTE: this is getting only pt-br and en chapters for now, we may want to make this configurable
|
// NOTE: this is getting only pt-br and en chapters for now, we may want to make this
|
||||||
|
// configurable
|
||||||
// later
|
// later
|
||||||
var languagesToImport = Map.of("pt-br", "pt-BR", "en", "en-US");
|
var languagesToImport = Map.of("pt-br", "pt-BR", "en", "en-US");
|
||||||
|
|
||||||
|
|||||||
@ -74,6 +74,19 @@ resilience4j:
|
|||||||
seconds: 5
|
seconds: 5
|
||||||
retry-exceptions:
|
retry-exceptions:
|
||||||
- feign.FeignException
|
- feign.FeignException
|
||||||
|
AniListRetry:
|
||||||
|
max-attempts: 5
|
||||||
|
wait-duration:
|
||||||
|
seconds: 5
|
||||||
|
retry-exceptions:
|
||||||
|
- feign.FeignException
|
||||||
|
ImageDownloadRetry:
|
||||||
|
max-attempts: 3
|
||||||
|
wait-duration:
|
||||||
|
seconds: 5
|
||||||
|
retry-exceptions:
|
||||||
|
- java.io.IOException
|
||||||
|
- java.net.SocketTimeoutException
|
||||||
|
|
||||||
rabbit-mq:
|
rabbit-mq:
|
||||||
queues:
|
queues:
|
||||||
|
|||||||
@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE mangas ADD COLUMN ani_list_id BIGINT;
|
||||||
Loading…
x
Reference in New Issue
Block a user