Compare commits
35 Commits
feat/manua
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d398ac8f06 | ||
|
|
331df77d28 | ||
|
|
7d8cdd2be8 | ||
|
|
9938d4649b | ||
|
|
cc4c8105ef | ||
|
|
9fd3fc9fb7 | ||
|
|
3b8a8f87e2 | ||
|
|
424b8e28b2 | ||
|
|
b51e119bcc | ||
|
|
fd151f2a9e | ||
|
|
953ebf590a | ||
|
|
095c27c87d | ||
|
|
b90954c0d4 | ||
|
|
affc136043 | ||
|
|
cfcf954a4c | ||
|
|
733e90b9d5 | ||
|
|
abf695ff1c | ||
|
|
119fe51b3f | ||
|
|
8043761bce | ||
|
|
9ec20c7d24 | ||
|
|
117148a28e | ||
|
|
22a905220e | ||
|
|
b388bf3740 | ||
|
|
24e0f48427 | ||
|
|
3c3b243984 | ||
|
|
189769b848 | ||
|
|
0d9cf7c2a2 | ||
|
|
4c0c994990 | ||
|
|
90dab24ae8 | ||
|
|
ebd38e06ee | ||
|
|
c6605cb322 | ||
|
|
77c1e49749 | ||
|
|
8ea64fef82 | ||
|
|
b6db38686a | ||
|
|
9640902c4e |
@ -14,6 +14,9 @@ RUN mvn clean package -DskipTests
|
||||
FROM eclipse-temurin:25-jre-jammy
|
||||
WORKDIR /app
|
||||
|
||||
# Install rar for CBR file creation
|
||||
RUN apt-get update && apt-get install -y rar && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy the jar from the builder stage
|
||||
COPY --from=builder /app/target/*.jar app.jar
|
||||
|
||||
|
||||
@ -1,12 +1,16 @@
|
||||
package com.magamochi.catalog.controller;
|
||||
|
||||
import com.magamochi.catalog.model.dto.MangaIngestReviewDTO;
|
||||
import com.magamochi.catalog.model.dto.ManualMangaIngestRequestDTO;
|
||||
import com.magamochi.catalog.service.MangaIngestReviewService;
|
||||
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import jakarta.validation.Valid;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/catalog")
|
||||
@ -50,4 +54,20 @@ public class MangaIngestReviewController {
|
||||
|
||||
return DefaultResponseDTO.ok().build();
|
||||
}
|
||||
|
||||
@Operation(
|
||||
summary = "Resolve manga ingest review manually",
|
||||
description =
|
||||
"Resolve a manga ingest review by manually providing all manga data and a cover image.",
|
||||
tags = {"Manga Ingest Review"},
|
||||
operationId = "resolveMangaIngestReviewManually")
|
||||
@PostMapping(value = "/ingest-reviews/manual", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
public DefaultResponseDTO<Void> resolveMangaIngestReviewManually(
|
||||
@RequestParam Long id,
|
||||
@RequestPart MultipartFile coverImage,
|
||||
@RequestPart @Valid ManualMangaIngestRequestDTO request) {
|
||||
mangaIngestReviewService.resolveIngestReviewManually(id, coverImage, request);
|
||||
|
||||
return DefaultResponseDTO.ok().build();
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
package com.magamochi.catalog.model.dto;
|
||||
|
||||
import com.magamochi.catalog.model.enumeration.MangaStatus;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import lombok.Builder;
|
||||
|
||||
@Builder
|
||||
public record ManualMangaIngestRequestDTO(
|
||||
@NotBlank String title,
|
||||
String synopsis,
|
||||
MangaStatus status,
|
||||
Double score,
|
||||
OffsetDateTime publishedFrom,
|
||||
OffsetDateTime publishedTo,
|
||||
Integer chapterCount,
|
||||
Boolean adult,
|
||||
List<String> authors,
|
||||
List<String> genres,
|
||||
List<String> alternativeTitles) {}
|
||||
@ -3,16 +3,19 @@ package com.magamochi.catalog.model.repository;
|
||||
import com.magamochi.catalog.model.entity.Manga;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
|
||||
public interface MangaRepository
|
||||
extends JpaRepository<Manga, Long>, JpaSpecificationExecutor<Manga> {
|
||||
Optional<Manga> findByTitleIgnoreCase(String title);
|
||||
List<Manga> findAllByTitleIgnoreCase(String title);
|
||||
|
||||
List<Manga> findByFollowTrue();
|
||||
|
||||
Optional<Manga> findByMalId(Long malId);
|
||||
|
||||
Optional<Manga> findByAniListId(Long aniListId);
|
||||
|
||||
List<Manga> findAllByCoverImage_Id(UUID imageId);
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package com.magamochi.model.specification;
|
||||
package com.magamochi.catalog.model.specification;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
import static java.util.Objects.nonNull;
|
||||
@ -13,7 +13,9 @@ import org.springframework.stereotype.Service;
|
||||
public class MangaIngestConsumer {
|
||||
private final MangaIngestService mangaIngestService;
|
||||
|
||||
@RabbitListener(queues = "${queues.manga-ingest}")
|
||||
@RabbitListener(
|
||||
queues = "${queues.manga-ingest}",
|
||||
concurrency = "${queues.manga-ingest-concurrency:3-5}")
|
||||
public void receiveMangaIngestCommand(MangaIngestCommand command) {
|
||||
log.info("Received manga ingest command: {}", command);
|
||||
mangaIngestService.ingestManga(command.providerId(), command.mangaTitle(), command.url());
|
||||
|
||||
@ -8,6 +8,7 @@ import com.magamochi.catalog.client.AniListClient;
|
||||
import com.magamochi.catalog.model.dto.MangaDataDTO;
|
||||
import com.magamochi.catalog.model.enumeration.MangaStatus;
|
||||
import com.magamochi.catalog.util.DoubleUtil;
|
||||
import com.magamochi.common.config.ExternalApiCircuitBreaker;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
@ -26,50 +27,77 @@ import org.springframework.stereotype.Service;
|
||||
public class AniListService {
|
||||
private final AniListClient aniListClient;
|
||||
private final RateLimiter aniListRateLimiter;
|
||||
private final ExternalApiCircuitBreaker circuitBreaker;
|
||||
|
||||
public Map<String, AniListClient.Manga> searchMangaByTitle(String title) {
|
||||
var request = getSearchGraphQLRequest(title);
|
||||
return circuitBreaker.execute(
|
||||
"aniList",
|
||||
() -> {
|
||||
var request = getSearchGraphQLRequest(title);
|
||||
|
||||
aniListRateLimiter.acquire();
|
||||
var response = aniListClient.searchManga(request);
|
||||
aniListRateLimiter.acquire();
|
||||
var response = aniListClient.searchManga(request);
|
||||
|
||||
if (nonNull(response) && nonNull(response.data()) && nonNull(response.data().page())) {
|
||||
return response.data().page().media().stream()
|
||||
.flatMap(
|
||||
manga ->
|
||||
Stream.of(
|
||||
manga.title().romaji(),
|
||||
manga.title().english(),
|
||||
manga.title().nativeTitle())
|
||||
.filter(Objects::nonNull)
|
||||
.filter(t -> !t.isBlank())
|
||||
.map(titleString -> Map.entry(titleString, manga)))
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey, Map.Entry::getValue, (existingManga, manga) -> existingManga));
|
||||
}
|
||||
if (nonNull(response) && nonNull(response.data()) && nonNull(response.data().page())) {
|
||||
return response.data().page().media().stream()
|
||||
.flatMap(
|
||||
manga ->
|
||||
Stream.of(
|
||||
manga.title().romaji(),
|
||||
manga.title().english(),
|
||||
manga.title().nativeTitle())
|
||||
.filter(Objects::nonNull)
|
||||
.filter(t -> !t.isBlank())
|
||||
.map(titleString -> Map.entry(titleString, manga)))
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey,
|
||||
Map.Entry::getValue,
|
||||
(existingManga, manga) -> existingManga));
|
||||
}
|
||||
|
||||
return Map.of();
|
||||
return Map.of();
|
||||
},
|
||||
throwable -> {
|
||||
log.error("AniList circuit breaker open for search '{}'", title);
|
||||
return Map.of();
|
||||
});
|
||||
}
|
||||
|
||||
public Map<Long, MangaDataDTO> getMangasByTitle(String title) {
|
||||
var request = getSearchGraphQLRequest(title);
|
||||
return circuitBreaker.execute(
|
||||
"aniList",
|
||||
() -> {
|
||||
var request = getSearchGraphQLRequest(title);
|
||||
|
||||
aniListRateLimiter.acquire();
|
||||
var response = aniListClient.searchManga(request);
|
||||
aniListRateLimiter.acquire();
|
||||
var response = aniListClient.searchManga(request);
|
||||
|
||||
return response.data().page().media().stream()
|
||||
.map(this::parseMangaData)
|
||||
.flatMap(mangaDataMap -> mangaDataMap.entrySet().stream())
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
return response.data().page().media().stream()
|
||||
.map(this::parseMangaData)
|
||||
.flatMap(mangaDataMap -> mangaDataMap.entrySet().stream())
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
},
|
||||
throwable -> {
|
||||
log.error("AniList circuit breaker open for search '{}'", title);
|
||||
return Map.of();
|
||||
});
|
||||
}
|
||||
|
||||
public MangaDataDTO getMangaDataById(Long aniListId) {
|
||||
var request = getGraphQLRequest(aniListId);
|
||||
return circuitBreaker.execute(
|
||||
"aniList",
|
||||
() -> {
|
||||
var request = getGraphQLRequest(aniListId);
|
||||
|
||||
aniListRateLimiter.acquire();
|
||||
var manga = aniListClient.getManga(request).data().Media();
|
||||
return parseMangaData(manga).get(aniListId);
|
||||
aniListRateLimiter.acquire();
|
||||
var manga = aniListClient.getManga(request).data().Media();
|
||||
return parseMangaData(manga).get(aniListId);
|
||||
},
|
||||
throwable -> {
|
||||
log.error("AniList circuit breaker open for manga ID {}", aniListId);
|
||||
throw new RuntimeException("AniList API unavailable");
|
||||
});
|
||||
}
|
||||
|
||||
private Map<Long, MangaDataDTO> parseMangaData(AniListClient.Manga manga) {
|
||||
|
||||
@ -1,18 +1,35 @@
|
||||
package com.magamochi.catalog.service;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
import static java.util.Objects.nonNull;
|
||||
|
||||
import com.magamochi.catalog.model.dto.MangaIngestReviewDTO;
|
||||
import com.magamochi.catalog.model.dto.ManualMangaIngestRequestDTO;
|
||||
import com.magamochi.catalog.model.entity.Manga;
|
||||
import com.magamochi.catalog.model.entity.MangaAlternativeTitle;
|
||||
import com.magamochi.catalog.model.entity.MangaAuthor;
|
||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||
import com.magamochi.catalog.model.entity.MangaGenre;
|
||||
import com.magamochi.catalog.model.entity.MangaIngestReview;
|
||||
import com.magamochi.catalog.model.enumeration.MangaState;
|
||||
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
|
||||
import com.magamochi.catalog.model.repository.MangaIngestReviewRepository;
|
||||
import com.magamochi.catalog.model.repository.MangaRepository;
|
||||
import com.magamochi.common.exception.NotFoundException;
|
||||
import com.magamochi.image.model.entity.Image;
|
||||
import com.magamochi.image.service.ImageService;
|
||||
import com.magamochi.image.util.ImageHashUtil;
|
||||
import com.magamochi.ingestion.service.ContentProviderService;
|
||||
import jakarta.transaction.Transactional;
|
||||
import java.io.IOException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.tika.Tika;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@ -20,9 +37,15 @@ import org.springframework.stereotype.Service;
|
||||
public class MangaIngestReviewService {
|
||||
private final MangaResolutionService mangaResolutionService;
|
||||
private final ContentProviderService contentProviderService;
|
||||
private final ImageService imageService;
|
||||
private final GenreService genreService;
|
||||
private final AuthorService authorService;
|
||||
|
||||
private final MangaIngestReviewRepository mangaIngestReviewRepository;
|
||||
private final MangaContentProviderRepository mangaContentProviderRepository;
|
||||
private final MangaRepository mangaRepository;
|
||||
|
||||
private final Tika tika = new Tika();
|
||||
|
||||
public List<MangaIngestReviewDTO> get() {
|
||||
return mangaIngestReviewRepository.findAll().stream().map(MangaIngestReviewDTO::from).toList();
|
||||
@ -90,6 +113,121 @@ public class MangaIngestReviewService {
|
||||
contentProviderId);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void resolveIngestReviewManually(
|
||||
Long reviewId, MultipartFile coverImage, ManualMangaIngestRequestDTO dto) {
|
||||
if (coverImage.isEmpty()) {
|
||||
throw new IllegalArgumentException("Cover image is required");
|
||||
}
|
||||
|
||||
var ingestReview = get(reviewId);
|
||||
var contentProviderId = ingestReview.getContentProvider().getId();
|
||||
|
||||
if (mangaContentProviderRepository.existsByMangaTitleIgnoreCaseAndContentProvider_Id(
|
||||
dto.title(), contentProviderId)) {
|
||||
log.info(
|
||||
"Manga with title '{}' already exists for provider '{}', skipping ingest",
|
||||
dto.title(),
|
||||
contentProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
var contentProvider = contentProviderService.find(contentProviderId);
|
||||
|
||||
var image = uploadCoverImage(coverImage);
|
||||
|
||||
var unsavedManga = buildManga(dto, image);
|
||||
var manga = mangaRepository.save(unsavedManga);
|
||||
|
||||
if (nonNull(dto.genres())) {
|
||||
var genres =
|
||||
dto.genres().stream()
|
||||
.map(name -> MangaGenre.builder().manga(manga).genre(genreService.findOrCreateGenre(name)).build())
|
||||
.toList();
|
||||
manga.setMangaGenres(genres);
|
||||
}
|
||||
|
||||
if (nonNull(dto.authors())) {
|
||||
var authors =
|
||||
dto.authors().stream()
|
||||
.map(
|
||||
name ->
|
||||
MangaAuthor.builder().manga(manga).author(authorService.findOrCreateAuthor(name)).build())
|
||||
.toList();
|
||||
manga.setMangaAuthors(authors);
|
||||
}
|
||||
|
||||
if (nonNull(dto.alternativeTitles())) {
|
||||
var titles =
|
||||
dto.alternativeTitles().stream()
|
||||
.map(title -> MangaAlternativeTitle.builder().title(title).manga(manga).build())
|
||||
.toList();
|
||||
manga.setAlternativeTitles(titles);
|
||||
}
|
||||
|
||||
mangaContentProviderRepository.save(
|
||||
MangaContentProvider.builder()
|
||||
.manga(manga)
|
||||
.mangaTitle(dto.title())
|
||||
.contentProvider(contentProvider)
|
||||
.url(ingestReview.getUrl())
|
||||
.build());
|
||||
|
||||
mangaIngestReviewRepository.delete(ingestReview);
|
||||
|
||||
log.info(
|
||||
"Successfully ingested manga '{}' manually from provider {}",
|
||||
dto.title(),
|
||||
contentProviderId);
|
||||
}
|
||||
|
||||
private Image uploadCoverImage(MultipartFile coverImage) {
|
||||
try {
|
||||
var bytes = coverImage.getBytes();
|
||||
var contentType = resolveContentType(coverImage.getContentType(), bytes);
|
||||
var hash = ImageHashUtil.computeSha256(bytes);
|
||||
var imageId = imageService.upload(bytes, contentType, "cover", hash);
|
||||
return imageService.find(imageId);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to read cover image bytes", e);
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException("SHA-256 algorithm not available", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveContentType(String headerContentType, byte[] fileBytes) {
|
||||
if (nonNull(headerContentType) && headerContentType.startsWith("image/")) {
|
||||
return headerContentType;
|
||||
}
|
||||
|
||||
return tika.detect(fileBytes);
|
||||
}
|
||||
|
||||
private Manga buildManga(ManualMangaIngestRequestDTO dto, Image coverImage) {
|
||||
var builder = Manga.builder()
|
||||
.title(dto.title())
|
||||
.synopsis(dto.synopsis())
|
||||
.status(dto.status())
|
||||
.score(dto.score())
|
||||
.publishedFrom(dto.publishedFrom())
|
||||
.publishedTo(dto.publishedTo())
|
||||
.coverImage(coverImage)
|
||||
.state(MangaState.AVAILABLE)
|
||||
.mangaAuthors(Collections.emptyList())
|
||||
.mangaGenres(Collections.emptyList())
|
||||
.alternativeTitles(Collections.emptyList());
|
||||
|
||||
if (nonNull(dto.chapterCount())) {
|
||||
builder.chapterCount(dto.chapterCount());
|
||||
}
|
||||
|
||||
if (nonNull(dto.adult())) {
|
||||
builder.adult(dto.adult());
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private MangaIngestReview get(Long id) {
|
||||
return mangaIngestReviewRepository
|
||||
.findById(id)
|
||||
|
||||
@ -27,6 +27,20 @@ public class MangaIngestService {
|
||||
log.info(
|
||||
"Ingesting manga with mangaTitle '{}' from provider {}", mangaTitle, contentProviderId);
|
||||
|
||||
try {
|
||||
doIngestManga(contentProviderId, mangaTitle, url);
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Unexpected error ingesting manga '{}' from provider {} at url '{}'",
|
||||
mangaTitle,
|
||||
contentProviderId,
|
||||
url,
|
||||
e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void doIngestManga(long contentProviderId, String mangaTitle, String url) {
|
||||
if (mangaContentProviderRepository.existsByMangaTitleIgnoreCaseAndContentProvider_Id(
|
||||
mangaTitle, contentProviderId)) {
|
||||
log.info(
|
||||
@ -36,12 +50,32 @@ public class MangaIngestService {
|
||||
return;
|
||||
}
|
||||
|
||||
if (mangaIngestReviewRepository.existsByMangaTitleIgnoreCaseAndContentProvider_Id(
|
||||
mangaTitle, contentProviderId)) {
|
||||
log.info(
|
||||
"Manga ingest review already exists for manga with mangaTitle '{}' and provider {}, skipping ingest",
|
||||
mangaTitle,
|
||||
contentProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
var manga = mangaResolutionService.findOrCreateManga(mangaTitle);
|
||||
if (isNull(manga)) {
|
||||
createMangaIngestReview(mangaTitle, url, contentProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (mangaContentProviderRepository
|
||||
.findByManga_IdAndContentProvider_Id(manga.getId(), contentProviderId)
|
||||
.isPresent()) {
|
||||
log.info(
|
||||
"MangaContentProvider already exists for manga '{}' (ID: {}) and provider '{}', skipping ingest",
|
||||
manga.getTitle(),
|
||||
manga.getId(),
|
||||
contentProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
var contentProvider = contentProviderService.find(contentProviderId);
|
||||
|
||||
@ -54,10 +88,11 @@ public class MangaIngestService {
|
||||
.build());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to ingest manga with mangaTitle '{}' from provider '{}'",
|
||||
"Failed to save manga content provider for manga '{}' from provider '{}'",
|
||||
mangaTitle,
|
||||
contentProviderId,
|
||||
e);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(
|
||||
@ -72,15 +107,6 @@ public class MangaIngestService {
|
||||
mangaTitle,
|
||||
contentProviderId);
|
||||
|
||||
if (mangaIngestReviewRepository.existsByMangaTitleIgnoreCaseAndContentProvider_Id(
|
||||
mangaTitle, contentProviderId)) {
|
||||
log.info(
|
||||
"Manga ingest review already exists for manga with mangaTitle '{}' and provider {}, skipping review creation",
|
||||
mangaTitle,
|
||||
contentProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
var contentProvider = contentProviderService.find(contentProviderId);
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package com.magamochi.catalog.service;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
import static java.util.Objects.nonNull;
|
||||
|
||||
import com.magamochi.catalog.model.entity.Manga;
|
||||
@ -9,6 +10,7 @@ import com.magamochi.catalog.queue.producer.MangaUpdateProducer;
|
||||
import java.util.Optional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.dao.DataIntegrityViolationException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@ -24,9 +26,14 @@ public class MangaResolutionService {
|
||||
private final MangaRepository mangaRepository;
|
||||
|
||||
public Manga findOrCreateManga(String searchTitle) {
|
||||
var existingManga = mangaRepository.findByTitleIgnoreCase(searchTitle);
|
||||
if (existingManga.isPresent()) {
|
||||
return existingManga.get();
|
||||
var existingMangas = mangaRepository.findAllByTitleIgnoreCase(searchTitle);
|
||||
if (existingMangas.size() == 1) {
|
||||
return existingMangas.get(0);
|
||||
}
|
||||
if (existingMangas.size() > 1) {
|
||||
log.warn(
|
||||
"Multiple mangas with title '{}', falling through to external ID resolution",
|
||||
searchTitle);
|
||||
}
|
||||
|
||||
var aniListResult = searchMangaOnAniList(searchTitle);
|
||||
@ -116,21 +123,28 @@ public class MangaResolutionService {
|
||||
if (nonNull(aniListId)) {
|
||||
var existingByAniList = mangaRepository.findByAniListId(aniListId);
|
||||
if (existingByAniList.isPresent()) {
|
||||
return existingByAniList.get();
|
||||
return mergeExternalIds(existingByAniList.get(), canonicalTitle, aniListId, malId);
|
||||
}
|
||||
}
|
||||
|
||||
if (nonNull(malId)) {
|
||||
var existingByMalId = mangaRepository.findByMalId(malId);
|
||||
if (existingByMalId.isPresent()) {
|
||||
return existingByMalId.get();
|
||||
return mergeExternalIds(existingByMalId.get(), canonicalTitle, aniListId, malId);
|
||||
}
|
||||
}
|
||||
|
||||
if (nonNull(canonicalTitle)) {
|
||||
var existingByTitle = mangaRepository.findByTitleIgnoreCase(canonicalTitle);
|
||||
if (existingByTitle.isPresent()) {
|
||||
return existingByTitle.get();
|
||||
var existingByTitle = mangaRepository.findAllByTitleIgnoreCase(canonicalTitle);
|
||||
if (existingByTitle.size() == 1) {
|
||||
return mergeExternalIds(existingByTitle.get(0), canonicalTitle, aniListId, malId);
|
||||
}
|
||||
if (existingByTitle.size() > 1) {
|
||||
log.warn(
|
||||
"Multiple mangas with canonical title '{}', merging into first (ID: {})",
|
||||
canonicalTitle,
|
||||
existingByTitle.get(0).getId());
|
||||
return mergeExternalIds(existingByTitle.get(0), canonicalTitle, aniListId, malId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -138,13 +152,86 @@ public class MangaResolutionService {
|
||||
}
|
||||
|
||||
private Manga createAndNotifyManga(String title, Long aniListId, Long malId) {
|
||||
var manga =
|
||||
mangaRepository.save(
|
||||
Manga.builder().title(title).aniListId(aniListId).malId(malId).build());
|
||||
try {
|
||||
var manga =
|
||||
mangaRepository.save(
|
||||
Manga.builder().title(title).aniListId(aniListId).malId(malId).build());
|
||||
|
||||
mangaUpdateProducer.sendMangaUpdateCommand(new MangaUpdateCommand(manga.getId()));
|
||||
if (nonNull(aniListId) || nonNull(malId)) {
|
||||
try {
|
||||
mangaUpdateProducer.sendMangaUpdateCommand(new MangaUpdateCommand(manga.getId()));
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to send manga update command for manga '{}' (ID: {})", title, manga.getId(), e);
|
||||
}
|
||||
}
|
||||
|
||||
return manga;
|
||||
return manga;
|
||||
} catch (DataIntegrityViolationException e) {
|
||||
log.warn(
|
||||
"Race condition while creating manga '{}' (aniListId={}, malId={}), re-querying existing",
|
||||
title,
|
||||
aniListId,
|
||||
malId);
|
||||
|
||||
if (nonNull(aniListId)) {
|
||||
var existing = mangaRepository.findByAniListId(aniListId);
|
||||
if (existing.isPresent()) {
|
||||
return mergeExternalIds(existing.get(), title, aniListId, malId);
|
||||
}
|
||||
}
|
||||
if (nonNull(malId)) {
|
||||
var existing = mangaRepository.findByMalId(malId);
|
||||
if (existing.isPresent()) {
|
||||
return mergeExternalIds(existing.get(), title, aniListId, malId);
|
||||
}
|
||||
}
|
||||
if (nonNull(title)) {
|
||||
var existingByTitle = mangaRepository.findAllByTitleIgnoreCase(title);
|
||||
if (existingByTitle.size() == 1) {
|
||||
return mergeExternalIds(existingByTitle.get(0), title, aniListId, malId);
|
||||
}
|
||||
if (existingByTitle.size() > 1) {
|
||||
log.warn(
|
||||
"Multiple mangas with title '{}', merging into first (ID: {})",
|
||||
title,
|
||||
existingByTitle.get(0).getId());
|
||||
return mergeExternalIds(existingByTitle.get(0), title, aniListId, malId);
|
||||
}
|
||||
}
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private Manga mergeExternalIds(
|
||||
Manga existing, String canonicalTitle, Long aniListId, Long malId) {
|
||||
boolean updated = false;
|
||||
|
||||
if (isNull(existing.getAniListId()) && nonNull(aniListId)) {
|
||||
existing.setAniListId(aniListId);
|
||||
updated = true;
|
||||
}
|
||||
if (isNull(existing.getMalId()) && nonNull(malId)) {
|
||||
existing.setMalId(malId);
|
||||
updated = true;
|
||||
}
|
||||
if (isNull(existing.getTitle()) && nonNull(canonicalTitle)) {
|
||||
existing.setTitle(canonicalTitle);
|
||||
updated = true;
|
||||
}
|
||||
|
||||
if (updated) {
|
||||
log.info(
|
||||
"Merged external IDs into manga '{}' (ID: {}): aniListId={}, malId={}",
|
||||
existing.getTitle(),
|
||||
existing.getId(),
|
||||
existing.getAniListId(),
|
||||
existing.getMalId());
|
||||
return mangaRepository.save(existing);
|
||||
}
|
||||
|
||||
return existing;
|
||||
}
|
||||
|
||||
private record ProviderResult(String title, Long externalId) {}
|
||||
|
||||
@ -7,8 +7,8 @@ import com.magamochi.catalog.model.dto.MangaListDTO;
|
||||
import com.magamochi.catalog.model.dto.MangaListFilterDTO;
|
||||
import com.magamochi.catalog.model.entity.Manga;
|
||||
import com.magamochi.catalog.model.repository.MangaRepository;
|
||||
import com.magamochi.catalog.model.specification.MangaSpecification;
|
||||
import com.magamochi.common.exception.NotFoundException;
|
||||
import com.magamochi.model.specification.MangaSpecification;
|
||||
import com.magamochi.user.service.UserService;
|
||||
import com.magamochi.userinteraction.model.repository.UserFavoriteMangaRepository;
|
||||
import com.magamochi.userinteraction.model.repository.UserMangaFollowRepository;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package com.magamochi.catalog.service;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
import static java.util.Objects.nonNull;
|
||||
|
||||
import com.magamochi.catalog.model.dto.MangaDataDTO;
|
||||
@ -49,6 +50,12 @@ public class MangaUpdateService {
|
||||
log.info("Updating manga with ID {}", mangaId);
|
||||
|
||||
var manga = mangaService.find(mangaId);
|
||||
|
||||
if (isNull(manga.getAniListId()) && isNull(manga.getMalId())) {
|
||||
log.info("Skipping update for manga {}: no external IDs available", mangaId);
|
||||
return;
|
||||
}
|
||||
|
||||
var mangaData = fetchExternalMangaData(manga);
|
||||
applyUpdatesToDatabase(manga, mangaData);
|
||||
|
||||
@ -95,8 +102,7 @@ public class MangaUpdateService {
|
||||
throw (RuntimeException) fetchException;
|
||||
}
|
||||
|
||||
throw new IllegalStateException(
|
||||
"Cannot update manga: No external provider IDs found for Manga " + manga.getId());
|
||||
return null;
|
||||
}
|
||||
|
||||
private void applyUpdatesToDatabase(Manga manga, MangaDataDTO mangaData) {
|
||||
|
||||
@ -8,6 +8,7 @@ import com.magamochi.catalog.client.JikanClient;
|
||||
import com.magamochi.catalog.model.dto.MangaDataDTO;
|
||||
import com.magamochi.catalog.model.enumeration.MangaStatus;
|
||||
import com.magamochi.catalog.util.DoubleUtil;
|
||||
import com.magamochi.common.config.ExternalApiCircuitBreaker;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -20,45 +21,70 @@ import org.springframework.stereotype.Service;
|
||||
public class MyAnimeListService {
|
||||
private final JikanClient jikanClient;
|
||||
private final RateLimiter jikanRateLimiter;
|
||||
private final ExternalApiCircuitBreaker circuitBreaker;
|
||||
|
||||
/// Searches for manga titles on MyAnimeList using the Jikan API and returns a map of title to MAL
|
||||
// ID.
|
||||
public Map<String, Long> searchMangaByTitle(String titleToSearch) {
|
||||
jikanRateLimiter.acquire();
|
||||
var results = jikanClient.mangaSearch(titleToSearch).data();
|
||||
if (results.isEmpty()) {
|
||||
log.warn("No manga found with title {}", titleToSearch);
|
||||
return Map.of();
|
||||
}
|
||||
return circuitBreaker.execute(
|
||||
"myAnimeList",
|
||||
() -> {
|
||||
jikanRateLimiter.acquire();
|
||||
var results = jikanClient.mangaSearch(titleToSearch).data();
|
||||
if (results.isEmpty()) {
|
||||
log.warn("No manga found with title {}", titleToSearch);
|
||||
return Map.of();
|
||||
}
|
||||
|
||||
return results.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
JikanClient.MangaData::title,
|
||||
JikanClient.MangaData::mal_id,
|
||||
(existing, second) -> existing));
|
||||
return results.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
JikanClient.MangaData::title,
|
||||
JikanClient.MangaData::mal_id,
|
||||
(existing, second) -> existing));
|
||||
},
|
||||
throwable -> {
|
||||
log.error("MyAnimeList circuit breaker open for search '{}'", titleToSearch);
|
||||
return Map.of();
|
||||
});
|
||||
}
|
||||
|
||||
public Map<Long, MangaDataDTO> getMangasByTitle(String title) {
|
||||
jikanRateLimiter.acquire();
|
||||
var response = jikanClient.mangaSearch(title).data();
|
||||
return circuitBreaker.execute(
|
||||
"myAnimeList",
|
||||
() -> {
|
||||
jikanRateLimiter.acquire();
|
||||
var response = jikanClient.mangaSearch(title).data();
|
||||
|
||||
return response.stream()
|
||||
.map(this::parseMangaData)
|
||||
.flatMap(mangaDataMap -> mangaDataMap.entrySet().stream())
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
return response.stream()
|
||||
.map(this::parseMangaData)
|
||||
.flatMap(mangaDataMap -> mangaDataMap.entrySet().stream())
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
},
|
||||
throwable -> {
|
||||
log.error("MyAnimeList circuit breaker open for search '{}'", title);
|
||||
return Map.of();
|
||||
});
|
||||
}
|
||||
|
||||
public MangaDataDTO getMangaDataById(Long malId) {
|
||||
jikanRateLimiter.acquire();
|
||||
var response = jikanClient.getMangaById(malId);
|
||||
if (isNull(response) || isNull(response.data())) {
|
||||
log.warn("No manga found with MAL ID {}", malId);
|
||||
return null;
|
||||
}
|
||||
return circuitBreaker.execute(
|
||||
"myAnimeList",
|
||||
() -> {
|
||||
jikanRateLimiter.acquire();
|
||||
var response = jikanClient.getMangaById(malId);
|
||||
if (isNull(response) || isNull(response.data())) {
|
||||
log.warn("No manga found with MAL ID {}", malId);
|
||||
return null;
|
||||
}
|
||||
|
||||
var responseData = response.data();
|
||||
return parseMangaData(responseData).get(malId);
|
||||
var responseData = response.data();
|
||||
return parseMangaData(responseData).get(malId);
|
||||
},
|
||||
throwable -> {
|
||||
log.error("MyAnimeList circuit breaker open for manga ID {}", malId);
|
||||
throw new RuntimeException("MyAnimeList API unavailable");
|
||||
});
|
||||
}
|
||||
|
||||
private Map<Long, MangaDataDTO> parseMangaData(JikanClient.MangaData data) {
|
||||
|
||||
@ -7,4 +7,5 @@ public class ContentProviders {
|
||||
public static final String TAIMU = "Taimu";
|
||||
public static final String MANGA_DEX = "MangaDex";
|
||||
public static final String MANUAL_IMPORT = "Manual Import";
|
||||
public static final String ROLIA_SCAN = "Rolia Scan";
|
||||
}
|
||||
|
||||
@ -0,0 +1,40 @@
|
||||
package com.magamochi.common.config;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Function;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Log4j2
|
||||
@Component
|
||||
public class ExternalApiCircuitBreaker {
|
||||
|
||||
private static final long OPEN_DURATION_SECONDS = 30;
|
||||
|
||||
private final ConcurrentHashMap<String, AtomicReference<Instant>> circuits =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
public <T> T execute(String name, Callable<T> callable, Function<Throwable, T> fallback) {
|
||||
var circuit = circuits.computeIfAbsent(name, k -> new AtomicReference<>(Instant.MIN));
|
||||
var now = Instant.now();
|
||||
var openUntil = circuit.get();
|
||||
|
||||
if (now.isBefore(openUntil)) {
|
||||
log.warn("Circuit '{}' OPEN — failing fast", name);
|
||||
return fallback.apply(null);
|
||||
}
|
||||
|
||||
try {
|
||||
T result = callable.call();
|
||||
circuit.set(Instant.MIN);
|
||||
return result;
|
||||
} catch (Exception e) {
|
||||
circuit.set(now.plusSeconds(OPEN_DURATION_SECONDS));
|
||||
log.error("Circuit '{}' OPEN for {}s: {}", name, OPEN_DURATION_SECONDS, e.getMessage());
|
||||
return fallback.apply(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,11 +1,13 @@
|
||||
package com.magamochi.common.config;
|
||||
|
||||
import com.magamochi.common.model.enumeration.ContentType;
|
||||
import org.springframework.amqp.core.AmqpAdmin;
|
||||
import org.springframework.amqp.core.Binding;
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.amqp.core.TopicExchange;
|
||||
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
|
||||
import org.springframework.amqp.rabbit.core.RabbitAdmin;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.amqp.support.converter.JacksonJsonMessageConverter;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@ -38,6 +40,9 @@ public class RabbitConfig {
|
||||
@Value("${queues.image-fetch}")
|
||||
private String imageFetchQueue;
|
||||
|
||||
@Value("${queues.manga-content-image-delete}")
|
||||
private String mangaContentImageDeleteQueue;
|
||||
|
||||
@Value("${queues.file-import}")
|
||||
private String fileImportQueue;
|
||||
|
||||
@ -47,9 +52,21 @@ public class RabbitConfig {
|
||||
@Value("${routing-key.image-update}")
|
||||
private String imageUpdateRoutingKey;
|
||||
|
||||
@Value("${routing-key.image-delete}")
|
||||
private String imageDeleteRoutingKey;
|
||||
|
||||
@Value("${queues.manga-content-download}")
|
||||
private String mangaContentDownloadQueue;
|
||||
|
||||
@Value("${queues.manga-follow-update-chapter}")
|
||||
private String mangaFollowUpdateChapterQueue;
|
||||
|
||||
@Value("${queues.image-cleanup-s3-delete}")
|
||||
private String imageCleanupS3DeleteQueue;
|
||||
|
||||
@Value("${queues.image-cleanup-db-delete}")
|
||||
private String imageCleanupDbDeleteQueue;
|
||||
|
||||
@Bean
|
||||
public TopicExchange imageUpdatesExchange() {
|
||||
return new TopicExchange(imageUpdatesTopic);
|
||||
@ -57,7 +74,8 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue imageFetchQueue() {
|
||||
return QueueBuilder.nonDurable(imageFetchQueue)
|
||||
return QueueBuilder.durable(imageFetchQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(imageFetchQueue + ".dlq")
|
||||
.build();
|
||||
@ -65,12 +83,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue imageFetchDlq() {
|
||||
return QueueBuilder.nonDurable(imageFetchQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(imageFetchQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaUpdateQueue() {
|
||||
return QueueBuilder.nonDurable(mangaUpdateQueue)
|
||||
return QueueBuilder.durable(mangaUpdateQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaUpdateQueue + ".dlq")
|
||||
.build();
|
||||
@ -78,12 +97,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaUpdateDlq() {
|
||||
return QueueBuilder.nonDurable(mangaUpdateQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaUpdateQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentImageUpdateQueue() {
|
||||
return QueueBuilder.nonDurable(mangaContentImageUpdateQueue)
|
||||
return QueueBuilder.durable(mangaContentImageUpdateQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaContentImageUpdateQueue + ".dlq")
|
||||
.build();
|
||||
@ -91,12 +111,27 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentImageUpdateDlq() {
|
||||
return QueueBuilder.nonDurable(mangaContentImageUpdateQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaContentImageUpdateQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentImageDeleteQueue() {
|
||||
return QueueBuilder.durable(mangaContentImageDeleteQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaContentImageDeleteQueue + ".dlq")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentImageDeleteDlq() {
|
||||
return QueueBuilder.durable(mangaContentImageDeleteQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaCoverUpdateQueue() {
|
||||
return QueueBuilder.nonDurable(mangaCoverUpdateQueue)
|
||||
return QueueBuilder.durable(mangaCoverUpdateQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaCoverUpdateQueue + ".dlq")
|
||||
.build();
|
||||
@ -104,12 +139,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaCoverUpdateDlq() {
|
||||
return QueueBuilder.nonDurable(mangaCoverUpdateQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaCoverUpdateQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue fileImportQueue() {
|
||||
return QueueBuilder.nonDurable(fileImportQueue)
|
||||
return QueueBuilder.durable(fileImportQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(fileImportQueue + ".dlq")
|
||||
.build();
|
||||
@ -117,7 +153,7 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue fileImportDlq() {
|
||||
return QueueBuilder.nonDurable(fileImportQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(fileImportQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ -143,9 +179,22 @@ public class RabbitConfig {
|
||||
null);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Binding bindingMangaContentImageDeleteQueue(
|
||||
Queue mangaContentImageDeleteQueue, TopicExchange imageUpdatesExchange) {
|
||||
return new Binding(
|
||||
mangaContentImageDeleteQueue.getName(),
|
||||
Binding.DestinationType.QUEUE,
|
||||
imageUpdatesExchange.getName(),
|
||||
String.format(
|
||||
imageDeleteRoutingKey + ".%s", ContentType.CONTENT_IMAGE.name().toLowerCase()),
|
||||
null);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentIngestQueue() {
|
||||
return QueueBuilder.nonDurable(mangaContentIngestQueue)
|
||||
return QueueBuilder.durable(mangaContentIngestQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaContentIngestQueue + ".dlq")
|
||||
.build();
|
||||
@ -153,12 +202,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentIngestDlq() {
|
||||
return QueueBuilder.nonDurable(mangaContentIngestQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaContentIngestQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentImageIngestQueue() {
|
||||
return QueueBuilder.nonDurable(mangaContentImageIngestQueue)
|
||||
return QueueBuilder.durable(mangaContentImageIngestQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaContentImageIngestQueue + ".dlq")
|
||||
.build();
|
||||
@ -166,12 +216,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentImageIngestDlq() {
|
||||
return QueueBuilder.nonDurable(mangaContentImageIngestQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaContentImageIngestQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaIngestQueue() {
|
||||
return QueueBuilder.nonDurable(mangaIngestQueue)
|
||||
return QueueBuilder.durable(mangaIngestQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaIngestQueue + ".dlq")
|
||||
.build();
|
||||
@ -179,12 +230,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaIngestDlq() {
|
||||
return QueueBuilder.nonDurable(mangaIngestQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaIngestQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue providerPageIngestQueue() {
|
||||
return QueueBuilder.nonDurable(providerPageIngestQueue)
|
||||
return QueueBuilder.durable(providerPageIngestQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(providerPageIngestQueue + ".dlq")
|
||||
.build();
|
||||
@ -192,12 +244,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue providerPageIngestDlq() {
|
||||
return QueueBuilder.nonDurable(providerPageIngestQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(providerPageIngestQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentDownloadQueue() {
|
||||
return QueueBuilder.nonDurable(mangaContentDownloadQueue)
|
||||
return QueueBuilder.durable(mangaContentDownloadQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaContentDownloadQueue + ".dlq")
|
||||
.build();
|
||||
@ -205,17 +258,13 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaContentDownloadDlq() {
|
||||
return QueueBuilder.nonDurable(mangaContentDownloadQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaContentDownloadQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
// TODO: remove unused queues
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
|
||||
private String mangaFollowUpdateChapterQueue;
|
||||
|
||||
@Bean
|
||||
public Queue mangaFollowUpdateChapterQueue() {
|
||||
return QueueBuilder.nonDurable(mangaFollowUpdateChapterQueue)
|
||||
return QueueBuilder.durable(mangaFollowUpdateChapterQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(mangaFollowUpdateChapterQueue + ".dlq")
|
||||
.build();
|
||||
@ -223,7 +272,40 @@ public class RabbitConfig {
|
||||
|
||||
@Bean
|
||||
public Queue mangaFollowUpdateChapterDlq() {
|
||||
return QueueBuilder.nonDurable(mangaFollowUpdateChapterQueue + ".dlq").build();
|
||||
return QueueBuilder.durable(mangaFollowUpdateChapterQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue imageCleanupS3DeleteQueue() {
|
||||
return QueueBuilder.durable(imageCleanupS3DeleteQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(imageCleanupS3DeleteQueue + ".dlq")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue imageCleanupS3DeleteDlq() {
|
||||
return QueueBuilder.durable(imageCleanupS3DeleteQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue imageCleanupDbDeleteQueue() {
|
||||
return QueueBuilder.durable(imageCleanupDbDeleteQueue)
|
||||
.quorum()
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(imageCleanupDbDeleteQueue + ".dlq")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue imageCleanupDbDeleteDlq() {
|
||||
return QueueBuilder.durable(imageCleanupDbDeleteQueue + ".dlq").quorum().build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public AmqpAdmin amqpAdmin(ConnectionFactory connectionFactory) {
|
||||
return new RabbitAdmin(connectionFactory);
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@ -16,6 +16,16 @@ public class RateLimiterConfig {
|
||||
return RateLimiter.create(1);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RateLimiter roliaScanRateLimiter() {
|
||||
return RateLimiter.create(1);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RateLimiter taimuRateLimiter() {
|
||||
return RateLimiter.create(1);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RateLimiter aniListRateLimiter() {
|
||||
return RateLimiter.create(0.5);
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
package com.magamochi.common.exception;
|
||||
|
||||
public class ExcludedImageException extends RuntimeException {
|
||||
public ExcludedImageException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ExcludedImageException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
package com.magamochi.common.queue.command;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
public record ImageCleanupDbDeleteCommand(Set<UUID> imageIds) {}
|
||||
@ -0,0 +1,5 @@
|
||||
package com.magamochi.common.queue.command;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
public record ImageCleanupS3DeleteCommand(Set<String> objectKeys) {}
|
||||
@ -0,0 +1,3 @@
|
||||
package com.magamochi.common.queue.command;
|
||||
|
||||
public record MangaContentImageDeleteCommand(long entityId) {}
|
||||
@ -6,22 +6,30 @@ import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public record MangaContentImagesDTO(
|
||||
@NotNull Long id,
|
||||
@NotBlank String mangaTitle,
|
||||
Long previousContentId,
|
||||
Long nextContentId,
|
||||
@NotNull List<@NotBlank String> contentImageKeys) {
|
||||
public static MangaContentImagesDTO from(MangaContent mangaContent, Long prevId, Long nextId) {
|
||||
@NotNull List<MangaContentImageDTO> contentImages) {
|
||||
public static MangaContentImagesDTO from(
|
||||
MangaContent mangaContent, List<MangaContentImage> images, Long prevId, Long nextId) {
|
||||
return new MangaContentImagesDTO(
|
||||
mangaContent.getId(),
|
||||
mangaContent.getTitle(),
|
||||
prevId,
|
||||
nextId,
|
||||
mangaContent.getMangaContentImages().stream()
|
||||
images.stream()
|
||||
.sorted(Comparator.comparing(MangaContentImage::getPosition))
|
||||
.map(mangaContentImage -> mangaContentImage.getImage().getObjectKey())
|
||||
.map(
|
||||
mangaContentImage ->
|
||||
new MangaContentImageDTO(
|
||||
mangaContentImage.getImage().getId(),
|
||||
mangaContentImage.getImage().getObjectKey()))
|
||||
.toList());
|
||||
}
|
||||
|
||||
public record MangaContentImageDTO(UUID uuid, String objectKey) {}
|
||||
}
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
package com.magamochi.content.model.repository;
|
||||
|
||||
import com.magamochi.content.model.entity.MangaContent;
|
||||
import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
public interface MangaContentImageRepository extends JpaRepository<MangaContentImage, Long> {
|
||||
List<MangaContentImage> findAllByMangaContent(MangaContent mangaContent);
|
||||
|
||||
boolean existsByMangaContent_IdAndPosition(Long mangaContentId, int position);
|
||||
|
||||
List<MangaContentImage> findAllByImage_Id(UUID imageId);
|
||||
|
||||
List<MangaContentImage> findAllByMangaContent_IdOrderByPositionAsc(Long mangaContentId);
|
||||
}
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
package com.magamochi.content.queue.consumer;
|
||||
|
||||
import com.magamochi.common.queue.command.MangaContentImageDeleteCommand;
|
||||
import com.magamochi.content.service.ContentIngestService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class MangaContentImageDeleteConsumer {
|
||||
private final ContentIngestService contentIngestService;
|
||||
|
||||
@RabbitListener(queues = "${queues.manga-content-image-delete}")
|
||||
public void receiveMangaContentImageDeleteCommand(MangaContentImageDeleteCommand command) {
|
||||
log.info("Received manga content image delete command: {}", command);
|
||||
|
||||
contentIngestService.deleteMangaContentImage(command.entityId());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
package com.magamochi.content.service;
|
||||
|
||||
import com.magamochi.common.exception.UnprocessableException;
|
||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||
import com.magamochi.content.service.strategy.ContentArchiveStrategy;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ContentArchiveStrategyFactory {
|
||||
private final List<ContentArchiveStrategy> strategies;
|
||||
|
||||
public ContentArchiveStrategy getStrategy(ContentArchiveFileType contentArchiveFileType) {
|
||||
return strategies.stream()
|
||||
.filter(strategy -> strategy.supports(contentArchiveFileType))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new UnprocessableException(
|
||||
"Unsupported archive file type: " + contentArchiveFileType.name()));
|
||||
}
|
||||
}
|
||||
@ -1,19 +1,11 @@
|
||||
package com.magamochi.content.service;
|
||||
|
||||
import com.magamochi.common.exception.UnprocessableException;
|
||||
import com.magamochi.content.model.dto.MangaContentArchiveDTO;
|
||||
import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||
import com.magamochi.image.service.ImageService;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import com.magamochi.image.service.ImageExclusionService;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.tomcat.util.http.fileupload.IOUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@ -21,47 +13,15 @@ import org.springframework.stereotype.Service;
|
||||
@RequiredArgsConstructor
|
||||
public class ContentDownloadService {
|
||||
private final ContentService contentService;
|
||||
private final ImageService imageService;
|
||||
private final ImageExclusionService imageExclusionService;
|
||||
private final ContentArchiveStrategyFactory contentArchiveStrategyFactory;
|
||||
|
||||
public MangaContentArchiveDTO downloadContent(
|
||||
Long mangaContentId, ContentArchiveFileType contentArchiveFileType) throws IOException {
|
||||
var chapter = contentService.find(mangaContentId);
|
||||
var chapterImages = chapter.getMangaContentImages();
|
||||
var chapterImages = imageExclusionService.filterExcludedImages(chapter.getMangaContentImages());
|
||||
|
||||
var byteArrayOutputStream =
|
||||
switch (contentArchiveFileType) {
|
||||
case CBZ -> getChapterCbzArchive(chapterImages);
|
||||
default ->
|
||||
throw new UnprocessableException(
|
||||
"Unsupported archive file type: " + contentArchiveFileType.name());
|
||||
};
|
||||
|
||||
return new MangaContentArchiveDTO(
|
||||
chapter.getTitle() + ".cbz", byteArrayOutputStream.toByteArray());
|
||||
}
|
||||
|
||||
private ByteArrayOutputStream getChapterCbzArchive(List<MangaContentImage> chapterImages)
|
||||
throws IOException {
|
||||
var byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
var bufferedOutputStream = new BufferedOutputStream(byteArrayOutputStream);
|
||||
var zipOutputStream = new ZipOutputStream(bufferedOutputStream);
|
||||
|
||||
var totalPages = chapterImages.size();
|
||||
var paddingLength = String.valueOf(totalPages).length();
|
||||
|
||||
for (var pageNumber = 1; pageNumber <= totalPages; pageNumber++) {
|
||||
var imgSrc = chapterImages.get(pageNumber - 1);
|
||||
|
||||
var paddedFileName = String.format("%0" + paddingLength + "d.jpg", imgSrc.getPosition());
|
||||
|
||||
zipOutputStream.putNextEntry(new ZipEntry(paddedFileName));
|
||||
IOUtils.copy(imageService.getStream(imgSrc.getImage()), zipOutputStream);
|
||||
zipOutputStream.closeEntry();
|
||||
}
|
||||
|
||||
zipOutputStream.finish();
|
||||
zipOutputStream.flush();
|
||||
IOUtils.closeQuietly(zipOutputStream);
|
||||
return byteArrayOutputStream;
|
||||
var strategy = contentArchiveStrategyFactory.getStrategy(contentArchiveFileType);
|
||||
return strategy.createArchive(chapter.getTitle(), chapterImages);
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||
import com.magamochi.catalog.model.specification.MangaImportJobSpecification;
|
||||
import com.magamochi.catalog.service.MangaContentProviderService;
|
||||
import com.magamochi.catalog.service.MangaResolutionService;
|
||||
import com.magamochi.common.exception.ExcludedImageException;
|
||||
import com.magamochi.common.exception.NotFoundException;
|
||||
import com.magamochi.common.exception.UnprocessableException;
|
||||
import com.magamochi.common.model.enumeration.ContentType;
|
||||
@ -202,15 +203,25 @@ public class ContentImportService {
|
||||
for (var sortedEntry : entryMap.entrySet()) {
|
||||
var bytes = sortedEntry.getValue();
|
||||
|
||||
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
|
||||
var image = imageService.find(imageId);
|
||||
try {
|
||||
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
|
||||
var image = imageService.find(imageId);
|
||||
|
||||
mangaContentImageRepository.save(
|
||||
MangaContentImage.builder()
|
||||
.image(image)
|
||||
.mangaContent(mangaContent)
|
||||
.position(position++)
|
||||
.build());
|
||||
mangaContentImageRepository.save(
|
||||
MangaContentImage.builder()
|
||||
.image(image)
|
||||
.mangaContent(mangaContent)
|
||||
.position(position)
|
||||
.build());
|
||||
} catch (ExcludedImageException e) {
|
||||
log.info(
|
||||
"Image at position {} in content {} is excluded and will be skipped.",
|
||||
position,
|
||||
mangaContent.getId());
|
||||
} finally {
|
||||
// Ensure we always increment position even if an image is excluded
|
||||
position++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import com.magamochi.content.model.repository.MangaContentImageRepository;
|
||||
import com.magamochi.content.model.repository.MangaContentRepository;
|
||||
import com.magamochi.image.service.ImageService;
|
||||
import com.magamochi.userinteraction.service.UserMangaFollowService;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -28,6 +29,7 @@ public class ContentIngestService {
|
||||
private final ContentService contentService;
|
||||
private final MangaContentProviderService mangaContentProviderService;
|
||||
private final LanguageService languageService;
|
||||
private final UserMangaFollowService userMangaFollowService;
|
||||
|
||||
private final MangaContentRepository mangaContentRepository;
|
||||
private final MangaContentImageRepository mangaContentImageRepository;
|
||||
@ -66,6 +68,8 @@ public class ContentIngestService {
|
||||
.language(language)
|
||||
.build());
|
||||
|
||||
userMangaFollowService.notifyNewMangaContent(mangaContent);
|
||||
|
||||
log.info(
|
||||
"Ingested Manga Content ({}) for provider {}: {}",
|
||||
title,
|
||||
@ -115,4 +119,10 @@ public class ContentIngestService {
|
||||
var image = imageService.find(imageId);
|
||||
mangaContentImage.setImage(image);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteMangaContentImage(long mangaContentImageId) {
|
||||
mangaContentImageRepository.deleteById(mangaContentImageId);
|
||||
log.info("Deleted Manga Content Image entity with ID {}", mangaContentImageId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import com.magamochi.content.model.dto.MangaContentDTO;
|
||||
import com.magamochi.content.model.dto.MangaContentImagesDTO;
|
||||
import com.magamochi.content.model.entity.MangaContent;
|
||||
import com.magamochi.content.model.repository.MangaContentRepository;
|
||||
import com.magamochi.image.service.ImageExclusionService;
|
||||
import com.magamochi.userinteraction.service.UserMangaContentReadService;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import java.util.Comparator;
|
||||
@ -19,6 +20,7 @@ import org.springframework.stereotype.Service;
|
||||
public class ContentService {
|
||||
private final MangaContentProviderService mangaContentProviderService;
|
||||
private final UserMangaContentReadService userMangaContentReadService;
|
||||
private final ImageExclusionService imageExclusionService;
|
||||
|
||||
private final MangaContentRepository mangaContentRepository;
|
||||
|
||||
@ -70,6 +72,9 @@ public class ContentService {
|
||||
}
|
||||
}
|
||||
|
||||
return MangaContentImagesDTO.from(mangaContent, prevId, nextId);
|
||||
var filteredImages =
|
||||
imageExclusionService.filterExcludedImages(mangaContent.getMangaContentImages());
|
||||
|
||||
return MangaContentImagesDTO.from(mangaContent, filteredImages, prevId, nextId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,84 @@
|
||||
package com.magamochi.content.service.strategy;
|
||||
|
||||
import com.magamochi.common.exception.UnprocessableException;
|
||||
import com.magamochi.content.model.dto.MangaContentArchiveDTO;
|
||||
import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||
import com.magamochi.image.service.ImageService;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Log4j2
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class CbrArchiveStrategy implements ContentArchiveStrategy {
|
||||
private final ImageService imageService;
|
||||
|
||||
@Override
|
||||
public boolean supports(ContentArchiveFileType contentArchiveFileType) {
|
||||
return ContentArchiveFileType.CBR.equals(contentArchiveFileType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MangaContentArchiveDTO createArchive(String title, List<MangaContentImage> chapterImages)
|
||||
throws IOException {
|
||||
var tempDir = Files.createTempDirectory("manga_cbr_");
|
||||
|
||||
try {
|
||||
var totalPages = chapterImages.size();
|
||||
var paddingLength = String.valueOf(totalPages).length();
|
||||
|
||||
for (var pageNumber = 1; pageNumber <= totalPages; pageNumber++) {
|
||||
var imgSrc = chapterImages.get(pageNumber - 1);
|
||||
var paddedFileName = String.format("%0" + paddingLength + "d.jpg", imgSrc.getPosition());
|
||||
|
||||
var imagePath = tempDir.resolve(paddedFileName);
|
||||
Files.copy(imageService.getStream(imgSrc.getImage()), imagePath);
|
||||
}
|
||||
|
||||
var archiveName = "archive.cbr";
|
||||
var processBuilder = new ProcessBuilder("rar", "a", "-ep1", archiveName, "*");
|
||||
processBuilder.directory(tempDir.toFile());
|
||||
|
||||
var process = processBuilder.start();
|
||||
var exitCode = process.waitFor();
|
||||
|
||||
if (exitCode != 0) {
|
||||
log.error(
|
||||
"Failed to extract CBR file, rar process exited with code {} for directory {}",
|
||||
exitCode,
|
||||
tempDir);
|
||||
throw new UnprocessableException("Failed to generate CBR archive");
|
||||
}
|
||||
|
||||
var archivePath = tempDir.resolve(archiveName);
|
||||
var archiveBytes = Files.readAllBytes(archivePath);
|
||||
return new MangaContentArchiveDTO(title + ".cbr", archiveBytes);
|
||||
|
||||
} catch (IOException | InterruptedException e) {
|
||||
log.error("Error creating CBR archive", e);
|
||||
if (e instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
|
||||
throw new UnprocessableException("Failed to generate CBR archive");
|
||||
} finally {
|
||||
// Clean up temp dir
|
||||
try {
|
||||
Files.walk(tempDir)
|
||||
.sorted(Comparator.reverseOrder())
|
||||
.map(Path::toFile)
|
||||
.forEach(File::delete);
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to clean up temporary directory: {}", tempDir, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
package com.magamochi.content.service.strategy;
|
||||
|
||||
import com.magamochi.content.model.dto.MangaContentArchiveDTO;
|
||||
import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||
import com.magamochi.image.service.ImageService;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.tomcat.util.http.fileupload.IOUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Log4j2
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class CbzArchiveStrategy implements ContentArchiveStrategy {
|
||||
private final ImageService imageService;
|
||||
|
||||
@Override
|
||||
public boolean supports(ContentArchiveFileType contentArchiveFileType) {
|
||||
return ContentArchiveFileType.CBZ.equals(contentArchiveFileType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MangaContentArchiveDTO createArchive(String title, List<MangaContentImage> chapterImages)
|
||||
throws IOException {
|
||||
var byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
var bufferedOutputStream = new BufferedOutputStream(byteArrayOutputStream);
|
||||
var zipOutputStream = new ZipOutputStream(bufferedOutputStream);
|
||||
|
||||
var totalPages = chapterImages.size();
|
||||
var paddingLength = String.valueOf(totalPages).length();
|
||||
|
||||
for (var pageNumber = 1; pageNumber <= totalPages; pageNumber++) {
|
||||
var imgSrc = chapterImages.get(pageNumber - 1);
|
||||
|
||||
var paddedFileName = String.format("%0" + paddingLength + "d.jpg", imgSrc.getPosition());
|
||||
|
||||
zipOutputStream.putNextEntry(new ZipEntry(paddedFileName));
|
||||
IOUtils.copy(imageService.getStream(imgSrc.getImage()), zipOutputStream);
|
||||
zipOutputStream.closeEntry();
|
||||
}
|
||||
|
||||
zipOutputStream.finish();
|
||||
zipOutputStream.flush();
|
||||
IOUtils.closeQuietly(zipOutputStream);
|
||||
|
||||
return new MangaContentArchiveDTO(title + ".cbz", byteArrayOutputStream.toByteArray());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
package com.magamochi.content.service.strategy;
|
||||
|
||||
import com.magamochi.content.model.dto.MangaContentArchiveDTO;
|
||||
import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
public interface ContentArchiveStrategy {
|
||||
boolean supports(ContentArchiveFileType contentArchiveFileType);
|
||||
|
||||
MangaContentArchiveDTO createArchive(String title, List<MangaContentImage> chapterImages)
|
||||
throws IOException;
|
||||
}
|
||||
@ -2,9 +2,10 @@ package com.magamochi.controller;
|
||||
|
||||
import com.magamochi.client.NtfyClient;
|
||||
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
||||
import com.magamochi.image.service.ImageExclusionService;
|
||||
import com.magamochi.image.task.ImageCleanupTask;
|
||||
import com.magamochi.task.MangaFollowUpdateTask;
|
||||
import com.magamochi.user.repository.UserRepository;
|
||||
import com.magamochi.userinteraction.task.MangaFollowUpdateTask;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
@ -14,6 +15,7 @@ import org.springframework.web.bind.annotation.*;
|
||||
@RequiredArgsConstructor
|
||||
public class ManagementController {
|
||||
private final ImageCleanupTask imageCleanupTask;
|
||||
private final ImageExclusionService imageExclusionService;
|
||||
private final MangaFollowUpdateTask mangaFollowUpdateTask;
|
||||
private final UserRepository userRepository;
|
||||
private final NtfyClient ntfyClient;
|
||||
@ -30,6 +32,18 @@ public class ManagementController {
|
||||
return DefaultResponseDTO.ok().build();
|
||||
}
|
||||
|
||||
@Operation(
|
||||
summary = "Cleanup excluded images",
|
||||
description = "Triggers the cleanup of images that are in the exclusion list",
|
||||
tags = {"Management"},
|
||||
operationId = "imageExclusionCleanup")
|
||||
@PostMapping("image-exclusion-cleanup")
|
||||
public DefaultResponseDTO<Void> imageExclusionCleanup() {
|
||||
imageExclusionService.cleanupExcludedImages();
|
||||
|
||||
return DefaultResponseDTO.ok().build();
|
||||
}
|
||||
|
||||
@Operation(
|
||||
summary = "Trigger user follow update",
|
||||
description = "Trigger user follow update",
|
||||
|
||||
@ -0,0 +1,27 @@
|
||||
package com.magamochi.image.controller;
|
||||
|
||||
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
||||
import com.magamochi.image.service.ImageExclusionService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/management/image-exclusion")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "Image Exclusion", description = "Management API for image exclusion")
|
||||
public class ImageExclusionController {
|
||||
private final ImageExclusionService exclusionService;
|
||||
|
||||
@Operation(
|
||||
summary = "Exclude image by UUID",
|
||||
description =
|
||||
"Gets the hash of the image with the given UUID and adds it to the exclusion list.")
|
||||
@PostMapping("/{uuid}")
|
||||
public DefaultResponseDTO<Void> excludeImage(@PathVariable UUID uuid) {
|
||||
exclusionService.excludeImageByUuid(uuid);
|
||||
return DefaultResponseDTO.ok().build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package com.magamochi.image.model.entity;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import java.time.Instant;
|
||||
import lombok.*;
|
||||
import org.hibernate.annotations.CreationTimestamp;
|
||||
|
||||
@Entity
|
||||
@Table(name = "image_exclusion_list")
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Getter
|
||||
@Setter
|
||||
public class ImageExclusion {
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
private String fileHash;
|
||||
|
||||
@CreationTimestamp private Instant createdAt;
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package com.magamochi.image.model.repository;
|
||||
|
||||
import com.magamochi.image.model.entity.ImageExclusion;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
public interface ImageExclusionRepository extends JpaRepository<ImageExclusion, Long> {
|
||||
boolean existsByFileHash(String fileHash);
|
||||
|
||||
List<ImageExclusion> findAllByFileHashIn(Collection<String> fileHashes);
|
||||
}
|
||||
@ -1,10 +1,17 @@
|
||||
package com.magamochi.image.model.repository;
|
||||
|
||||
import com.magamochi.image.model.entity.Image;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
public interface ImageRepository extends JpaRepository<Image, UUID> {
|
||||
Optional<Image> findByFileHash(String fileHash);
|
||||
|
||||
@Query("SELECT i.objectKey FROM Image i WHERE i.objectKey IN :keys")
|
||||
Set<String> findExistingObjectKeysIn(@Param("keys") Collection<String> keys);
|
||||
}
|
||||
|
||||
@ -0,0 +1,35 @@
|
||||
package com.magamochi.image.queue.consumer;
|
||||
|
||||
import com.magamochi.common.queue.command.ImageCleanupDbDeleteCommand;
|
||||
import com.magamochi.image.service.ImageCleanupService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageCleanupDbDeleteConsumer {
|
||||
private final ImageCleanupService imageCleanupService;
|
||||
|
||||
@RabbitListener(
|
||||
queues = "${queues.image-cleanup-db-delete}",
|
||||
concurrency = "${queues.image-cleanup-db-delete-concurrency:1-3}")
|
||||
public void receive(ImageCleanupDbDeleteCommand command) {
|
||||
var imageIds = command.imageIds();
|
||||
log.info("Processing DB cleanup batch: {} image IDs", imageIds.size());
|
||||
|
||||
try {
|
||||
imageCleanupService.deleteOrphanedImages(imageIds);
|
||||
log.info("DB cleanup batch completed successfully: {} image IDs", imageIds.size());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to process DB cleanup batch ({} image IDs), will be retried. Error: {}",
|
||||
imageIds.size(),
|
||||
e.getMessage(),
|
||||
e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package com.magamochi.image.queue.consumer;
|
||||
|
||||
import com.magamochi.common.queue.command.ImageCleanupS3DeleteCommand;
|
||||
import com.magamochi.image.service.S3Service;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageCleanupS3DeleteConsumer {
|
||||
private final S3Service s3Service;
|
||||
|
||||
@RabbitListener(
|
||||
queues = "${queues.image-cleanup-s3-delete}",
|
||||
concurrency = "${queues.image-cleanup-s3-delete-concurrency:3-5}")
|
||||
public void receive(ImageCleanupS3DeleteCommand command) {
|
||||
var keys = command.objectKeys();
|
||||
log.info("Processing S3 delete batch: {} keys", keys.size());
|
||||
|
||||
try {
|
||||
s3Service.deleteObjects(keys);
|
||||
log.info("S3 delete batch completed successfully: {} keys", keys.size());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to delete S3 batch ({} keys), will be retried. Error: {}",
|
||||
keys.size(),
|
||||
e.getMessage(),
|
||||
e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,10 @@
|
||||
package com.magamochi.image.queue.consumer;
|
||||
|
||||
import com.magamochi.common.exception.ExcludedImageException;
|
||||
import com.magamochi.common.model.enumeration.ContentType;
|
||||
import com.magamochi.common.queue.command.ImageFetchCommand;
|
||||
import com.magamochi.common.queue.command.ImageUpdateCommand;
|
||||
import com.magamochi.common.queue.command.MangaContentImageDeleteCommand;
|
||||
import com.magamochi.image.queue.producer.ImageUpdateProducer;
|
||||
import com.magamochi.image.service.ImageFetchService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -16,7 +19,9 @@ public class ImageFetchConsumer {
|
||||
private final ImageFetchService imageFetchService;
|
||||
private final ImageUpdateProducer imageUpdateProducer;
|
||||
|
||||
@RabbitListener(queues = "${queues.image-fetch}")
|
||||
@RabbitListener(
|
||||
queues = "${queues.image-fetch}",
|
||||
concurrency = "${queues.image-fetch-concurrency:5-10}")
|
||||
public void receiveImageFetchCommand(ImageFetchCommand command) {
|
||||
log.info("Received image fetch command: {}", command);
|
||||
|
||||
@ -25,6 +30,14 @@ public class ImageFetchConsumer {
|
||||
|
||||
imageUpdateProducer.publishImageUpdateCommand(
|
||||
new ImageUpdateCommand(command.entityId(), imageId), command.contentType());
|
||||
} catch (ExcludedImageException e) {
|
||||
log.info("Image at URL {} is excluded from upload, skipping.", command.url());
|
||||
|
||||
if (ContentType.CONTENT_IMAGE.equals(command.contentType())) {
|
||||
log.info("Deleting excluded manga content image entity for ID {}.", command.entityId());
|
||||
imageUpdateProducer.publishImageDeleteCommand(
|
||||
new MangaContentImageDeleteCommand(command.entityId()), command.contentType());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to fetch image from URL.", e);
|
||||
}
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
package com.magamochi.image.queue.producer;
|
||||
|
||||
import com.magamochi.common.queue.command.ImageCleanupDbDeleteCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageCleanupDbDeleteProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${queues.image-cleanup-db-delete}")
|
||||
private String queueName;
|
||||
|
||||
public void send(ImageCleanupDbDeleteCommand command) {
|
||||
log.info("Enqueued DB cleanup batch: {} image IDs", command.imageIds().size());
|
||||
rabbitTemplate.convertAndSend(queueName, command);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package com.magamochi.image.queue.producer;
|
||||
|
||||
import com.magamochi.common.queue.command.ImageCleanupS3DeleteCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageCleanupS3DeleteProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${queues.image-cleanup-s3-delete}")
|
||||
private String queueName;
|
||||
|
||||
public void send(ImageCleanupS3DeleteCommand command) {
|
||||
log.info("Enqueued S3 delete batch: {} keys", command.objectKeys().size());
|
||||
rabbitTemplate.convertAndSend(queueName, command);
|
||||
}
|
||||
}
|
||||
@ -2,6 +2,7 @@ package com.magamochi.image.queue.producer;
|
||||
|
||||
import com.magamochi.common.model.enumeration.ContentType;
|
||||
import com.magamochi.common.queue.command.ImageUpdateCommand;
|
||||
import com.magamochi.common.queue.command.MangaContentImageDeleteCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
@ -20,9 +21,19 @@ public class ImageUpdateProducer {
|
||||
@Value("${routing-key.image-update}")
|
||||
private String imageUpdateRoutingKey;
|
||||
|
||||
@Value("${routing-key.image-delete}")
|
||||
private String imageDeleteRoutingKey;
|
||||
|
||||
public void publishImageUpdateCommand(ImageUpdateCommand command, ContentType contentType) {
|
||||
var routingKey = String.format(imageUpdateRoutingKey + ".%s", contentType.name().toLowerCase());
|
||||
|
||||
rabbitTemplate.convertAndSend(imageUpdatesTopic, routingKey, command);
|
||||
}
|
||||
|
||||
public void publishImageDeleteCommand(
|
||||
MangaContentImageDeleteCommand command, ContentType contentType) {
|
||||
var routingKey = String.format(imageDeleteRoutingKey + ".%s", contentType.name().toLowerCase());
|
||||
|
||||
rabbitTemplate.convertAndSend(imageUpdatesTopic, routingKey, command);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,105 @@
|
||||
package com.magamochi.image.service;
|
||||
|
||||
import com.magamochi.catalog.model.repository.MangaRepository;
|
||||
import com.magamochi.content.model.repository.MangaContentImageRepository;
|
||||
import com.magamochi.image.model.repository.ImageRepository;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageCleanupService {
|
||||
private final ImageRepository imageRepository;
|
||||
private final MangaRepository mangaRepository;
|
||||
private final MangaContentImageRepository mangaContentImageRepository;
|
||||
|
||||
@Transactional
|
||||
public void deleteOrphanedImages(Set<UUID> imageIds) {
|
||||
log.info(
|
||||
"Starting DB cleanup for {} orphaned images (S3 object no longer exists)", imageIds.size());
|
||||
|
||||
int deletedCount = 0;
|
||||
int skippedCount = 0;
|
||||
int errorCount = 0;
|
||||
|
||||
for (var imageId : imageIds) {
|
||||
try {
|
||||
if (deleteOrphanedImage(imageId)) {
|
||||
deletedCount++;
|
||||
} else {
|
||||
skippedCount++;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to delete orphaned image {} from database", imageId, e);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"DB orphan cleanup finished: {} deleted, {} skipped (already gone), {} errors",
|
||||
deletedCount,
|
||||
skippedCount,
|
||||
errorCount);
|
||||
}
|
||||
|
||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||
public boolean deleteOrphanedImage(UUID imageId) {
|
||||
var imageOpt = imageRepository.findById(imageId);
|
||||
if (imageOpt.isEmpty()) {
|
||||
log.info("Image {} already deleted, nothing to do", imageId);
|
||||
return false;
|
||||
}
|
||||
|
||||
var image = imageOpt.get();
|
||||
log.info(
|
||||
"Cleaning up orphaned DB record: imageId={}, objectKey={}, fileHash={}",
|
||||
image.getId(),
|
||||
image.getObjectKey(),
|
||||
image.getFileHash());
|
||||
|
||||
var mangasWithCover = mangaRepository.findAllByCoverImage_Id(imageId);
|
||||
if (!mangasWithCover.isEmpty()) {
|
||||
log.info("Nullifying cover image reference from {} manga(s)", mangasWithCover.size());
|
||||
mangasWithCover.forEach(
|
||||
manga -> {
|
||||
manga.setCoverImage(null);
|
||||
mangaRepository.save(manga);
|
||||
});
|
||||
}
|
||||
|
||||
var contentImages = mangaContentImageRepository.findAllByImage_Id(imageId);
|
||||
if (!contentImages.isEmpty()) {
|
||||
log.info("Removing {} manga content image reference(s) and reindexing", contentImages.size());
|
||||
contentImages.forEach(
|
||||
mci -> {
|
||||
var mangaContentId = mci.getMangaContent().getId();
|
||||
mangaContentImageRepository.delete(mci);
|
||||
reindexMangaContent(mangaContentId);
|
||||
});
|
||||
}
|
||||
|
||||
imageRepository.delete(image);
|
||||
log.info(
|
||||
"Orphaned image record deleted: id={}, objectKey={}", image.getId(), image.getObjectKey());
|
||||
return true;
|
||||
}
|
||||
|
||||
private void reindexMangaContent(Long mangaContentId) {
|
||||
var remainingImages =
|
||||
mangaContentImageRepository.findAllByMangaContent_IdOrderByPositionAsc(mangaContentId);
|
||||
|
||||
for (var i = 0; i < remainingImages.size(); i++) {
|
||||
var img = remainingImages.get(i);
|
||||
if (img.getPosition() != i) {
|
||||
img.setPosition(i);
|
||||
mangaContentImageRepository.save(img);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,143 @@
|
||||
package com.magamochi.image.service;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
|
||||
import com.magamochi.catalog.model.repository.MangaRepository;
|
||||
import com.magamochi.content.model.entity.MangaContentImage;
|
||||
import com.magamochi.content.model.repository.MangaContentImageRepository;
|
||||
import com.magamochi.image.model.entity.ImageExclusion;
|
||||
import com.magamochi.image.model.repository.ImageExclusionRepository;
|
||||
import com.magamochi.image.model.repository.ImageRepository;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageExclusionService {
|
||||
private final ImageService imageService;
|
||||
private final S3Service s3Service;
|
||||
|
||||
private final ImageExclusionRepository exclusionRepository;
|
||||
private final ImageRepository imageRepository;
|
||||
private final MangaRepository mangaRepository;
|
||||
private final MangaContentImageRepository mangaContentImageRepository;
|
||||
|
||||
@Transactional
|
||||
public void excludeImageByUuid(UUID imageId) {
|
||||
var image = imageService.find(imageId);
|
||||
var hash = image.getFileHash();
|
||||
|
||||
if (exclusionRepository.existsByFileHash(hash)) {
|
||||
log.info("Image hash {} (UUID {}) is already in the exclusion list", hash, imageId);
|
||||
return;
|
||||
}
|
||||
|
||||
exclusionRepository.save(ImageExclusion.builder().fileHash(hash).build());
|
||||
log.info("Image hash {} (UUID {}) added to exclusion list", hash, imageId);
|
||||
}
|
||||
|
||||
public Set<String> getExcludedHashes(Collection<String> hashes) {
|
||||
if (isNull(hashes) || hashes.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
|
||||
return exclusionRepository.findAllByFileHashIn(hashes).stream()
|
||||
.map(ImageExclusion::getFileHash)
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public List<MangaContentImage> filterExcludedImages(List<MangaContentImage> images) {
|
||||
var hashes = images.stream().map(img -> img.getImage().getFileHash()).toList();
|
||||
var excludedHashes = getExcludedHashes(hashes);
|
||||
|
||||
return images.stream()
|
||||
.filter(img -> !excludedHashes.contains(img.getImage().getFileHash()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void cleanupExcludedImages() {
|
||||
log.info("Starting cleanup of excluded images...");
|
||||
|
||||
var exclusions = exclusionRepository.findAll();
|
||||
log.info("Found {} excluded hashes to check", exclusions.size());
|
||||
|
||||
var keysToDelete =
|
||||
exclusions.stream()
|
||||
.map(exclusion -> processSingleExclusion(exclusion.getFileHash()))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
if (!keysToDelete.isEmpty()) {
|
||||
log.info("Deleting {} objects from S3", keysToDelete.size());
|
||||
s3Service.deleteObjects(keysToDelete);
|
||||
}
|
||||
|
||||
log.info("Finished cleanup of excluded images.");
|
||||
}
|
||||
|
||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||
public Optional<String> processSingleExclusion(String fileHash) {
|
||||
var imageOpt = imageRepository.findByFileHash(fileHash);
|
||||
if (imageOpt.isEmpty()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
var image = imageOpt.get();
|
||||
var imageId = image.getId();
|
||||
var objectKey = image.getObjectKey();
|
||||
|
||||
log.info("Processing exclusion for image {} (hash: {})", imageId, fileHash);
|
||||
|
||||
// 1. Remove from Manga covers
|
||||
var mangasWithCover = mangaRepository.findAllByCoverImage_Id(imageId);
|
||||
if (!mangasWithCover.isEmpty()) {
|
||||
log.info("Removing image from {} manga covers", mangasWithCover.size());
|
||||
mangasWithCover.forEach(
|
||||
manga -> {
|
||||
manga.setCoverImage(null);
|
||||
mangaRepository.save(manga);
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Remove from Manga content and reindex
|
||||
var contentImages = mangaContentImageRepository.findAllByImage_Id(imageId);
|
||||
if (!contentImages.isEmpty()) {
|
||||
log.info("Removing image from {} manga content entries", contentImages.size());
|
||||
contentImages.forEach(
|
||||
mci -> {
|
||||
var mangaContent = mci.getMangaContent();
|
||||
mangaContentImageRepository.delete(mci);
|
||||
reindexMangaContent(mangaContent.getId());
|
||||
});
|
||||
}
|
||||
|
||||
imageRepository.delete(image);
|
||||
|
||||
return Optional.of(objectKey);
|
||||
}
|
||||
|
||||
private void reindexMangaContent(Long mangaContentId) {
|
||||
var remainingImages =
|
||||
mangaContentImageRepository.findAllByMangaContent_IdOrderByPositionAsc(mangaContentId);
|
||||
|
||||
for (var i = 0; i < remainingImages.size(); i++) {
|
||||
var img = remainingImages.get(i);
|
||||
if (img.getPosition() != i) {
|
||||
img.setPosition(i);
|
||||
mangaContentImageRepository.save(img);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -4,12 +4,12 @@ import static java.util.Objects.nonNull;
|
||||
|
||||
import com.google.common.util.concurrent.RateLimiter;
|
||||
import com.magamochi.common.model.enumeration.ContentType;
|
||||
import com.magamochi.image.util.ImageHashUtil;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -54,7 +54,7 @@ public class ImageFetchService {
|
||||
throws NoSuchAlgorithmException {
|
||||
var fileContentType = resolveContentType(httpResponse, imageBytes);
|
||||
|
||||
var fileHash = computeHash(imageBytes);
|
||||
var fileHash = ImageHashUtil.computeSha256(imageBytes);
|
||||
|
||||
return imageManagerService.upload(
|
||||
imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash);
|
||||
@ -76,22 +76,4 @@ public class ImageFetchService {
|
||||
|
||||
return tika.detect(fileBytes);
|
||||
}
|
||||
|
||||
private String computeHash(byte[] content) throws NoSuchAlgorithmException {
|
||||
var digest = MessageDigest.getInstance("SHA-256");
|
||||
var hashBytes = digest.digest(content);
|
||||
var hexString = new StringBuilder(2 * hashBytes.length);
|
||||
|
||||
for (byte b : hashBytes) {
|
||||
var hex = Integer.toHexString(0xff & b);
|
||||
|
||||
if (hex.length() == 1) {
|
||||
hexString.append('0');
|
||||
}
|
||||
|
||||
hexString.append(hex);
|
||||
}
|
||||
|
||||
return hexString.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
package com.magamochi.image.service;
|
||||
|
||||
import com.magamochi.common.exception.ExcludedImageException;
|
||||
import com.magamochi.common.exception.NotFoundException;
|
||||
import com.magamochi.image.model.entity.Image;
|
||||
import com.magamochi.image.model.repository.ImageExclusionRepository;
|
||||
import com.magamochi.image.model.repository.ImageRepository;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
@ -18,8 +20,15 @@ import org.springframework.stereotype.Service;
|
||||
public class ImageService {
|
||||
private final S3Service s3Service;
|
||||
private final ImageRepository imageRepository;
|
||||
private final ImageExclusionRepository imageExclusionRepository;
|
||||
|
||||
public UUID upload(byte[] data, String contentType, String path, String fileHash) {
|
||||
var isExcluded = imageExclusionRepository.existsByFileHash(fileHash);
|
||||
if (isExcluded) {
|
||||
log.info("Image with hash {} is in the exclusion list, skipping upload", fileHash);
|
||||
throw new ExcludedImageException("Image with hash " + fileHash + " is excluded from upload");
|
||||
}
|
||||
|
||||
var existingImage = imageRepository.findByFileHash(fileHash);
|
||||
if (existingImage.isPresent()) {
|
||||
log.info("Image already exists with hash {}, returning existing ID", fileHash);
|
||||
|
||||
@ -7,6 +7,7 @@ import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
@ -42,9 +43,12 @@ public class S3Service {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public List<String> listAllObjectKeys() {
|
||||
var keys = new ArrayList<String>();
|
||||
public void forEachKeyPage(Consumer<List<String>> pageConsumer) {
|
||||
log.info("Starting S3 key enumeration for bucket: {}", bucket);
|
||||
|
||||
String continuationToken = null;
|
||||
int pageCount = 0;
|
||||
int totalKeys = 0;
|
||||
|
||||
do {
|
||||
var requestBuilder = ListObjectsV2Request.builder().bucket(bucket).maxKeys(1000);
|
||||
@ -55,12 +59,25 @@ public class S3Service {
|
||||
|
||||
var response = s3Client.listObjectsV2(requestBuilder.build());
|
||||
|
||||
response.contents().forEach(s3Object -> keys.add(s3Object.key()));
|
||||
var keys = response.contents().stream().map(S3Object::key).toList();
|
||||
|
||||
pageCount++;
|
||||
totalKeys += keys.size();
|
||||
log.debug("S3 page {}: {} keys (running total: {})", pageCount, keys.size(), totalKeys);
|
||||
|
||||
pageConsumer.accept(keys);
|
||||
|
||||
continuationToken = response.isTruncated() ? response.nextContinuationToken() : null;
|
||||
|
||||
} while (nonNull(continuationToken));
|
||||
|
||||
log.info("S3 key enumeration complete: {} total keys across {} pages", totalKeys, pageCount);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public List<String> listAllObjectKeys() {
|
||||
var keys = new ArrayList<String>();
|
||||
forEachKeyPage(keys::addAll);
|
||||
return keys;
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,28 @@
|
||||
package com.magamochi.image.task;
|
||||
|
||||
import com.magamochi.image.service.ImageExclusionService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Log4j2
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ExclusionCleanupTask {
|
||||
@Value("${image-service.exclusion-cleanup-enabled:true}")
|
||||
private Boolean cleanupEnabled;
|
||||
|
||||
private final ImageExclusionService exclusionService;
|
||||
|
||||
@Scheduled(cron = "${image-service.exclusion-cleanup-cron:@weekly}")
|
||||
public void cleanupExcludedImagesScheduled() {
|
||||
if (!cleanupEnabled) {
|
||||
log.info("Excluded image cleanup is disabled.");
|
||||
return;
|
||||
}
|
||||
|
||||
exclusionService.cleanupExcludedImages();
|
||||
}
|
||||
}
|
||||
@ -1,12 +1,25 @@
|
||||
package com.magamochi.image.task;
|
||||
|
||||
import com.magamochi.image.model.entity.Image;
|
||||
import com.magamochi.image.service.ImageService;
|
||||
import com.google.common.hash.BloomFilter;
|
||||
import com.google.common.hash.Funnels;
|
||||
import com.magamochi.common.queue.command.ImageCleanupDbDeleteCommand;
|
||||
import com.magamochi.common.queue.command.ImageCleanupS3DeleteCommand;
|
||||
import com.magamochi.image.model.repository.ImageRepository;
|
||||
import com.magamochi.image.queue.producer.ImageCleanupDbDeleteProducer;
|
||||
import com.magamochi.image.queue.producer.ImageCleanupS3DeleteProducer;
|
||||
import com.magamochi.image.service.S3Service;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@ -14,41 +27,135 @@ import org.springframework.stereotype.Component;
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ImageCleanupTask {
|
||||
private static final int PAGE_SIZE = 1000;
|
||||
private static final double BLOOM_FILTER_FPP = 0.001;
|
||||
private static final int BLOOM_FILTER_EXPECTED_INSERTIONS = 1_000_000;
|
||||
|
||||
@Value("${image-service.clean-up-enabled}")
|
||||
private Boolean cleanUpEnabled;
|
||||
|
||||
private final S3Service s3Service;
|
||||
private final ImageService imageService;
|
||||
private final ImageRepository imageRepository;
|
||||
private final ImageCleanupS3DeleteProducer s3DeleteProducer;
|
||||
private final ImageCleanupDbDeleteProducer dbDeleteProducer;
|
||||
|
||||
private final AtomicBoolean running = new AtomicBoolean(false);
|
||||
|
||||
@Scheduled(cron = "${image-service.cron-expression}")
|
||||
public void cleanUpImagesScheduled() {
|
||||
if (!cleanUpEnabled) {
|
||||
log.info("S3 Image cleanup disabled.");
|
||||
log.info("Image cleanup disabled (clean-up-enabled=false), skipping.");
|
||||
return;
|
||||
}
|
||||
|
||||
cleanupImages();
|
||||
if (!running.compareAndSet(false, true)) {
|
||||
log.warn("Previous image cleanup run is still in progress, skipping this execution.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
cleanupImages();
|
||||
} finally {
|
||||
running.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
public void cleanupImages() {
|
||||
log.info("Getting unused S3 object keys to remove.");
|
||||
log.info("=== Image cleanup started ===");
|
||||
|
||||
var imageKeys = s3Service.listAllObjectKeys();
|
||||
var bloomFilter =
|
||||
BloomFilter.create(
|
||||
Funnels.stringFunnel(StandardCharsets.UTF_8),
|
||||
BLOOM_FILTER_EXPECTED_INSERTIONS,
|
||||
BLOOM_FILTER_FPP);
|
||||
|
||||
var existingImages =
|
||||
imageService.findAll().parallelStream()
|
||||
.map(Image::getObjectKey)
|
||||
.collect(Collectors.toSet());
|
||||
var s3Pages = new AtomicInteger(0);
|
||||
var totalS3Orphans = new AtomicInteger(0);
|
||||
var s3BatchCount = new AtomicInteger(0);
|
||||
|
||||
var keysToRemove =
|
||||
imageKeys.parallelStream()
|
||||
.filter(imageKey -> !existingImages.contains(imageKey))
|
||||
.collect(Collectors.toSet());
|
||||
log.info("Phase 1 START — scanning S3 for orphaned objects (bucket enumeration + DB lookup)");
|
||||
|
||||
log.info("Removing {} objects from S3 storage", keysToRemove.size());
|
||||
s3Service.forEachKeyPage(
|
||||
page -> {
|
||||
s3Pages.incrementAndGet();
|
||||
|
||||
s3Service.deleteObjects(keysToRemove);
|
||||
var existingKeys = imageRepository.findExistingObjectKeysIn(page);
|
||||
|
||||
log.info("Image cleanup finished.");
|
||||
var orphanKeys =
|
||||
page.stream().filter(key -> !existingKeys.contains(key)).collect(Collectors.toSet());
|
||||
|
||||
if (!orphanKeys.isEmpty()) {
|
||||
s3DeleteProducer.send(new ImageCleanupS3DeleteCommand(orphanKeys));
|
||||
totalS3Orphans.addAndGet(orphanKeys.size());
|
||||
s3BatchCount.incrementAndGet();
|
||||
log.info(
|
||||
"Phase 1 — S3 page {}: {} keys, {} orphans enqueued",
|
||||
s3Pages.get(),
|
||||
page.size(),
|
||||
orphanKeys.size());
|
||||
}
|
||||
|
||||
page.forEach(bloomFilter::put);
|
||||
});
|
||||
|
||||
log.info(
|
||||
"Phase 1 complete: {} S3 pages, {} orphans enqueued across {} batches. BloomFilter FPP={}",
|
||||
s3Pages.get(),
|
||||
totalS3Orphans.get(),
|
||||
s3BatchCount.get(),
|
||||
BLOOM_FILTER_FPP);
|
||||
|
||||
log.info(
|
||||
"Phase 2 START — scanning DB for orphaned records (DB pagination + BloomFilter lookup)");
|
||||
|
||||
int dbPages = 0;
|
||||
int totalDbOrphans = 0;
|
||||
int dbBatchCount = 0;
|
||||
Pageable pageable = PageRequest.of(0, PAGE_SIZE);
|
||||
var imagePage = imageRepository.findAll(pageable);
|
||||
|
||||
while (!imagePage.isEmpty()) {
|
||||
dbPages++;
|
||||
var pageOrphans = new HashSet<UUID>();
|
||||
|
||||
imagePage.forEach(
|
||||
image -> {
|
||||
if (!bloomFilter.mightContain(image.getObjectKey())) {
|
||||
pageOrphans.add(image.getId());
|
||||
}
|
||||
});
|
||||
|
||||
if (!pageOrphans.isEmpty()) {
|
||||
dbDeleteProducer.send(new ImageCleanupDbDeleteCommand(Set.copyOf(pageOrphans)));
|
||||
totalDbOrphans += pageOrphans.size();
|
||||
dbBatchCount++;
|
||||
log.info(
|
||||
"Phase 2 — DB page {}: {} images checked, {} orphans enqueued",
|
||||
dbPages,
|
||||
imagePage.getNumberOfElements(),
|
||||
pageOrphans.size());
|
||||
} else {
|
||||
log.debug(
|
||||
"Phase 2 — DB page {}: {} images checked, no orphans found",
|
||||
dbPages,
|
||||
imagePage.getNumberOfElements());
|
||||
}
|
||||
|
||||
if (!imagePage.hasNext()) {
|
||||
break;
|
||||
}
|
||||
pageable = imagePage.nextPageable();
|
||||
imagePage = imageRepository.findAll(pageable);
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Phase 2 complete: {} DB pages processed, {} orphan DB batches enqueued",
|
||||
dbPages,
|
||||
dbBatchCount);
|
||||
|
||||
log.info(
|
||||
"=== Image cleanup finished. S3 orphans: {}, DB orphans: {} ===",
|
||||
totalS3Orphans.get(),
|
||||
totalDbOrphans);
|
||||
}
|
||||
}
|
||||
|
||||
26
src/main/java/com/magamochi/image/util/ImageHashUtil.java
Normal file
26
src/main/java/com/magamochi/image/util/ImageHashUtil.java
Normal file
@ -0,0 +1,26 @@
|
||||
package com.magamochi.image.util;
|
||||
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@NoArgsConstructor(access = lombok.AccessLevel.PRIVATE)
|
||||
public class ImageHashUtil {
|
||||
public static String computeSha256(byte[] content) throws NoSuchAlgorithmException {
|
||||
var digest = MessageDigest.getInstance("SHA-256");
|
||||
var hashBytes = digest.digest(content);
|
||||
var hexString = new StringBuilder(2 * hashBytes.length);
|
||||
|
||||
for (byte b : hashBytes) {
|
||||
var hex = Integer.toHexString(0xff & b);
|
||||
|
||||
if (hex.length() == 1) {
|
||||
hexString.append('0');
|
||||
}
|
||||
|
||||
hexString.append(hex);
|
||||
}
|
||||
|
||||
return hexString.toString();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,101 @@
|
||||
package com.magamochi.ingestion.client;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.List;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.resilience.annotation.Retryable;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
@FeignClient(name = "roliaScan", url = "https://roliascan.com")
|
||||
public interface RoliaScanClient {
|
||||
|
||||
@PostMapping("/wp-json/manga/v1/load")
|
||||
@Retryable(
|
||||
maxRetries = 3,
|
||||
delay = 1000,
|
||||
multiplier = 2,
|
||||
maxDelay = 5000,
|
||||
includes = Exception.class)
|
||||
List<RoliaScanMangaItem> loadMangas(@RequestBody RoliaScanMangaRequest request);
|
||||
|
||||
@GetMapping("/auth/manga-chapters")
|
||||
@Retryable(
|
||||
maxRetries = 3,
|
||||
delay = 1000,
|
||||
multiplier = 2,
|
||||
maxDelay = 5000,
|
||||
includes = Exception.class)
|
||||
RoliaScanChaptersResponse getChapters(
|
||||
@RequestParam("manga_id") String mangaId,
|
||||
@RequestParam int offset,
|
||||
@RequestParam int limit,
|
||||
@RequestParam String order,
|
||||
@RequestParam("_t") String token,
|
||||
@RequestParam("_ts") long timestamp);
|
||||
|
||||
@GetMapping("/auth/chapter-content")
|
||||
@Retryable(
|
||||
maxRetries = 3,
|
||||
delay = 1000,
|
||||
multiplier = 2,
|
||||
maxDelay = 5000,
|
||||
includes = Exception.class)
|
||||
RoliaScanChapterContentResponse getChapterContent(@RequestParam("chapter_id") String chapterId);
|
||||
|
||||
record RoliaScanMangaRequest(
|
||||
Integer page,
|
||||
String search,
|
||||
String years,
|
||||
String genres,
|
||||
String types,
|
||||
String statuses,
|
||||
String sort,
|
||||
String genreMatchMode) {
|
||||
public RoliaScanMangaRequest(Integer page) {
|
||||
this(page, "", "[]", "[]", "[]", "[]", "post_desc", "any");
|
||||
}
|
||||
}
|
||||
|
||||
record RoliaScanMangaItem(
|
||||
String id,
|
||||
String title,
|
||||
String url,
|
||||
String cover,
|
||||
String score,
|
||||
Integer votes,
|
||||
String status,
|
||||
String year,
|
||||
String type,
|
||||
String description) {}
|
||||
|
||||
record RoliaScanChaptersResponse(
|
||||
boolean success,
|
||||
List<RoliaScanChapterItem> chapters,
|
||||
int total,
|
||||
int offset,
|
||||
int limit,
|
||||
@JsonProperty("has_more") boolean hasMore) {}
|
||||
|
||||
record RoliaScanChapterItem(
|
||||
String id,
|
||||
String chapter,
|
||||
String title,
|
||||
String date,
|
||||
@JsonProperty("chapter_type") String chapterType,
|
||||
@JsonProperty("group_id") String groupId,
|
||||
String language,
|
||||
@JsonProperty("group_name") String groupName,
|
||||
@JsonProperty("group_avatar") String groupAvatar,
|
||||
String likes,
|
||||
String url) {}
|
||||
|
||||
record RoliaScanChapterContentResponse(
|
||||
boolean success,
|
||||
@JsonProperty("chapter_id") String chapterId,
|
||||
@JsonProperty("chapter_type") String chapterType,
|
||||
List<String> images,
|
||||
int total) {}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package com.magamochi.ingestion.client;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
@FeignClient(name = "scrollable-scrapper", url = "${scrollable-scrapper.endpoint}")
|
||||
public interface ScrollableScrapperClient {
|
||||
@PostMapping(
|
||||
consumes = MediaType.APPLICATION_JSON_VALUE,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
GetResponse get(@RequestBody GetRequest request);
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
class GetRequest {
|
||||
private final String url;
|
||||
}
|
||||
|
||||
record GetResponse(String pageSource) {}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
package com.magamochi.ingestion.client;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.List;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.resilience.annotation.Retryable;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
@FeignClient(name = "taimu", url = "https://apiv2.taimumangas.com")
|
||||
public interface TaimuClient {
|
||||
|
||||
@GetMapping("/api/v1/reader/library")
|
||||
@Retryable(
|
||||
maxRetries = 3,
|
||||
delay = 1000,
|
||||
multiplier = 2,
|
||||
maxDelay = 5000,
|
||||
includes = Exception.class)
|
||||
TaimuLibraryResponse getLibrary(
|
||||
@RequestParam int page,
|
||||
@RequestParam("per_page") int perPage,
|
||||
@RequestParam String sort,
|
||||
@RequestParam("adult") boolean adult);
|
||||
|
||||
record TaimuLibraryResponse(
|
||||
List<TaimuMangaItem> items, int total, int page, @JsonProperty("per_page") int perPage) {}
|
||||
|
||||
record TaimuMangaItem(
|
||||
String title,
|
||||
String identifier,
|
||||
String cover,
|
||||
@JsonProperty("chapter_count") int chapterCount,
|
||||
String status,
|
||||
String type,
|
||||
@JsonProperty("release_year") Integer releaseYear) {}
|
||||
|
||||
@GetMapping("/api/v1/reader/series/{seriesId}/chapters")
|
||||
@Retryable(
|
||||
maxRetries = 3,
|
||||
delay = 1000,
|
||||
multiplier = 2,
|
||||
maxDelay = 5000,
|
||||
includes = Exception.class)
|
||||
TaimuChaptersResponse getChapters(
|
||||
@PathVariable String seriesId,
|
||||
@RequestParam int page,
|
||||
@RequestParam("per_page") int perPage,
|
||||
@RequestParam String order);
|
||||
|
||||
record TaimuChaptersResponse(
|
||||
@JsonProperty("has_more") boolean hasMore,
|
||||
List<TaimuChapterItem> items,
|
||||
int page,
|
||||
@JsonProperty("per_page") int perPage) {}
|
||||
|
||||
record TaimuChapterItem(String identifier, double number) {}
|
||||
|
||||
@GetMapping("/api/v1/reader/chapters/{identifier}")
|
||||
@Retryable(
|
||||
maxRetries = 3,
|
||||
delay = 1000,
|
||||
multiplier = 2,
|
||||
maxDelay = 5000,
|
||||
includes = Exception.class)
|
||||
TaimuChapterResponse getChapter(
|
||||
@PathVariable String identifier, @RequestParam("adult") boolean adult);
|
||||
|
||||
record TaimuChapterResponse(List<TaimuPageItem> pages) {}
|
||||
|
||||
record TaimuPageItem(int number, String url) {}
|
||||
}
|
||||
@ -0,0 +1,212 @@
|
||||
package com.magamochi.ingestion.providers.impl;
|
||||
|
||||
import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||
|
||||
import com.google.common.util.concurrent.RateLimiter;
|
||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||
import com.magamochi.common.ContentProviders;
|
||||
import com.magamochi.ingestion.client.RoliaScanClient;
|
||||
import com.magamochi.ingestion.client.RoliaScanClient.RoliaScanMangaRequest;
|
||||
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
||||
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
||||
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
||||
import com.magamochi.ingestion.providers.ContentProvider;
|
||||
import com.magamochi.ingestion.providers.PagedContentProvider;
|
||||
import com.magamochi.ingestion.service.FlareService;
|
||||
import com.magamochi.ingestion.service.RoliaScanAuthService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.stream.IntStream;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service(ContentProviders.ROLIA_SCAN)
|
||||
@RequiredArgsConstructor
|
||||
public class RoliaScanProvider implements ContentProvider, PagedContentProvider {
|
||||
private final FlareService flareService;
|
||||
private final RoliaScanClient roliaScanClient;
|
||||
private final RoliaScanAuthService roliaScanAuthService;
|
||||
private final RateLimiter roliaScanRateLimiter;
|
||||
|
||||
@Override
|
||||
public List<ContentInfoDTO> getAvailableChapters(MangaContentProvider provider) {
|
||||
log.info(
|
||||
"Getting available chapters from {}, manga {}",
|
||||
ContentProviders.ROLIA_SCAN,
|
||||
provider.getManga().getTitle());
|
||||
|
||||
try {
|
||||
var mangaId = extractMangaId(provider.getUrl());
|
||||
|
||||
log.info("Extracted manga id {} from {}", mangaId, provider.getUrl());
|
||||
|
||||
var auth = roliaScanAuthService.generateToken();
|
||||
|
||||
return fetchAllChapters(mangaId, auth.token(), auth.timestamp());
|
||||
} catch (NoSuchElementException e) {
|
||||
log.error("Error parsing chapters from Rolia Scan", e);
|
||||
return List.of();
|
||||
} catch (Exception e) {
|
||||
log.error("Error fetching chapters from Rolia Scan API", e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private String extractMangaId(String mangaUrl) {
|
||||
var document = flareService.getContentAsJsoupDocument(mangaUrl, ContentProviders.ROLIA_SCAN);
|
||||
|
||||
var mangaId = document.body().attr("data-manga-id");
|
||||
|
||||
if (isBlank(mangaId)) {
|
||||
throw new NoSuchElementException("data-manga-id not found in page body");
|
||||
}
|
||||
|
||||
return mangaId;
|
||||
}
|
||||
|
||||
private List<ContentInfoDTO> fetchAllChapters(String mangaId, String token, long timestamp) {
|
||||
log.info("Fetching all chapters for manga {}", mangaId);
|
||||
|
||||
var allChapters = new ArrayList<ContentInfoDTO>();
|
||||
|
||||
int offset = 0;
|
||||
boolean hasMore = true;
|
||||
int page = 0;
|
||||
|
||||
while (hasMore) {
|
||||
page++;
|
||||
|
||||
log.info("Fetching chapters for manga {}, page {}, offset {}", mangaId, page, offset);
|
||||
|
||||
roliaScanRateLimiter.acquire();
|
||||
var response = roliaScanClient.getChapters(mangaId, offset, 500, "DESC", token, timestamp);
|
||||
|
||||
log.info(
|
||||
"Page {} fetched {} chapters for manga {} (offset {}, total {}, hasMore {})",
|
||||
page,
|
||||
response.chapters().size(),
|
||||
mangaId,
|
||||
offset,
|
||||
response.total(),
|
||||
response.hasMore());
|
||||
|
||||
response.chapters().stream().map(this::toContentInfo).forEach(allChapters::add);
|
||||
|
||||
hasMore = response.hasMore();
|
||||
offset += 500;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Fetched total {} chapters for manga {} across {} pages",
|
||||
allChapters.size(),
|
||||
mangaId,
|
||||
page);
|
||||
|
||||
return allChapters;
|
||||
}
|
||||
|
||||
private ContentInfoDTO toContentInfo(RoliaScanClient.RoliaScanChapterItem chapter) {
|
||||
var title =
|
||||
isBlank(chapter.title()) || "N/A".equals(chapter.title())
|
||||
? "Ch. " + chapter.chapter()
|
||||
: chapter.chapter() + " - " + chapter.title();
|
||||
|
||||
return new ContentInfoDTO(title, chapter.url(), mapLanguage(chapter.language()));
|
||||
}
|
||||
|
||||
private static String mapLanguage(String language) {
|
||||
return switch (language) {
|
||||
case "en" -> "en-US";
|
||||
case "pt" -> "pt-BR";
|
||||
default -> language;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ContentImageInfoDTO> getContentImages(String chapterUrl) {
|
||||
log.info("Getting content images from {}, url {}", ContentProviders.ROLIA_SCAN, chapterUrl);
|
||||
|
||||
try {
|
||||
var chapterId = extractChapterId(chapterUrl);
|
||||
|
||||
log.info("Extracted chapter id {} from url {}", chapterId, chapterUrl);
|
||||
|
||||
roliaScanRateLimiter.acquire();
|
||||
var response = roliaScanClient.getChapterContent(chapterId);
|
||||
|
||||
log.info("Fetched {} images for chapter {}", response.images().size(), chapterId);
|
||||
|
||||
return IntStream.range(0, response.images().size())
|
||||
.boxed()
|
||||
.map(position -> new ContentImageInfoDTO(position, response.images().get(position)))
|
||||
.toList();
|
||||
} catch (Exception e) {
|
||||
log.error("Error fetching chapter images from Rolia Scan API, url {}", chapterUrl, e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private String extractChapterId(String chapterUrl) {
|
||||
var lastDash = chapterUrl.lastIndexOf('-');
|
||||
if (lastDash < 0) {
|
||||
throw new NoSuchElementException("Invalid chapter URL format: " + chapterUrl);
|
||||
}
|
||||
return chapterUrl.substring(lastDash + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTotalPages() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<MangaInfoDTO> getMangasFromPage(int page) {
|
||||
log.info("Getting mangas from {}, starting at page {}", ContentProviders.ROLIA_SCAN, page);
|
||||
|
||||
try {
|
||||
var allMangas = new ArrayList<MangaInfoDTO>();
|
||||
|
||||
int currentPage = page;
|
||||
while (true) {
|
||||
log.info("Fetching mangas from {}, page {}", ContentProviders.ROLIA_SCAN, currentPage);
|
||||
|
||||
roliaScanRateLimiter.acquire();
|
||||
var items = roliaScanClient.loadMangas(new RoliaScanMangaRequest(currentPage));
|
||||
|
||||
var filteredItems =
|
||||
items.stream().filter(item -> !"Novel".equalsIgnoreCase(item.type())).toList();
|
||||
|
||||
log.info(
|
||||
"Page {} returned {} mangas ({} novels filtered out)",
|
||||
currentPage,
|
||||
filteredItems.size(),
|
||||
items.size() - filteredItems.size());
|
||||
|
||||
if (filteredItems.isEmpty()) {
|
||||
log.info("No more mangas on page {}, stopping pagination", currentPage);
|
||||
break;
|
||||
}
|
||||
|
||||
filteredItems.stream()
|
||||
.map(item -> new MangaInfoDTO(item.title(), item.url()))
|
||||
.forEach(allMangas::add);
|
||||
|
||||
currentPage++;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Loaded {} mangas from {} across {} pages",
|
||||
allMangas.size(),
|
||||
ContentProviders.ROLIA_SCAN,
|
||||
currentPage - page);
|
||||
|
||||
return allMangas;
|
||||
} catch (Exception e) {
|
||||
log.error("Error loading mangas from Rolia Scan API, page {}", page, e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,34 +1,29 @@
|
||||
package com.magamochi.ingestion.providers.impl;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
|
||||
import com.google.common.util.concurrent.RateLimiter;
|
||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||
import com.magamochi.common.ContentProviders;
|
||||
import com.magamochi.common.exception.UnprocessableException;
|
||||
import com.magamochi.ingestion.client.TaimuClient;
|
||||
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
||||
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
||||
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
||||
import com.magamochi.ingestion.providers.ContentProvider;
|
||||
import com.magamochi.ingestion.providers.PagedContentProvider;
|
||||
import com.magamochi.ingestion.service.FlareService;
|
||||
import com.magamochi.ingestion.service.ScrollableScrapperService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.stream.IntStream;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service(ContentProviders.TAIMU)
|
||||
@RequiredArgsConstructor
|
||||
public class TaimuProvider implements ContentProvider, PagedContentProvider {
|
||||
private final String baseUrl = "https://taimumangas.rzword.xyz";
|
||||
private final String baseUrl = "https://beta.taimumangas.com";
|
||||
|
||||
private final FlareService flareService;
|
||||
private final ScrollableScrapperService scrollableScrapperService;
|
||||
private final TaimuClient taimuClient;
|
||||
private final RateLimiter taimuRateLimiter;
|
||||
|
||||
@Override
|
||||
public List<ContentInfoDTO> getAvailableChapters(MangaContentProvider provider) {
|
||||
@ -38,59 +33,45 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
|
||||
provider.getManga().getTitle());
|
||||
|
||||
try {
|
||||
var document =
|
||||
flareService.getContentAsJsoupDocument(
|
||||
provider.getUrl() + "?page=1&order=desc", ContentProviders.TAIMU);
|
||||
var seriesId = extractSeriesId(provider.getUrl());
|
||||
var allChapters = new ArrayList<ContentInfoDTO>();
|
||||
|
||||
var totalPages = extractContentPagesFromDocument(document);
|
||||
var contentInfoList = new ArrayList<>(extractContentInfoFromDocument(document));
|
||||
int page = 1;
|
||||
boolean hasMore = true;
|
||||
|
||||
for (int page = 2; page <= totalPages; page++) {
|
||||
var pageDocument =
|
||||
flareService.getContentAsJsoupDocument(
|
||||
provider.getUrl() + "?page=" + page + "&order=desc", ContentProviders.TAIMU);
|
||||
contentInfoList.addAll(extractContentInfoFromDocument(pageDocument));
|
||||
while (hasMore) {
|
||||
taimuRateLimiter.acquire();
|
||||
var response = taimuClient.getChapters(seriesId, page, 30, "desc");
|
||||
|
||||
response.items().stream()
|
||||
.map(
|
||||
item -> {
|
||||
var number = item.number();
|
||||
var formatted =
|
||||
number % 1 == 0 ? String.valueOf((long) number) : String.valueOf(number);
|
||||
return new ContentInfoDTO(
|
||||
"Capítulo " + formatted, baseUrl + "/reader/" + item.identifier(), "pt-BR");
|
||||
})
|
||||
.forEach(allChapters::add);
|
||||
|
||||
hasMore = response.hasMore();
|
||||
page++;
|
||||
}
|
||||
|
||||
return contentInfoList;
|
||||
} catch (NoSuchElementException e) {
|
||||
log.error("Error parsing mangas from MangaLivre", e);
|
||||
log.info("Fetched {} chapters for series {}", allChapters.size(), seriesId);
|
||||
return allChapters;
|
||||
} catch (Exception e) {
|
||||
log.error("Error parsing chapters from " + ContentProviders.TAIMU, e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private int extractContentPagesFromDocument(Document document) {
|
||||
try {
|
||||
var buttonsContainer =
|
||||
document.selectFirst(
|
||||
"div.flex.items-center.justify-between.gap-3.mt-4.pt-4.border-t div.flex.items-center.gap-1");
|
||||
var highNumberButton = buttonsContainer.selectFirst("button:nth-last-child(2)");
|
||||
return Integer.parseInt(highNumberButton.text());
|
||||
} catch (Exception e) {
|
||||
// In case of any error during parsing, we assume there is only one page of content
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
private List<ContentInfoDTO> extractContentInfoFromDocument(Document document) {
|
||||
try {
|
||||
var grid = document.selectFirst("div.grid.grid-cols-1.gap-1");
|
||||
var chapters = grid.select("a");
|
||||
|
||||
return chapters.stream()
|
||||
.map(
|
||||
chapter -> {
|
||||
var chapterUrl = baseUrl + chapter.attr("href");
|
||||
var title =
|
||||
chapter.selectFirst("div.flex-1.min-w-0 div.flex.items-center.gap-2 p").text();
|
||||
|
||||
return new ContentInfoDTO(title, chapterUrl.trim(), "pt-BR");
|
||||
})
|
||||
.toList();
|
||||
} catch (Exception e) {
|
||||
log.error("Error parsing content info from " + ContentProviders.TAIMU, e);
|
||||
return List.of();
|
||||
private String extractSeriesId(String url) {
|
||||
var lastSlash = url.lastIndexOf('/');
|
||||
if (lastSlash < 0) {
|
||||
throw new NoSuchElementException("Invalid series URL: " + url);
|
||||
}
|
||||
return url.substring(lastSlash + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -98,20 +79,14 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
|
||||
log.info("Getting images from {}, url {}", ContentProviders.TAIMU, chapterUrl);
|
||||
|
||||
try {
|
||||
var document = scrollableScrapperService.getContentAsJsoupDocument(chapterUrl);
|
||||
var identifier = extractSeriesId(chapterUrl);
|
||||
taimuRateLimiter.acquire();
|
||||
var response = taimuClient.getChapter(identifier, true);
|
||||
|
||||
var chapterImages = document.select("img.w-full.h-auto.object-contain.cursor-pointer");
|
||||
|
||||
var imageUrls =
|
||||
chapterImages.stream()
|
||||
.map(chapterImagesElement -> chapterImagesElement.attr("src"))
|
||||
.toList();
|
||||
|
||||
return IntStream.range(0, imageUrls.size())
|
||||
.boxed()
|
||||
.map(position -> new ContentImageInfoDTO(position, imageUrls.get(position)))
|
||||
return response.pages().stream()
|
||||
.map(page -> new ContentImageInfoDTO(page.number() - 1, page.url()))
|
||||
.toList();
|
||||
} catch (NoSuchElementException e) {
|
||||
} catch (Exception e) {
|
||||
log.error("Error parsing manga images from " + ContentProviders.TAIMU, e);
|
||||
return List.of();
|
||||
}
|
||||
@ -122,22 +97,13 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
|
||||
log.info("Getting mangas from {}, page {}", ContentProviders.TAIMU, page);
|
||||
|
||||
try {
|
||||
var document =
|
||||
flareService.getContentAsJsoupDocument(
|
||||
baseUrl + "/biblioteca?page=" + page, ContentProviders.TAIMU);
|
||||
taimuRateLimiter.acquire();
|
||||
var response = taimuClient.getLibrary(page, 24, "added", true);
|
||||
|
||||
var mangas = document.select("a.group");
|
||||
|
||||
return mangas.stream()
|
||||
.map(
|
||||
element -> {
|
||||
var mangaUrl = element.attr("href");
|
||||
var title = element.selectFirst("div h3").text();
|
||||
|
||||
return new MangaInfoDTO(title.trim(), baseUrl + mangaUrl.trim());
|
||||
})
|
||||
return response.items().stream()
|
||||
.map(item -> new MangaInfoDTO(item.title(), baseUrl + "/series/" + item.identifier()))
|
||||
.toList();
|
||||
} catch (NoSuchElementException e) {
|
||||
} catch (Exception e) {
|
||||
log.error("Error parsing mangas from " + ContentProviders.TAIMU, e);
|
||||
return List.of();
|
||||
}
|
||||
@ -148,27 +114,11 @@ public class TaimuProvider implements ContentProvider, PagedContentProvider {
|
||||
log.info("Getting total pages for {}", ContentProviders.TAIMU);
|
||||
|
||||
try {
|
||||
var document =
|
||||
flareService.getContentAsJsoupDocument(baseUrl + "/biblioteca", ContentProviders.TAIMU);
|
||||
|
||||
var container = document.selectFirst("div.flex.flex-col.items-center.space-y-3.mt-6");
|
||||
var pagination = container.selectFirst("div.flex.items-center.gap-2");
|
||||
var buttonsContainer = pagination.selectFirst("div.flex.items-center.gap-1");
|
||||
|
||||
var lastButton = buttonsContainer.select("button").last();
|
||||
|
||||
if (isNull(lastButton)) {
|
||||
throw new UnprocessableException(
|
||||
"Pagination buttons not found in " + ContentProviders.TAIMU);
|
||||
}
|
||||
|
||||
var buttonText = lastButton.text();
|
||||
return Integer.parseInt(buttonText);
|
||||
taimuRateLimiter.acquire();
|
||||
var response = taimuClient.getLibrary(1, 24, "added", true);
|
||||
return (int) Math.ceil((double) response.total() / response.perPage());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Error parsing total pages from "
|
||||
+ ContentProviders.TAIMU
|
||||
+ ": pagination container not found");
|
||||
log.error("Error parsing total pages from " + ContentProviders.TAIMU, e);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,7 +13,9 @@ import org.springframework.stereotype.Service;
|
||||
public class ProviderPageIngestConsumer {
|
||||
private final IngestionService ingestionService;
|
||||
|
||||
@RabbitListener(queues = "${queues.provider-page-ingest}")
|
||||
@RabbitListener(
|
||||
queues = "${queues.provider-page-ingest}",
|
||||
concurrency = "${queues.provider-page-ingest-concurrency:2-4}")
|
||||
public void receiveProviderPageIngestCommand(ProviderPageIngestCommand command) {
|
||||
log.info("Received provider page ingest command: {}", command);
|
||||
ingestionService.fetchProviderPageMangas(command.providerId(), command.page());
|
||||
|
||||
@ -0,0 +1,42 @@
|
||||
package com.magamochi.ingestion.service;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class RoliaScanAuthService {
|
||||
|
||||
public AuthToken generateToken() {
|
||||
var timestamp = Instant.now().getEpochSecond();
|
||||
var hour = getCurrentHourUtc();
|
||||
var secret = "mng_ch_" + hour;
|
||||
var hash = md5(timestamp + secret).substring(0, 16);
|
||||
return new AuthToken(hash, timestamp);
|
||||
}
|
||||
|
||||
private String getCurrentHourUtc() {
|
||||
return LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyyMMddHH"));
|
||||
}
|
||||
|
||||
private String md5(String input) {
|
||||
try {
|
||||
var digest = MessageDigest.getInstance("MD5");
|
||||
var hashBytes = digest.digest(input.getBytes(StandardCharsets.UTF_8));
|
||||
var hexString = new StringBuilder();
|
||||
for (byte b : hashBytes) {
|
||||
hexString.append(String.format("%02x", b));
|
||||
}
|
||||
return hexString.toString();
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new IllegalStateException("MD5 algorithm not available", e);
|
||||
}
|
||||
}
|
||||
|
||||
public record AuthToken(String token, long timestamp) {}
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
package com.magamochi.ingestion.service;
|
||||
|
||||
import com.magamochi.ingestion.client.ScrollableScrapperClient;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ScrollableScrapperService {
|
||||
private final ScrollableScrapperClient client;
|
||||
|
||||
public Document getContentAsJsoupDocument(String url) {
|
||||
return Jsoup.parse(getContent(url));
|
||||
}
|
||||
|
||||
private String getContent(String url) {
|
||||
|
||||
return client.get(ScrollableScrapperClient.GetRequest.builder().url(url).build()).pageSource();
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,7 @@
|
||||
package com.magamochi.ingestion.task;
|
||||
|
||||
import static java.util.Objects.isNull;
|
||||
|
||||
import com.magamochi.ingestion.client.FlareClient;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
@ -17,6 +19,11 @@ public class FlareStartupCleanupTask {
|
||||
public void cleanupExistingSessions() {
|
||||
var sessions = client.listSessions(FlareClient.SessionListRequest.builder().build()).sessions();
|
||||
|
||||
if (isNull(sessions)) {
|
||||
log.info("No sessions found.");
|
||||
return;
|
||||
}
|
||||
|
||||
for (var sessionId : sessions) {
|
||||
client.destroySession(FlareClient.SessionDestroyRequest.builder().session(sessionId).build());
|
||||
}
|
||||
|
||||
@ -1,45 +0,0 @@
|
||||
package com.magamochi.service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Log4j2
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class OldMangaService {
|
||||
// public void fetchFollowedMangaChapters(Long mangaProviderId) {
|
||||
// var mangaProvider =
|
||||
// mangaContentProviderRepository
|
||||
// .findById(mangaProviderId)
|
||||
// .orElseThrow(
|
||||
// () -> new NotFoundException("Manga Provider not found for ID: " +
|
||||
// mangaProviderId));
|
||||
//
|
||||
// var currentAvailableChapterCount =
|
||||
// mangaChapterRepository.findByMangaContentProviderId(mangaProviderId).size();
|
||||
//
|
||||
// fetchMangaChapters(mangaProviderId);
|
||||
// mangaChapterRepository.flush();
|
||||
//
|
||||
// var availableChapterCount =
|
||||
// mangaChapterRepository.findByMangaContentProviderId(mangaProviderId).size();
|
||||
//
|
||||
// if (availableChapterCount <= currentAvailableChapterCount) {
|
||||
// return;
|
||||
// }
|
||||
//
|
||||
// log.info("New chapters found for Manga Provider {}", mangaProviderId);
|
||||
//
|
||||
// var userMangaFollows = userMangaFollowRepository.findByManga(mangaProvider.getManga());
|
||||
// userMangaFollows.forEach(
|
||||
// umf ->
|
||||
// ntfyClient.notify(
|
||||
// new NtfyClient.Request(
|
||||
// "mangamochi-" + umf.getUser().getId().toString(),
|
||||
// umf.getManga().getTitle(),
|
||||
// "New chapter available on " +
|
||||
// mangaProvider.getContentProvider().getName())));
|
||||
// }
|
||||
|
||||
}
|
||||
@ -1,3 +1,3 @@
|
||||
package com.magamochi.model.dto;
|
||||
package com.magamochi.userinteraction.queue.command;
|
||||
|
||||
public record UpdateMangaFollowChapterListCommand(Long mangaProviderId) {}
|
||||
@ -1,7 +1,7 @@
|
||||
package com.magamochi.queue;
|
||||
package com.magamochi.userinteraction.queue.consumer;
|
||||
|
||||
import com.magamochi.model.dto.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.service.OldMangaService;
|
||||
import com.magamochi.userinteraction.queue.command.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.userinteraction.service.UserMangaFollowService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
@ -11,11 +11,11 @@ import org.springframework.stereotype.Service;
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class UpdateMangaFollowChapterListConsumer {
|
||||
private final OldMangaService oldMangaService;
|
||||
private final UserMangaFollowService userMangaFollowService;
|
||||
|
||||
@RabbitListener(queues = "${rabbit-mq.queues.manga-follow-update-chapter}")
|
||||
@RabbitListener(queues = "${queues.manga-follow-update-chapter}")
|
||||
public void receiveMangaFollowUpdateChapterCommand(UpdateMangaFollowChapterListCommand command) {
|
||||
log.info("Received update followed manga chapter list command: {}", command);
|
||||
// oldMangaService.fetchFollowedMangaChapters(command.mangaProviderId());
|
||||
userMangaFollowService.checkForMangaUpdates(command.mangaProviderId());
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
package com.magamochi.queue;
|
||||
package com.magamochi.userinteraction.queue.producer;
|
||||
|
||||
import com.magamochi.model.dto.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.userinteraction.queue.command.UpdateMangaFollowChapterListCommand;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
@ -13,7 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
public class UpdateMangaFollowChapterListProducer {
|
||||
private final RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Value("${rabbit-mq.queues.manga-follow-update-chapter}")
|
||||
@Value("${queues.manga-follow-update-chapter}")
|
||||
private String mangaFollowUpdateChapterQueue;
|
||||
|
||||
public void sendUpdateMangaFollowChapterListCommand(UpdateMangaFollowChapterListCommand command) {
|
||||
@ -2,7 +2,10 @@ package com.magamochi.userinteraction.service;
|
||||
|
||||
import com.magamochi.catalog.model.entity.Manga;
|
||||
import com.magamochi.catalog.model.repository.MangaRepository;
|
||||
import com.magamochi.client.NtfyClient;
|
||||
import com.magamochi.common.exception.NotFoundException;
|
||||
import com.magamochi.content.model.entity.MangaContent;
|
||||
import com.magamochi.ingestion.service.IngestionService;
|
||||
import com.magamochi.user.service.UserService;
|
||||
import com.magamochi.userinteraction.model.entity.UserMangaFollow;
|
||||
import com.magamochi.userinteraction.model.repository.UserMangaFollowRepository;
|
||||
@ -14,10 +17,13 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
@RequiredArgsConstructor
|
||||
public class UserMangaFollowService {
|
||||
private final UserService userService;
|
||||
private final IngestionService ingestionService;
|
||||
|
||||
private final MangaRepository mangaRepository;
|
||||
private final UserMangaFollowRepository userMangaFollowRepository;
|
||||
|
||||
private final NtfyClient ntfyClient;
|
||||
|
||||
@Transactional
|
||||
public void follow(Long mangaId) {
|
||||
var user = userService.getLoggedUserThrowIfNotFound();
|
||||
@ -50,4 +56,21 @@ public class UserMangaFollowService {
|
||||
.findById(mangaId)
|
||||
.orElseThrow(() -> new NotFoundException("Manga not found for ID: " + mangaId));
|
||||
}
|
||||
|
||||
public void checkForMangaUpdates(Long contentProviderId) {
|
||||
ingestionService.fetchMangaContentProviderContentList(contentProviderId);
|
||||
}
|
||||
|
||||
public void notifyNewMangaContent(MangaContent mangaContent) {
|
||||
var userMangaFollows =
|
||||
userMangaFollowRepository.findByManga(mangaContent.getMangaContentProvider().getManga());
|
||||
userMangaFollows.forEach(
|
||||
umf ->
|
||||
ntfyClient.notify(
|
||||
new NtfyClient.Request(
|
||||
"mangamochi-" + umf.getUser().getId().toString(),
|
||||
umf.getManga().getTitle(),
|
||||
"New content available on "
|
||||
+ mangaContent.getMangaContentProvider().getContentProvider().getName())));
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,12 +1,13 @@
|
||||
package com.magamochi.task;
|
||||
package com.magamochi.userinteraction.task;
|
||||
|
||||
import com.magamochi.catalog.model.entity.Manga;
|
||||
import com.magamochi.catalog.model.repository.MangaRepository;
|
||||
import com.magamochi.model.dto.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.queue.UpdateMangaFollowChapterListProducer;
|
||||
import com.magamochi.userinteraction.queue.command.UpdateMangaFollowChapterListCommand;
|
||||
import com.magamochi.userinteraction.queue.producer.UpdateMangaFollowChapterListProducer;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@ -21,7 +22,7 @@ public class MangaFollowUpdateTask {
|
||||
|
||||
private final UpdateMangaFollowChapterListProducer producer;
|
||||
|
||||
// @Scheduled(cron = "${manga-follow.cron-expression}")
|
||||
@Scheduled(cron = "${manga-follow.cron-expression}")
|
||||
@Transactional
|
||||
public void updateMangaFollow() {
|
||||
if (!updateEnabled) {
|
||||
@ -0,0 +1,15 @@
|
||||
package db.migration;
|
||||
|
||||
import java.sql.Statement;
|
||||
import org.flywaydb.core.api.migration.BaseJavaMigration;
|
||||
import org.flywaydb.core.api.migration.Context;
|
||||
|
||||
public class V0017__AddIndexImagesObjectKey extends BaseJavaMigration {
|
||||
|
||||
@Override
|
||||
public void migrate(Context context) throws Exception {
|
||||
try (Statement stmt = context.getConnection().createStatement()) {
|
||||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_object_key ON images(object_key)");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -22,9 +22,9 @@ spring:
|
||||
openfeign:
|
||||
client:
|
||||
config:
|
||||
scrollable-scrapper:
|
||||
connect-timeout: 480000
|
||||
read-timeout: 480000
|
||||
default:
|
||||
connect-timeout: 10000
|
||||
read-timeout: 30000
|
||||
rabbitmq:
|
||||
host: ${RABBITMQ_HOST}
|
||||
port: ${RABBITMQ_PORT}
|
||||
@ -33,6 +33,7 @@ spring:
|
||||
listener:
|
||||
simple:
|
||||
default-requeue-rejected: false
|
||||
prefetch: 10
|
||||
data:
|
||||
redis:
|
||||
host: ${REDIS_HOST:localhost}
|
||||
@ -47,9 +48,6 @@ springdoc:
|
||||
flare-solverr:
|
||||
endpoint: ${FLARESOLVERR_ENDPOINT}
|
||||
|
||||
scrollable-scrapper:
|
||||
endpoint: ${SCROLLABLE_SCRAPPER_ENDPOINT}
|
||||
|
||||
minio:
|
||||
endpoint: ${MINIO_ENDPOINT}
|
||||
accessKey: ${MINIO_USER}
|
||||
@ -73,26 +71,33 @@ topics:
|
||||
|
||||
queues:
|
||||
manga-ingest: ${MANGA_INGEST_QUEUE:mangamochi.manga.ingest}
|
||||
manga-ingest-concurrency: 3-5
|
||||
manga-update: ${MANGA_UPDATE_QUEUE:mangamochi.manga.update}
|
||||
manga-content-ingest: ${MANGA_CONTENT_INGEST_QUEUE:mangamochi.manga.content.ingest}
|
||||
manga-content-image-ingest: ${MANGA_CONTENT_IMAGE_INGEST_QUEUE:mangamochi.manga.content.image.ingest}
|
||||
manga-content-image-update: ${MANGA_CONTENT_IMAGE_UPDATE_QUEUE:mangamochi.manga.content.image.update}
|
||||
manga-content-image-delete: ${MANGA_CONTENT_IMAGE_DELETE_QUEUE:mangamochi.manga.content.image.delete}
|
||||
provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:mangamochi.provider.page.ingest}
|
||||
provider-page-ingest-concurrency: 2-4
|
||||
image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch}
|
||||
manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update}
|
||||
file-import: ${FILE_IMPORT_QUEUE:mangamochi.file.import}
|
||||
manga-content-download: ${MANGA_CONTENT_DOWNLOAD_QUEUE:mangamochi.manga.content.download}
|
||||
manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangamochi.follow.update}
|
||||
image-cleanup-s3-delete: ${IMAGE_CLEANUP_S3_DELETE_QUEUE:mangamochi.image.cleanup.s3.delete}
|
||||
image-cleanup-db-delete: ${IMAGE_CLEANUP_DB_DELETE_QUEUE:mangamochi.image.cleanup.db.delete}
|
||||
image-cleanup-s3-delete-concurrency: 3-5
|
||||
image-cleanup-db-delete-concurrency: 1-3
|
||||
|
||||
routing-key:
|
||||
image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update}
|
||||
|
||||
rabbit-mq:
|
||||
queues:
|
||||
manga-follow-update-chapter: ${MANGA_FOLLOW_UPDATE_CHAPTER_QUEUE:mangaFollowUpdateChapterQueue}
|
||||
image-delete: ${IMAGE_DELETE_ROUTING_KEY:mangamochi.image.delete}
|
||||
|
||||
image-service:
|
||||
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
|
||||
cron-expression: "@weekly"
|
||||
exclusion-cleanup-enabled: ${IMAGE_SERVICE_EXCLUSION_CLEANUP_ENABLED:true}
|
||||
exclusion-cleanup-cron: "@weekly"
|
||||
|
||||
content-providers:
|
||||
update-enabled: ${CONTENT_PROVIDER_UPDATE_ENABLED:false}
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
CREATE TABLE image_exclusion_list
|
||||
(
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
file_hash VARCHAR(64) NOT NULL UNIQUE,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
@ -0,0 +1,2 @@
|
||||
INSERT INTO content_providers(name, url, active, supports_content_fetch, manual_import)
|
||||
VALUES ('Rolia Scan', 'https://roliascan.com', TRUE, TRUE, FALSE);
|
||||
@ -0,0 +1,3 @@
|
||||
UPDATE content_providers
|
||||
SET url = 'https://beta.taimumangas.com/'
|
||||
WHERE name = 'Taimu';
|
||||
@ -0,0 +1,15 @@
|
||||
CREATE INDEX IF NOT EXISTS idx_manga_ingest_reviews_provider ON manga_ingest_reviews(content_provider_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_provider ON manga_content_provider(content_provider_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_manga_content_images_image ON manga_content_images(image_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_mangas_cover_image ON mangas(cover_image_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_content_providers_name ON content_providers(name);
|
||||
|
||||
DELETE FROM manga_ingest_reviews
|
||||
WHERE content_provider_id = (SELECT id FROM content_providers WHERE name = 'Taimu');
|
||||
|
||||
DELETE FROM manga_content_provider
|
||||
WHERE content_provider_id = (SELECT id FROM content_providers WHERE name = 'Taimu');
|
||||
|
||||
DELETE FROM images
|
||||
WHERE NOT EXISTS (SELECT 1 FROM manga_content_images WHERE image_id = images.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM mangas WHERE cover_image_id = images.id);
|
||||
Loading…
x
Reference in New Issue
Block a user