mangamochi/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat/image-exclusion-list #57

Merged
rov merged 2 commits from feat/image-exclusion-list into main 2026-05-08 11:37:03 -03:00
Conversation 0 Commits 2 Files Changed 17 +326 -17
17 changed files with 326 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/magamochi/catalog/model/repository/MangaRepository.java
View File

@ -3,6 +3,7 @@ package com.magamochi.catalog.model.repository;
import com.magamochi.catalog.model.entity.Manga;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
@ -15,4 +16,6 @@ public interface MangaRepository
Optional<Manga> findByMalId(Long malId);
Optional<Manga> findByAniListId(Long aniListId);
List<Manga> findAllByCoverImage_Id(UUID imageId);
}

11
src/main/java/com/magamochi/common/exception/ExcludedImageException.java Normal file
View File

@ -0,0 +1,11 @@
package com.magamochi.common.exception;
public class ExcludedImageException extends RuntimeException {
public ExcludedImageException(String message) {
super(message);
}
public ExcludedImageException(String message, Throwable cause) {
super(message, cause);
}
}

16
src/main/java/com/magamochi/content/model/dto/MangaContentImagesDTO.java
View File

@ -6,22 +6,30 @@ import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import java.util.Comparator;
import java.util.List;
import java.util.UUID;
public record MangaContentImagesDTO(
@NotNull Long id,
@NotBlank String mangaTitle,
Long previousContentId,
Long nextContentId,
@NotNull List<@NotBlank String> contentImageKeys) {
public static MangaContentImagesDTO from(MangaContent mangaContent, Long prevId, Long nextId) {
@NotNull List<MangaContentImageDTO> contentImages) {
public static MangaContentImagesDTO from(
MangaContent mangaContent, List<MangaContentImage> images, Long prevId, Long nextId) {
return new MangaContentImagesDTO(
mangaContent.getId(),
mangaContent.getTitle(),
prevId,
nextId,
mangaContent.getMangaContentImages().stream()
images.stream()
.sorted(Comparator.comparing(MangaContentImage::getPosition))
.map(mangaContentImage -> mangaContentImage.getImage().getObjectKey())
.map(
mangaContentImage ->
new MangaContentImageDTO(
mangaContentImage.getImage().getId(),
mangaContentImage.getImage().getObjectKey()))
.toList());
}
public record MangaContentImageDTO(UUID uuid, String objectKey) {}
}

8
src/main/java/com/magamochi/content/model/repository/MangaContentImageRepository.java
View File

@ -1,12 +1,14 @@
package com.magamochi.content.model.repository;
import com.magamochi.content.model.entity.MangaContent;
import com.magamochi.content.model.entity.MangaContentImage;
import java.util.List;
import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
public interface MangaContentImageRepository extends JpaRepository<MangaContentImage, Long> {
List<MangaContentImage> findAllByMangaContent(MangaContent mangaContent);
boolean existsByMangaContent_IdAndPosition(Long mangaContentId, int position);
List<MangaContentImage> findAllByImage_Id(UUID imageId);
List<MangaContentImage> findAllByMangaContent_IdOrderByPositionAsc(Long mangaContentId);
}

4
src/main/java/com/magamochi/content/service/ContentDownloadService.java
View File

@ -4,6 +4,7 @@ import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.content.model.dto.MangaContentArchiveDTO;
import com.magamochi.content.model.entity.MangaContentImage;
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
import com.magamochi.image.service.ImageExclusionService;
import com.magamochi.image.service.ImageService;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
@ -22,11 +23,12 @@ import org.springframework.stereotype.Service;
public class ContentDownloadService {
private final ContentService contentService;
private final ImageService imageService;
private final ImageExclusionService imageExclusionService;
public MangaContentArchiveDTO downloadContent(
Long mangaContentId, ContentArchiveFileType contentArchiveFileType) throws IOException {
var chapter = contentService.find(mangaContentId);
var chapterImages = chapter.getMangaContentImages();
var chapterImages = imageExclusionService.filterExcludedImages(chapter.getMangaContentImages());
var byteArrayOutputStream =
switch (contentArchiveFileType) {

27
src/main/java/com/magamochi/content/service/ContentImportService.java
View File

@ -7,6 +7,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank;
import com.magamochi.catalog.model.specification.MangaImportJobSpecification;
import com.magamochi.catalog.service.MangaContentProviderService;
import com.magamochi.catalog.service.MangaResolutionService;
import com.magamochi.common.exception.ExcludedImageException;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.common.model.enumeration.ContentType;
@ -202,15 +203,25 @@ public class ContentImportService {
for (var sortedEntry : entryMap.entrySet()) {
var bytes = sortedEntry.getValue();
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
var image = imageService.find(imageId);
try {
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
var image = imageService.find(imageId);
mangaContentImageRepository.save(
MangaContentImage.builder()
.image(image)
.mangaContent(mangaContent)
.position(position++)
.build());
mangaContentImageRepository.save(
MangaContentImage.builder()
.image(image)
.mangaContent(mangaContent)
.position(position)
.build());
} catch (ExcludedImageException e) {
log.info(
"Image at position {} in content {} is excluded and will be skipped.",
position,
mangaContent.getId());
} finally {
// Ensure we always increment position even if an image is excluded
position++;
}
}
}

7
src/main/java/com/magamochi/content/service/ContentService.java
View File

@ -6,6 +6,7 @@ import com.magamochi.content.model.dto.MangaContentDTO;
import com.magamochi.content.model.dto.MangaContentImagesDTO;
import com.magamochi.content.model.entity.MangaContent;
import com.magamochi.content.model.repository.MangaContentRepository;
import com.magamochi.image.service.ImageExclusionService;
import com.magamochi.userinteraction.service.UserMangaContentReadService;
import jakarta.validation.constraints.NotNull;
import java.util.Comparator;
@ -19,6 +20,7 @@ import org.springframework.stereotype.Service;
public class ContentService {
private final MangaContentProviderService mangaContentProviderService;
private final UserMangaContentReadService userMangaContentReadService;
private final ImageExclusionService imageExclusionService;
private final MangaContentRepository mangaContentRepository;
@ -70,6 +72,9 @@ public class ContentService {
}
}
return MangaContentImagesDTO.from(mangaContent, prevId, nextId);
var filteredImages =
imageExclusionService.filterExcludedImages(mangaContent.getMangaContentImages());
return MangaContentImagesDTO.from(mangaContent, filteredImages, prevId, nextId);
}
}

14
src/main/java/com/magamochi/controller/ManagementController.java
View File

@ -2,6 +2,7 @@ package com.magamochi.controller;
import com.magamochi.client.NtfyClient;
import com.magamochi.common.model.dto.DefaultResponseDTO;
import com.magamochi.image.service.ImageExclusionService;
import com.magamochi.image.task.ImageCleanupTask;
import com.magamochi.user.repository.UserRepository;
import com.magamochi.userinteraction.task.MangaFollowUpdateTask;
@ -14,6 +15,7 @@ import org.springframework.web.bind.annotation.*;
@RequiredArgsConstructor
public class ManagementController {
private final ImageCleanupTask imageCleanupTask;
private final ImageExclusionService imageExclusionService;
private final MangaFollowUpdateTask mangaFollowUpdateTask;
private final UserRepository userRepository;
private final NtfyClient ntfyClient;
@ -30,6 +32,18 @@ public class ManagementController {
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Cleanup excluded images",
description = "Triggers the cleanup of images that are in the exclusion list",
tags = {"Management"},
operationId = "imageExclusionCleanup")
@PostMapping("image-exclusion-cleanup")
public DefaultResponseDTO<Void> imageExclusionCleanup() {
imageExclusionService.cleanupExcludedImages();
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Trigger user follow update",
description = "Trigger user follow update",

27
src/main/java/com/magamochi/image/controller/ImageExclusionController.java Normal file
View File

@ -0,0 +1,27 @@
package com.magamochi.image.controller;
import com.magamochi.common.model.dto.DefaultResponseDTO;
import com.magamochi.image.service.ImageExclusionService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/management/image-exclusion")
@RequiredArgsConstructor
@Tag(name = "Image Exclusion", description = "Management API for image exclusion")
public class ImageExclusionController {
private final ImageExclusionService exclusionService;
@Operation(
summary = "Exclude image by UUID",
description =
"Gets the hash of the image with the given UUID and adds it to the exclusion list.")
@PostMapping("/{uuid}")
public DefaultResponseDTO<Void> excludeImage(@PathVariable UUID uuid) {
exclusionService.excludeImageByUuid(uuid);
return DefaultResponseDTO.ok().build();
}
}

23
src/main/java/com/magamochi/image/model/entity/ImageExclusion.java Normal file
View File

@ -0,0 +1,23 @@
package com.magamochi.image.model.entity;
import jakarta.persistence.*;
import java.time.Instant;
import lombok.*;
import org.hibernate.annotations.CreationTimestamp;
@Entity
@Table(name = "image_exclusion_list")
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Getter
@Setter
public class ImageExclusion {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
private String fileHash;
@CreationTimestamp private Instant createdAt;
}

12
src/main/java/com/magamochi/image/model/repository/ImageExclusionRepository.java Normal file
View File

@ -0,0 +1,12 @@
package com.magamochi.image.model.repository;
import com.magamochi.image.model.entity.ImageExclusion;
import java.util.Collection;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
public interface ImageExclusionRepository extends JpaRepository<ImageExclusion, Long> {
boolean existsByFileHash(String fileHash);
List<ImageExclusion> findAllByFileHashIn(Collection<String> fileHashes);
}

3
src/main/java/com/magamochi/image/queue/consumer/ImageFetchConsumer.java
View File

@ -1,5 +1,6 @@
package com.magamochi.image.queue.consumer;
import com.magamochi.common.exception.ExcludedImageException;
import com.magamochi.common.queue.command.ImageFetchCommand;
import com.magamochi.common.queue.command.ImageUpdateCommand;
import com.magamochi.image.queue.producer.ImageUpdateProducer;
@ -25,6 +26,8 @@ public class ImageFetchConsumer {
imageUpdateProducer.publishImageUpdateCommand(
new ImageUpdateCommand(command.entityId(), imageId), command.contentType());
} catch (ExcludedImageException e) {
log.info("Image at URL {} is excluded from upload, skipping.", command.url());
} catch (Exception e) {
log.error("Failed to fetch image from URL.", e);
}

143
src/main/java/com/magamochi/image/service/ImageExclusionService.java Normal file
View File

@ -0,0 +1,143 @@
package com.magamochi.image.service;
import static java.util.Objects.isNull;
import com.magamochi.catalog.model.repository.MangaRepository;
import com.magamochi.content.model.entity.MangaContentImage;
import com.magamochi.content.model.repository.MangaContentImageRepository;
import com.magamochi.image.model.entity.ImageExclusion;
import com.magamochi.image.model.repository.ImageExclusionRepository;
import com.magamochi.image.model.repository.ImageRepository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
@Log4j2
@Service
@RequiredArgsConstructor
public class ImageExclusionService {
private final ImageService imageService;
private final S3Service s3Service;
private final ImageExclusionRepository exclusionRepository;
private final ImageRepository imageRepository;
private final MangaRepository mangaRepository;
private final MangaContentImageRepository mangaContentImageRepository;
@Transactional
public void excludeImageByUuid(UUID imageId) {
var image = imageService.find(imageId);
var hash = image.getFileHash();
if (exclusionRepository.existsByFileHash(hash)) {
log.info("Image hash {} (UUID {}) is already in the exclusion list", hash, imageId);
return;
}
exclusionRepository.save(ImageExclusion.builder().fileHash(hash).build());
log.info("Image hash {} (UUID {}) added to exclusion list", hash, imageId);
}
public Set<String> getExcludedHashes(Collection<String> hashes) {
if (isNull(hashes) || hashes.isEmpty()) {
return Set.of();
}
return exclusionRepository.findAllByFileHashIn(hashes).stream()
.map(ImageExclusion::getFileHash)
.collect(Collectors.toSet());
}
public List<MangaContentImage> filterExcludedImages(List<MangaContentImage> images) {
var hashes = images.stream().map(img -> img.getImage().getFileHash()).toList();
var excludedHashes = getExcludedHashes(hashes);
return images.stream()
.filter(img -> !excludedHashes.contains(img.getImage().getFileHash()))
.toList();
}
@Transactional
public void cleanupExcludedImages() {
log.info("Starting cleanup of excluded images...");
var exclusions = exclusionRepository.findAll();
log.info("Found {} excluded hashes to check", exclusions.size());
var keysToDelete =
exclusions.stream()
.map(exclusion -> processSingleExclusion(exclusion.getFileHash()))
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toSet());
if (!keysToDelete.isEmpty()) {
log.info("Deleting {} objects from S3", keysToDelete.size());
s3Service.deleteObjects(keysToDelete);
}
log.info("Finished cleanup of excluded images.");
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public Optional<String> processSingleExclusion(String fileHash) {
var imageOpt = imageRepository.findByFileHash(fileHash);
if (imageOpt.isEmpty()) {
return Optional.empty();
}
var image = imageOpt.get();
var imageId = image.getId();
var objectKey = image.getObjectKey();
log.info("Processing exclusion for image {} (hash: {})", imageId, fileHash);
// 1. Remove from Manga covers
var mangasWithCover = mangaRepository.findAllByCoverImage_Id(imageId);
if (!mangasWithCover.isEmpty()) {
log.info("Removing image from {} manga covers", mangasWithCover.size());
mangasWithCover.forEach(
manga -> {
manga.setCoverImage(null);
mangaRepository.save(manga);
});
}
// 2. Remove from Manga content and reindex
var contentImages = mangaContentImageRepository.findAllByImage_Id(imageId);
if (!contentImages.isEmpty()) {
log.info("Removing image from {} manga content entries", contentImages.size());
contentImages.forEach(
mci -> {
var mangaContent = mci.getMangaContent();
mangaContentImageRepository.delete(mci);
reindexMangaContent(mangaContent.getId());
});
}
imageRepository.delete(image);
return Optional.of(objectKey);
}
private void reindexMangaContent(Long mangaContentId) {
var remainingImages =
mangaContentImageRepository.findAllByMangaContent_IdOrderByPositionAsc(mangaContentId);
for (var i = 0; i < remainingImages.size(); i++) {
var img = remainingImages.get(i);
if (img.getPosition() != i) {
img.setPosition(i);
mangaContentImageRepository.save(img);
}
}
}
}

9
src/main/java/com/magamochi/image/service/ImageService.java
View File

@ -1,7 +1,9 @@
package com.magamochi.image.service;
import com.magamochi.common.exception.ExcludedImageException;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.image.model.entity.Image;
import com.magamochi.image.model.repository.ImageExclusionRepository;
import com.magamochi.image.model.repository.ImageRepository;
import java.io.InputStream;
import java.util.List;
@ -18,8 +20,15 @@ import org.springframework.stereotype.Service;
public class ImageService {
private final S3Service s3Service;
private final ImageRepository imageRepository;
private final ImageExclusionRepository imageExclusionRepository;
public UUID upload(byte[] data, String contentType, String path, String fileHash) {
var isExcluded = imageExclusionRepository.existsByFileHash(fileHash);
if (isExcluded) {
log.info("Image with hash {} is in the exclusion list, skipping upload", fileHash);
throw new ExcludedImageException("Image with hash " + fileHash + " is excluded from upload");
}
var existingImage = imageRepository.findByFileHash(fileHash);
if (existingImage.isPresent()) {
log.info("Image already exists with hash {}, returning existing ID", fileHash);

28
src/main/java/com/magamochi/image/task/ExclusionCleanupTask.java Normal file
View File

@ -0,0 +1,28 @@
package com.magamochi.image.task;
import com.magamochi.image.service.ImageExclusionService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Log4j2
@Component
@RequiredArgsConstructor
public class ExclusionCleanupTask {
@Value("${image-service.exclusion-cleanup-enabled:true}")
private Boolean cleanupEnabled;
private final ImageExclusionService exclusionService;
@Scheduled(cron = "${image-service.exclusion-cleanup-cron:@weekly}")
public void cleanupExcludedImagesScheduled() {
if (!cleanupEnabled) {
log.info("Excluded image cleanup is disabled.");
return;
}
exclusionService.cleanupExcludedImages();
}
}

2
src/main/resources/application.yml
View File

@ -90,6 +90,8 @@ routing-key:
image-service:
clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}
cron-expression: "@weekly"
exclusion-cleanup-enabled: ${IMAGE_SERVICE_EXCLUSION_CLEANUP_ENABLED:true}
exclusion-cleanup-cron: "@weekly"
content-providers:
update-enabled: ${CONTENT_PROVIDER_UPDATE_ENABLED:false}

6
src/main/resources/db/migration/V0013__CREATE_IMAGE_EXCLUSION_LIST.sql Normal file
View File

@ -0,0 +1,6 @@
CREATE TABLE image_exclusion_list
(
id BIGSERIAL PRIMARY KEY,
file_hash VARCHAR(64) NOT NULL UNIQUE,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);