diff --git a/src/main/java/com/magamochi/image/model/repository/ImageRepository.java b/src/main/java/com/magamochi/image/model/repository/ImageRepository.java index 342a464..51ad357 100644 --- a/src/main/java/com/magamochi/image/model/repository/ImageRepository.java +++ b/src/main/java/com/magamochi/image/model/repository/ImageRepository.java @@ -1,10 +1,16 @@ package com.magamochi.image.model.repository; import com.magamochi.image.model.entity.Image; +import java.util.List; import java.util.Optional; import java.util.UUID; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; public interface ImageRepository extends JpaRepository { Optional findByFileHash(String fileHash); + + @Query("SELECT i.objectKey FROM Image i WHERE i.objectKey IN :objectKeys") + List findExistingObjectKeys(@Param("objectKeys") List objectKeys); } diff --git a/src/main/java/com/magamochi/image/service/ImageService.java b/src/main/java/com/magamochi/image/service/ImageService.java index d53e9d3..55de1ba 100644 --- a/src/main/java/com/magamochi/image/service/ImageService.java +++ b/src/main/java/com/magamochi/image/service/ImageService.java @@ -4,7 +4,9 @@ import com.magamochi.common.exception.NotFoundException; import com.magamochi.image.model.entity.Image; import com.magamochi.image.model.repository.ImageRepository; import java.io.InputStream; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -49,8 +51,8 @@ public class ImageService { .orElseThrow(() -> new NotFoundException("Image not found with ID " + id)); } - public List findAll() { - return imageRepository.findAll(); + public Set findExistingObjectKeys(List objectKeys) { + return new HashSet<>(imageRepository.findExistingObjectKeys(objectKeys)); } public InputStream getStream(Image image) { diff --git a/src/main/java/com/magamochi/image/service/S3Service.java b/src/main/java/com/magamochi/image/service/S3Service.java index 125b8f2..8316230 100644 --- a/src/main/java/com/magamochi/image/service/S3Service.java +++ b/src/main/java/com/magamochi/image/service/S3Service.java @@ -4,7 +4,6 @@ import static java.util.Objects.nonNull; import java.io.InputStream; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.Set; import lombok.Getter; @@ -42,8 +41,7 @@ public class S3Service { return filename; } - public List listAllObjectKeys() { - var keys = new ArrayList(); + public void processObjectKeyPages(java.util.function.Consumer> pageConsumer) { String continuationToken = null; do { @@ -55,13 +53,12 @@ public class S3Service { var response = s3Client.listObjectsV2(requestBuilder.build()); - response.contents().forEach(s3Object -> keys.add(s3Object.key())); + var page = response.contents().stream().map(S3Object::key).toList(); + pageConsumer.accept(page); continuationToken = response.isTruncated() ? response.nextContinuationToken() : null; } while (nonNull(continuationToken)); - - return keys; } public void deleteObjects(Set objectKeys) { diff --git a/src/main/java/com/magamochi/image/task/ImageCleanupTask.java b/src/main/java/com/magamochi/image/task/ImageCleanupTask.java index 6360ea4..af7a0f1 100644 --- a/src/main/java/com/magamochi/image/task/ImageCleanupTask.java +++ b/src/main/java/com/magamochi/image/task/ImageCleanupTask.java @@ -1,9 +1,7 @@ package com.magamochi.image.task; -import com.magamochi.image.model.entity.Image; import com.magamochi.image.service.ImageService; import com.magamochi.image.service.S3Service; -import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Value; @@ -31,21 +29,21 @@ public class ImageCleanupTask { } public void cleanupImages() { - log.info("Getting unused S3 object keys to remove."); + log.info("Scanning S3 pages for orphaned object keys."); + var keysToRemove = new java.util.HashSet(); - var imageKeys = s3Service.listAllObjectKeys(); + s3Service.processObjectKeyPages( + page -> { + var existing = imageService.findExistingObjectKeys(page); + page.stream().filter(key -> !existing.contains(key)).forEach(keysToRemove::add); + }); - var existingImages = - imageService.findAll().parallelStream() - .map(Image::getObjectKey) - .collect(Collectors.toSet()); + if (keysToRemove.isEmpty()) { + log.info("No orphaned objects found."); + return; + } - var keysToRemove = - imageKeys.parallelStream() - .filter(imageKey -> !existingImages.contains(imageKey)) - .collect(Collectors.toSet()); - - log.info("Removing {} objects from S3 storage", keysToRemove.size()); + log.info("Removing {} orphaned objects from S3 storage", keysToRemove.size()); s3Service.deleteObjects(keysToRemove);