diff --git a/src/main/java/com/magamochi/mangamochi/client/JikanClient.java b/src/main/java/com/magamochi/mangamochi/client/JikanClient.java index 11f4d17..5cba7f0 100644 --- a/src/main/java/com/magamochi/mangamochi/client/JikanClient.java +++ b/src/main/java/com/magamochi/mangamochi/client/JikanClient.java @@ -1,9 +1,8 @@ package com.magamochi.mangamochi.client; +import io.github.resilience4j.retry.annotation.Retry; import java.time.OffsetDateTime; import java.util.List; - -import io.github.resilience4j.retry.annotation.Retry; import org.springframework.cloud.openfeign.FeignClient; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; diff --git a/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java b/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java index a3d5144..40a3ed7 100644 --- a/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java +++ b/src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java @@ -1,10 +1,9 @@ package com.magamochi.mangamochi.client; import com.magamochi.mangamochi.model.dto.MangaDexMangaDTO; +import io.github.resilience4j.retry.annotation.Retry; import java.util.List; import java.util.UUID; - -import io.github.resilience4j.retry.annotation.Retry; import org.springframework.cloud.openfeign.FeignClient; import org.springframework.web.bind.annotation.*; diff --git a/src/main/java/com/magamochi/mangamochi/service/S3Service.java b/src/main/java/com/magamochi/mangamochi/service/S3Service.java index eee40cc..f76ecd7 100644 --- a/src/main/java/com/magamochi/mangamochi/service/S3Service.java +++ b/src/main/java/com/magamochi/mangamochi/service/S3Service.java @@ -1,14 +1,19 @@ package com.magamochi.mangamochi.service; +import static java.util.Objects.nonNull; + import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; import java.util.UUID; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.*; @Service @RequiredArgsConstructor @@ -34,4 +39,43 @@ public class S3Service { return s3Client.getObject(request); } + + public void deleteObjects(Set objectKeys) { + if (CollectionUtils.isEmpty(objectKeys)) { + throw new IllegalArgumentException("Object key list cannot be null or empty"); + } + + var objectsToDelete = + objectKeys.stream().map(key -> ObjectIdentifier.builder().key(key).build()).toList(); + + var deleteRequest = + DeleteObjectsRequest.builder() + .bucket(bucket) + .delete(Delete.builder().objects(objectsToDelete).build()) + .build(); + + s3Client.deleteObjects(deleteRequest); + } + + public List listAllObjectKeys() { + var keys = new ArrayList(); + String continuationToken = null; + + do { + var requestBuilder = ListObjectsV2Request.builder().bucket(bucket).maxKeys(1000); + + if (nonNull(continuationToken)) { + requestBuilder.continuationToken(continuationToken); + } + + var response = s3Client.listObjectsV2(requestBuilder.build()); + + response.contents().forEach(s3Object -> keys.add(s3Object.key())); + + continuationToken = response.isTruncated() ? response.nextContinuationToken() : null; + + } while (nonNull(continuationToken)); + + return keys; + } } diff --git a/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java b/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java new file mode 100644 index 0000000..0fce0d3 --- /dev/null +++ b/src/main/java/com/magamochi/mangamochi/task/ImageCleanupTask.java @@ -0,0 +1,50 @@ +package com.magamochi.mangamochi.task; + +import com.magamochi.mangamochi.model.entity.Image; +import com.magamochi.mangamochi.model.repository.*; +import com.magamochi.mangamochi.service.S3Service; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Log4j2 +@Component +@RequiredArgsConstructor +public class ImageCleanupTask { + @Value("${image-service.clean-up-enabled}") + private Boolean cleanUpEnabled; + + private final S3Service s3Service; + private final ImageRepository imageRepository; + + @Scheduled(cron = "@weekly") + public void cleanupImages() { + if (!cleanUpEnabled) { + log.info("S3 Image cleanup disabled."); + return; + } + + log.info("Getting unused S3 object keys to remove."); + + var imageKeys = s3Service.listAllObjectKeys(); + + var existingImages = + imageRepository.findAll().parallelStream() + .map(Image::getFileKey) + .collect(Collectors.toSet()); + + var keysToRemove = + imageKeys.parallelStream() + .filter(imageKey -> !existingImages.contains(imageKey)) + .collect(Collectors.toSet()); + + log.info("Removing {} objects from S3 storage", keysToRemove.size()); + + s3Service.deleteObjects(keysToRemove); + + log.info("Image cleanup finished."); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e59f9d4..74ce9c1 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -67,3 +67,6 @@ resilience4j: seconds: 5 retry-exceptions: - feign.FeignException + +image-service: + clean-up-enabled: ${IMAGE_SERVICE_CLEAN_UP_ENABLED:false}