Merge pull request 'fix: optimize S3 object deletion by batching requests and adding error handling' (#16) from fix/image-cleanup into main
Reviewed-on: #16
This commit is contained in:
commit
e3ba04a087
@ -45,16 +45,47 @@ public class S3Service {
|
|||||||
throw new IllegalArgumentException("Object key list cannot be null or empty");
|
throw new IllegalArgumentException("Object key list cannot be null or empty");
|
||||||
}
|
}
|
||||||
|
|
||||||
var objectsToDelete =
|
final int BATCH_SIZE = 500;
|
||||||
|
List<ObjectIdentifier> allObjects =
|
||||||
objectKeys.stream().map(key -> ObjectIdentifier.builder().key(key).build()).toList();
|
objectKeys.stream().map(key -> ObjectIdentifier.builder().key(key).build()).toList();
|
||||||
|
|
||||||
var deleteRequest =
|
for (int i = 0; i < allObjects.size(); i += BATCH_SIZE) {
|
||||||
|
int end = Math.min(i + BATCH_SIZE, allObjects.size());
|
||||||
|
List<ObjectIdentifier> batch = allObjects.subList(i, end);
|
||||||
|
|
||||||
|
DeleteObjectsRequest deleteRequest =
|
||||||
DeleteObjectsRequest.builder()
|
DeleteObjectsRequest.builder()
|
||||||
.bucket(bucket)
|
.bucket(bucket)
|
||||||
.delete(Delete.builder().objects(objectsToDelete).build())
|
.delete(Delete.builder().objects(batch).build())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
s3Client.deleteObjects(deleteRequest);
|
try {
|
||||||
|
var response = s3Client.deleteObjects(deleteRequest);
|
||||||
|
|
||||||
|
if (!response.errors().isEmpty()) {
|
||||||
|
response
|
||||||
|
.errors()
|
||||||
|
.forEach(
|
||||||
|
error ->
|
||||||
|
System.err.println(
|
||||||
|
"Error deleting key: " + error.key() + " -> " + error.message()));
|
||||||
|
} else {
|
||||||
|
System.out.println(
|
||||||
|
"Deleted "
|
||||||
|
+ batch.size()
|
||||||
|
+ " objects successfully (batch "
|
||||||
|
+ (i / BATCH_SIZE + 1)
|
||||||
|
+ ")");
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (S3Exception e) {
|
||||||
|
System.err.println(
|
||||||
|
"Failed to delete batch starting at index "
|
||||||
|
+ i
|
||||||
|
+ ": "
|
||||||
|
+ e.awsErrorDetails().errorMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<String> listAllObjectKeys() {
|
public List<String> listAllObjectKeys() {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user