fix: optimize S3 object deletion by batching requests and adding error handling #16
@ -45,16 +45,47 @@ public class S3Service {
|
||||
throw new IllegalArgumentException("Object key list cannot be null or empty");
|
||||
}
|
||||
|
||||
var objectsToDelete =
|
||||
final int BATCH_SIZE = 500;
|
||||
List<ObjectIdentifier> allObjects =
|
||||
objectKeys.stream().map(key -> ObjectIdentifier.builder().key(key).build()).toList();
|
||||
|
||||
var deleteRequest =
|
||||
DeleteObjectsRequest.builder()
|
||||
.bucket(bucket)
|
||||
.delete(Delete.builder().objects(objectsToDelete).build())
|
||||
.build();
|
||||
for (int i = 0; i < allObjects.size(); i += BATCH_SIZE) {
|
||||
int end = Math.min(i + BATCH_SIZE, allObjects.size());
|
||||
List<ObjectIdentifier> batch = allObjects.subList(i, end);
|
||||
|
||||
s3Client.deleteObjects(deleteRequest);
|
||||
DeleteObjectsRequest deleteRequest =
|
||||
DeleteObjectsRequest.builder()
|
||||
.bucket(bucket)
|
||||
.delete(Delete.builder().objects(batch).build())
|
||||
.build();
|
||||
|
||||
try {
|
||||
var response = s3Client.deleteObjects(deleteRequest);
|
||||
|
||||
if (!response.errors().isEmpty()) {
|
||||
response
|
||||
.errors()
|
||||
.forEach(
|
||||
error ->
|
||||
System.err.println(
|
||||
"Error deleting key: " + error.key() + " -> " + error.message()));
|
||||
} else {
|
||||
System.out.println(
|
||||
"Deleted "
|
||||
+ batch.size()
|
||||
+ " objects successfully (batch "
|
||||
+ (i / BATCH_SIZE + 1)
|
||||
+ ")");
|
||||
}
|
||||
|
||||
} catch (S3Exception e) {
|
||||
System.err.println(
|
||||
"Failed to delete batch starting at index "
|
||||
+ i
|
||||
+ ": "
|
||||
+ e.awsErrorDetails().errorMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public List<String> listAllObjectKeys() {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user