support retry

apache · Oct 23, 2024 · dcddcc3 · dcddcc3
1 parent 3edbeec
commit dcddcc3
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/multipart-uploader/src/main/java/org/apache/celeborn/S3MultipartUploadHandler.java b/multipart-uploader/src/main/java/org/apache/celeborn/S3MultipartUploadHandler.java
@@ -23,8 +23,10 @@
 import java.util.Collections;
 import java.util.List;
 
+import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.retry.PredefinedRetryPolicies;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
@@ -58,10 +60,16 @@ public S3MultipartUploadHandler(AWSCredentials awsCredentials, String key) {
     this.awsCredentials = awsCredentials;
     BasicAWSCredentials basicAWSCredentials =
         new BasicAWSCredentials(awsCredentials.getS3AccessKey(), awsCredentials.getS3SecretKey());
+    ClientConfiguration clientConfig = new ClientConfiguration()
+            .withRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(5))
+            .withMaxErrorRetry(5)
+            .withConnectionTimeout(10000)
+            .withRequestTimeout(50000);
     this.s3Client =
         AmazonS3ClientBuilder.standard()
             .withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials))
             .withRegion(awsCredentials.getS3EndpointRegion())
+            .withClientConfiguration(clientConfig)
             .build();
     this.key = key;
   }