enable defaultAwsCredentialChain when botoCfgPath is neglected (#645)

Yelp · Apr 8, 2024 · 86ce4b7 · 86ce4b7
1 parent ba4ab9c
commit 86ce4b7
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 37 deletions.
diff --git a/docs/server_configuration.rst b/docs/server_configuration.rst
@@ -81,8 +81,8 @@ Example server configuration
 
    * - botoCfgPath
      - str
-     - Path to AWS credentials (if using S3 for remote storage)
-     - `<DEFAULT_USER_DIR> <https://github.com/Yelp/nrtsearch/blob/f612f5d3e14e468ab8c9b45dd4be0ab84231b9de/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java#L35>`_/boto.cfg
+     - Path to AWS credentials (if using S3 for remote storage); Will use the DefaultAWSCredentialsProviderChain if omitted.
+     - null
 
    * - archiveDirectory
      - str

diff --git a/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java b/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java
@@ -44,8 +44,6 @@ public class LuceneServerConfiguration {
       Paths.get(System.getProperty("user.home"), "lucene", "server");
   public static final Path DEFAULT_ARCHIVER_DIR =
       Paths.get(DEFAULT_USER_DIR.toString(), "archiver");
-  public static final Path DEFAULT_BOTO_CFG_PATH =
-      Paths.get(DEFAULT_USER_DIR.toString(), "boto.cfg");
   public static final Path DEFAULT_STATE_DIR =
       Paths.get(DEFAULT_USER_DIR.toString(), "default_state");
   public static final Path DEFAULT_INDEX_DIR =
@@ -135,7 +133,7 @@ public LuceneServerConfiguration(InputStream yamlStream) {
     stateDir = configReader.getString("stateDir", DEFAULT_STATE_DIR.toString());
     indexDir = configReader.getString("indexDir", DEFAULT_INDEX_DIR.toString());
     archiveDirectory = configReader.getString("archiveDirectory", DEFAULT_ARCHIVER_DIR.toString());
-    botoCfgPath = configReader.getString("botoCfgPath", DEFAULT_BOTO_CFG_PATH.toString());
+    botoCfgPath = configReader.getString("botoCfgPath", null);
     bucketName = configReader.getString("bucketName", DEFAULT_BUCKET_NAME);
     maxS3ClientRetries =
         configReader.getInteger("maxS3ClientRetries", DEFAULT_MAX_S3_CLIENT_RETRIES);

diff --git a/src/main/java/com/yelp/nrtsearch/server/remote/s3/S3Util.java b/src/main/java/com/yelp/nrtsearch/server/remote/s3/S3Util.java
@@ -16,9 +16,11 @@
 package com.yelp.nrtsearch.server.remote.s3;
 
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.AnonymousAWSCredentials;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.amazonaws.auth.profile.ProfilesConfigFile;
 import com.amazonaws.client.builder.AwsClientBuilder;
@@ -48,21 +50,26 @@ private S3Util() {}
    * @return s3 client
    */
   public static AmazonS3 buildS3Client(LuceneServerConfiguration luceneServerConfiguration) {
-    if (luceneServerConfiguration
-        .getBotoCfgPath()
-        .equals(LuceneServerConfiguration.DEFAULT_BOTO_CFG_PATH.toString())) {
-      return AmazonS3ClientBuilder.standard()
-          .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
-          .withEndpointConfiguration(
-              new AwsClientBuilder.EndpointConfiguration("dummyService", "dummyRegion"))
-          .build();
+    AWSCredentialsProvider awsCredentialsProvider;
+    if (luceneServerConfiguration.getBotoCfgPath() == null) {
+      awsCredentialsProvider = new DefaultAWSCredentialsProviderChain();
     } else {
       Path botoCfgPath = Paths.get(luceneServerConfiguration.getBotoCfgPath());
       final ProfilesConfigFile profilesConfigFile = new ProfilesConfigFile(botoCfgPath.toFile());
-      final AWSCredentialsProvider awsCredentialsProvider =
-          new ProfileCredentialsProvider(profilesConfigFile, "default");
+      awsCredentialsProvider = new ProfileCredentialsProvider(profilesConfigFile, "default");
+    }
+    final boolean globalBucketAccess = luceneServerConfiguration.getEnableGlobalBucketAccess();
+
+    AmazonS3ClientBuilder clientBuilder =
+        AmazonS3ClientBuilder.standard()
+            .withCredentials(awsCredentialsProvider)
+            .withForceGlobalBucketAccessEnabled(globalBucketAccess);
+    try {
       AmazonS3 s3ClientInterim =
-          AmazonS3ClientBuilder.standard().withCredentials(awsCredentialsProvider).build();
+          AmazonS3ClientBuilder.standard()
+              .withCredentials(awsCredentialsProvider)
+              .withForceGlobalBucketAccessEnabled(globalBucketAccess)
+              .build();
       String region = s3ClientInterim.getBucketLocation(luceneServerConfiguration.getBucketName());
       // In useast-1, the region is returned as "US" which is an equivalent to "us-east-1"
       // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html#US_Standard
@@ -72,29 +79,35 @@ public static AmazonS3 buildS3Client(LuceneServerConfiguration luceneServerConfi
       }
       String serviceEndpoint = String.format("s3.%s.amazonaws.com", region);
       logger.info(String.format("S3 ServiceEndpoint: %s", serviceEndpoint));
-      AmazonS3ClientBuilder clientBuilder =
-          AmazonS3ClientBuilder.standard()
-              .withCredentials(awsCredentialsProvider)
-              .withEndpointConfiguration(new EndpointConfiguration(serviceEndpoint, region));
-
-      int maxRetries = luceneServerConfiguration.getMaxS3ClientRetries();
-      if (maxRetries > 0) {
-        RetryPolicy retryPolicy =
-            new RetryPolicy(
-                PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
-                PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY,
-                maxRetries,
-                true);
-        ClientConfiguration clientConfiguration =
-            new ClientConfiguration().withRetryPolicy(retryPolicy);
-        clientBuilder.setClientConfiguration(clientConfiguration);
-      }
+      clientBuilder.withEndpointConfiguration(new EndpointConfiguration(serviceEndpoint, region));
+    } catch (SdkClientException sdkClientException) {
+      logger.warn(
+          "failed to get the location of S3 bucket: "
+              + luceneServerConfiguration.getBucketName()
+              + ". This could be caused by missing credentials and/or regions, or wrong bucket name.",
+          sdkClientException);
+      logger.info("return a dummy AmazonS3.");
+      return AmazonS3ClientBuilder.standard()
+          .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
+          .withEndpointConfiguration(
+              new AwsClientBuilder.EndpointConfiguration("dummyService", "dummyRegion"))
+          .build();
+    }
 
-      if (luceneServerConfiguration.getEnableGlobalBucketAccess()) {
-        clientBuilder.enableForceGlobalBucketAccess();
-      }
-      return clientBuilder.build();
+    int maxRetries = luceneServerConfiguration.getMaxS3ClientRetries();
+    if (maxRetries > 0) {
+      RetryPolicy retryPolicy =
+          new RetryPolicy(
+              PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
+              PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY,
+              maxRetries,
+              true);
+      ClientConfiguration clientConfiguration =
+          new ClientConfiguration().withRetryPolicy(retryPolicy);
+      clientBuilder.setClientConfiguration(clientConfiguration);
     }
+
+    return clientBuilder.build();
   }
 
   /**