From 915da1a71bf7e2e119bdd5b826c87bc0ffe6089e Mon Sep 17 00:00:00 2001 From: waziqi89 <89210409+waziqi89@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:53:23 -0400 Subject: [PATCH] enable defaultAwsCredentialChain when botoCfgPath is neglected (#627) --- docs/server_configuration.rst | 4 +- .../config/LuceneServerConfiguration.java | 4 +- .../nrtsearch/server/module/S3Module.java | 77 +++++++++++-------- 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/docs/server_configuration.rst b/docs/server_configuration.rst index 909621761..5dac2421a 100644 --- a/docs/server_configuration.rst +++ b/docs/server_configuration.rst @@ -83,8 +83,8 @@ Example server configuration * - botoCfgPath - str - - Path to AWS credentials (if using S3 for remote storage) - - ` `_/boto.cfg + - Path to AWS credentials (if using S3 for remote storage); Will use the DefaultAWSCredentialsProviderChain if omitted. + - null * - archiveDirectory - str diff --git a/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java b/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java index 63fe32171..f1de0740f 100644 --- a/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java +++ b/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java @@ -44,8 +44,6 @@ public class LuceneServerConfiguration { Paths.get(System.getProperty("user.home"), "lucene", "server"); public static final Path DEFAULT_ARCHIVER_DIR = Paths.get(DEFAULT_USER_DIR.toString(), "archiver"); - public static final Path DEFAULT_BOTO_CFG_PATH = - Paths.get(DEFAULT_USER_DIR.toString(), "boto.cfg"); public static final Path DEFAULT_STATE_DIR = Paths.get(DEFAULT_USER_DIR.toString(), "default_state"); public static final Path DEFAULT_INDEX_DIR = @@ -137,7 +135,7 @@ public LuceneServerConfiguration(InputStream yamlStream) { stateDir = configReader.getString("stateDir", DEFAULT_STATE_DIR.toString()); indexDir = configReader.getString("indexDir", DEFAULT_INDEX_DIR.toString()); archiveDirectory = configReader.getString("archiveDirectory", DEFAULT_ARCHIVER_DIR.toString()); - botoCfgPath = configReader.getString("botoCfgPath", DEFAULT_BOTO_CFG_PATH.toString()); + botoCfgPath = configReader.getString("botoCfgPath", null); bucketName = configReader.getString("bucketName", DEFAULT_BUCKET_NAME); maxS3ClientRetries = configReader.getInteger("maxS3ClientRetries", DEFAULT_MAX_S3_CLIENT_RETRIES); diff --git a/src/main/java/com/yelp/nrtsearch/server/module/S3Module.java b/src/main/java/com/yelp/nrtsearch/server/module/S3Module.java index 1456a340c..8d1a9ac10 100644 --- a/src/main/java/com/yelp/nrtsearch/server/module/S3Module.java +++ b/src/main/java/com/yelp/nrtsearch/server/module/S3Module.java @@ -16,9 +16,11 @@ package com.yelp.nrtsearch.server.module; import com.amazonaws.ClientConfiguration; +import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.amazonaws.auth.profile.ProfilesConfigFile; import com.amazonaws.client.builder.AwsClientBuilder; @@ -44,21 +46,26 @@ public class S3Module extends AbstractModule { @Singleton @Provides protected AmazonS3 providesAmazonS3(LuceneServerConfiguration luceneServerConfiguration) { - if (luceneServerConfiguration - .getBotoCfgPath() - .equals(LuceneServerConfiguration.DEFAULT_BOTO_CFG_PATH.toString())) { - return AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration("dummyService", "dummyRegion")) - .build(); + AWSCredentialsProvider awsCredentialsProvider; + if (luceneServerConfiguration.getBotoCfgPath() == null) { + awsCredentialsProvider = new DefaultAWSCredentialsProviderChain(); } else { Path botoCfgPath = Paths.get(luceneServerConfiguration.getBotoCfgPath()); final ProfilesConfigFile profilesConfigFile = new ProfilesConfigFile(botoCfgPath.toFile()); - final AWSCredentialsProvider awsCredentialsProvider = - new ProfileCredentialsProvider(profilesConfigFile, "default"); + awsCredentialsProvider = new ProfileCredentialsProvider(profilesConfigFile, "default"); + } + final boolean globalBucketAccess = luceneServerConfiguration.getEnableGlobalBucketAccess(); + + AmazonS3ClientBuilder clientBuilder = + AmazonS3ClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withForceGlobalBucketAccessEnabled(globalBucketAccess); + try { AmazonS3 s3ClientInterim = - AmazonS3ClientBuilder.standard().withCredentials(awsCredentialsProvider).build(); + AmazonS3ClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withForceGlobalBucketAccessEnabled(globalBucketAccess) + .build(); String region = s3ClientInterim.getBucketLocation(luceneServerConfiguration.getBucketName()); // In useast-1, the region is returned as "US" which is an equivalent to "us-east-1" // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html#US_Standard @@ -68,28 +75,34 @@ protected AmazonS3 providesAmazonS3(LuceneServerConfiguration luceneServerConfig } String serviceEndpoint = String.format("s3.%s.amazonaws.com", region); logger.info(String.format("S3 ServiceEndpoint: %s", serviceEndpoint)); - AmazonS3ClientBuilder clientBuilder = - AmazonS3ClientBuilder.standard() - .withCredentials(awsCredentialsProvider) - .withEndpointConfiguration(new EndpointConfiguration(serviceEndpoint, region)); - - int maxRetries = luceneServerConfiguration.getMaxS3ClientRetries(); - if (maxRetries > 0) { - RetryPolicy retryPolicy = - new RetryPolicy( - PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, - PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, - maxRetries, - true); - ClientConfiguration clientConfiguration = - new ClientConfiguration().withRetryPolicy(retryPolicy); - clientBuilder.setClientConfiguration(clientConfiguration); - } + clientBuilder.withEndpointConfiguration(new EndpointConfiguration(serviceEndpoint, region)); + } catch (SdkClientException sdkClientException) { + logger.warn( + "failed to get the location of S3 bucket: " + + luceneServerConfiguration.getBucketName() + + ". This could be caused by missing credentials and/or regions, or wrong bucket name.", + sdkClientException); + logger.info("return a dummy AmazonS3."); + return AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration("dummyService", "dummyRegion")) + .build(); + } - if (luceneServerConfiguration.getEnableGlobalBucketAccess()) { - clientBuilder.enableForceGlobalBucketAccess(); - } - return clientBuilder.build(); + int maxRetries = luceneServerConfiguration.getMaxS3ClientRetries(); + if (maxRetries > 0) { + RetryPolicy retryPolicy = + new RetryPolicy( + PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, + PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, + maxRetries, + true); + ClientConfiguration clientConfiguration = + new ClientConfiguration().withRetryPolicy(retryPolicy); + clientBuilder.setClientConfiguration(clientConfiguration); } + + return clientBuilder.build(); } }