honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

S3 repository: Deprecate specifying credentials through env vars, sys props, and remove profile files (#22567) 

* S3 repository: Deprecate specifying credentials through env vars and sys props

This is a follow up to #22479, where storing credentials secure way was
added.
This commit is contained in:
Ryan Ernst 2017-01-19 12:36:32 -08:00 committed by GitHub
parent 3ea37ec817
commit c5b4bba30b
8 changed files with 157 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
View File

@ -80,6 +80,7 @@ class RandomizedTestingTask extends DefaultTask {
String argLine = null
Map<String, Object> systemProperties = new HashMap<>()
Map<String, Object> environmentVariables = new HashMap<>()
PatternFilterable patternSet = new PatternSet()
RandomizedTestingTask() {
@ -105,6 +106,10 @@ class RandomizedTestingTask extends DefaultTask {
systemProperties.put(property, value)
}
void environment(String key, Object value) {
environmentVariables.put(key, value)
}
void include(String... includes) {
this.patternSet.include(includes);
}
@ -193,7 +198,8 @@ class RandomizedTestingTask extends DefaultTask {
haltOnFailure: true, // we want to capture when a build failed, but will decide whether to rethrow later
shuffleOnSlave: shuffleOnSlave,
leaveTemporary: leaveTemporary,
ifNoTests: ifNoTests
ifNoTests: ifNoTests,
newenvironment: true
]
DefaultLogger listener = null
@ -249,6 +255,9 @@ class RandomizedTestingTask extends DefaultTask {
for (Map.Entry<String, Object> prop : systemProperties) {
sysproperty key: prop.getKey(), value: prop.getValue().toString()
}
for (Map.Entry<String, Object> envvar : environmentVariables) {
env key: envvar.getKey(), value: envvar.getValue().toString()
}
makeListeners()
}
} catch (BuildException e) {

14
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.gradle
import com.carrotsearch.gradle.junit4.RandomizedTestingTask
import nebula.plugin.extraconfigurations.ProvidedBasePlugin
import org.elasticsearch.gradle.precommit.PrecommitTasks
import org.gradle.api.GradleException
@ -560,11 +561,22 @@ class BuildPlugin implements Plugin<Project> {
/** Configures the test task */
static Task configureTest(Project project) {
Task test = project.tasks.getByName('test')
RandomizedTestingTask test = project.tasks.getByName('test')
test.configure(commonTestConfig(project))
test.configure {
include '**/*Tests.class'
}
// Add a method to create additional unit tests for a project, which will share the same
// randomized testing setup, but by default run no tests.
project.extensions.add('additionalTest', { String name, Closure config ->
RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
additionalTest.classpath = test.classpath
additionalTest.testClassesDir = test.testClassesDir
additionalTest.configure(commonTestConfig(project))
additionalTest.configure(config)
test.dependsOn(additionalTest)
});
return test
}

16
plugins/repository-s3/build.gradle
View File

@ -54,9 +54,21 @@ bundlePlugin {
}
}
additionalTest('testEnvCreds'){
include '**/EnvironmentCredentialsTests.class'
environment 'AWS_ACCESS_KEY_ID', 'env_access'
environment 'AWS_SECRET_ACCESS_KEY', 'env_secret'
}
additionalTest('testSyspropCreds'){
include '**/SyspropCredentialsTests.class'
systemProperty 'aws.accessKeyId', 'sysprop_access'
systemProperty 'aws.secretKey', 'sysprop_secret'
}
test {
// this is needed for insecure plugins, remove if possible!
systemProperty 'tests.artifact', project.name
// these are tested explicitly in separate test tasks
exclude '**/*CredentialsTests.class'
}
thirdPartyAudit.excludes = [

47
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -21,9 +21,14 @@ package org.elasticsearch.cloud.aws;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
@ -35,6 +40,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.s3.S3Repository;
@ -43,6 +49,7 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;
import static org.elasticsearch.repositories.s3.S3Repository.getValue;
@ -62,7 +69,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
boolean useThrottleRetries, Boolean pathStyleAccess) {
String foundEndpoint = findEndpoint(logger, settings, endpoint, region);
AWSCredentialsProvider credentials = buildCredentials(logger, settings, repositorySettings);
AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings);
Tuple<String, String> clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId());
AmazonS3Client client = clients.get(clientDescriptor);
@ -126,7 +133,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return clientConfiguration;
}
public static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings, Settings repositorySettings) {
public static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger,
Settings settings, Settings repositorySettings) {
AWSCredentialsProvider credentials;
try (SecureString key = getValue(repositorySettings, settings,
S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
@ -134,9 +142,21 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) {
if (key.length() == 0 && secret.length() == 0) {
// TODO: add deprecation, except for using instance profile
logger.debug("Using either environment variables, system properties or instance profile credentials");
credentials = new DefaultAWSCredentialsProviderChain();
// create a "manual" chain of providers here, so we can log deprecation of unsupported methods
AWSCredentials envCredentials = getDeprecatedCredentials(logger, deprecationLogger,
new EnvironmentVariableCredentialsProvider(), "environment variables");
if (envCredentials != null) {
credentials = new StaticCredentialsProvider(envCredentials);
} else {
AWSCredentials syspropCredentials = getDeprecatedCredentials(logger, deprecationLogger,
new SystemPropertiesCredentialsProvider(), "system properties");
if (syspropCredentials != null) {
credentials = new StaticCredentialsProvider(syspropCredentials);
} else {
logger.debug("Using instance profile credentials");
credentials = new InstanceProfileCredentialsProvider();
}
}
} else {
logger.debug("Using basic key/secret credentials");
credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString()));
@ -146,6 +166,23 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
return credentials;
}
/** Return credentials from the given provider, or null if full credentials are not available */
private static AWSCredentials getDeprecatedCredentials(Logger logger, DeprecationLogger deprecationLogger,
AWSCredentialsProvider provider, String description) {
try {
AWSCredentials credentials = provider.getCredentials();
if (credentials.getAWSAccessKeyId() != null && credentials.getAWSSecretKey() != null) {
logger.debug("Using " + description + " credentials");
deprecationLogger.deprecated("Supplying S3 credentials through " + description + " is deprecated. " +
"See the breaking changes lists in the documentation for details.");
return credentials;
}
} catch (Exception e) {
logger.debug("Failed to get aws credentials from " + description, e);
}
return null;
}
protected static String findEndpoint(Logger logger, Settings settings, String endpoint, String region) {
if (Strings.isNullOrEmpty(endpoint)) {
logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);

9
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
View File

@ -24,6 +24,7 @@ import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.s3.S3Repository;
@ -36,8 +37,9 @@ import static org.hamcrest.Matchers.is;
public class AwsS3ServiceImplTests extends ESTestCase {
public void testAWSCredentialsWithSystemProviders() {
AWSCredentialsProvider credentialsProvider = InternalAwsS3Service.buildCredentials(logger, Settings.EMPTY, Settings.EMPTY);
assertThat(credentialsProvider, instanceOf(DefaultAWSCredentialsProviderChain.class));
AWSCredentialsProvider credentialsProvider =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
assertThat(credentialsProvider, instanceOf(InstanceProfileCredentialsProvider.class));
}
public void testAWSCredentialsWithElasticsearchAwsSettings() {
@ -251,7 +253,8 @@ public class AwsS3ServiceImplTests extends ESTestCase {
protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings,
String expectedKey, String expectedSecret) {
AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, settings, singleRepositorySettings).getCredentials();
AWSCredentials credentials =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings, singleRepositorySettings).getCredentials();
assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
}

36
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/EnvironmentCredentialsTests.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cloud.aws;
import com.amazonaws.auth.AWSCredentialsProvider;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
public class EnvironmentCredentialsTests extends ESTestCase {
public void test() {
AWSCredentialsProvider provider =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
// NOTE: env vars are setup by the test runner in gradle
assertEquals("env_access", provider.getCredentials().getAWSAccessKeyId());
assertEquals("env_secret", provider.getCredentials().getAWSSecretKey());
assertWarnings("Supplying S3 credentials through environment variables is deprecated");
}
}

35
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/SyspropCredentialsTests.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cloud.aws;
import com.amazonaws.auth.AWSCredentialsProvider;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
public class SyspropCredentialsTests extends ESTestCase {
public void test() {
AWSCredentialsProvider provider =
InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY);
// NOTE: sys props are setup by the test runner in gradle
assertEquals("sysprop_access", provider.getCredentials().getAWSAccessKeyId());
assertEquals("sysprop_secret", provider.getCredentials().getAWSSecretKey());
assertWarnings("Supplying S3 credentials through system properties is deprecated");
}
}

1
test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
View File

@ -178,6 +178,7 @@ public abstract class ESTestCase extends LuceneTestCase {
}
protected final Logger logger = Loggers.getLogger(getClass());
protected final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
private ThreadContext threadContext;
// -----------------------------------------------------------------