honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update Format, add new settings into the setting test

This commit is contained in:
Xu Zhang 2016-03-06 21:05:52 -08:00
parent 7499e3aa4a
commit 38923b89c2
8 changed files with 21 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -154,6 +154,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
}
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials);
}

10
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -62,12 +62,9 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
return client(endpoint, protocol, region, account, key, maxRetries, null);
}
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
EncryptionMaterials clientSideEncryptionMaterials) {
@Override
public AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) {
if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region
if (region != null) {
@ -144,8 +141,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
credentials,
encryptionMaterialsProvider,
clientConfiguration,
cryptoConfiguration
);
cryptoConfiguration);
} else {
client = new AmazonS3Client(credentials, clientConfiguration);
}

2
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
View File

@ -20,8 +20,6 @@
package org.elasticsearch.cloud.aws.blobstore;
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.Headers;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;

1
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -147,7 +147,6 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY);

9
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -247,7 +247,6 @@ public class S3Repository extends BlobStoreRepository {
* base_path
* @see Repositories#BASE_PATH_SETTING
*/
Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
/**
@ -319,7 +318,7 @@ public class S3Repository extends BlobStoreRepository {
String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
logger.debug("using bqucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
@ -327,7 +326,7 @@ public class S3Repository extends BlobStoreRepository {
// parse and validate the client side encryption setting
String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY);
String publicKeyBase64 =getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
String publicKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY);
EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name);
@ -366,7 +365,7 @@ public class S3Repository extends BlobStoreRepository {
EncryptionMaterials clientSideEncryptionMaterials = null;
if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) {
throw new RepositoryException(name.name(), "Client-side encryption: You can't specify an symmetric key AND a public/private key pair");
throw new RepositoryException(name.name(), "Client-side encryption: You can't specify a symmetric key AND a public/private key pair");
}
if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) {
@ -380,7 +379,7 @@ public class S3Repository extends BlobStoreRepository {
if (Strings.isNullOrEmpty(symmetricKey) == false) {
clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES"));
} else {
if (Strings.isNullOrEmpty(publicKey)|| Strings.isNullOrEmpty(privateKey)){
if (Strings.isNullOrEmpty(publicKey) || Strings.isNullOrEmpty(privateKey)) {
String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key";
throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing");
}

8
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
View File

@ -97,6 +97,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
.put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class")
.put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl")
.put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath")
.build();
/**
@ -125,6 +126,9 @@ public class RepositoryS3SettingsTests extends ESTestCase {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
}
/**
@ -153,6 +157,9 @@ public class RepositoryS3SettingsTests extends ESTestCase {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
}
/**
@ -329,6 +336,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
throws IOException {
Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder()
.put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)

5
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.cloud.aws;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -51,8 +52,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, clientSideEncryptionMaterials));
}
private AmazonS3 cachedWrapper(AmazonS3 client) {

5
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -104,7 +104,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(settings
).get();
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
createIndex("test-idx-1", "test-idx-2", "test-idx-3");
@ -209,6 +209,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
S3Repository.Repositories.REGION_SETTING.get(settings),
S3Repository.Repositories.KEY_SETTING.get(settings),
S3Repository.Repositories.SECRET_SETTING.get(settings),
null,
null);
String bucketName = bucket.get("bucket");
@ -553,7 +554,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null, null);
try {
ObjectListing prevListing = null;
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html