honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16843 from xuzha/s3-encryption 

S3 client side encryption
This commit is contained in:
Xu Zhang 2016-03-24 12:58:40 -07:00
parent 08903f1ed8 38923b89c2
commit 37a183d9ed
10 changed files with 252 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/plugins/repository-s3.asciidoc
View File

@ -236,6 +236,16 @@ The following settings are supported:
currently supported by the plugin. For more information about the
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
`client_symmetric_key`::
Sets the keys to use to encrypt your snapshots. You can specify either a symmetric key or a public/private key pair.
No encryption by default. This sets a Base64-encoded AES symmetric-key (128, 192 or 256 bits)
`client_public_key`::
Sets the a base64-encoded RSA public key
`client_private_key`::
Sets the a base64-encoded RSA private key
The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

3
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.cloud.aws;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
@ -153,5 +154,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
}
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials);
}

33
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -31,6 +31,11 @@ import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
@ -49,7 +54,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
/**
* (acceskey, endpoint) -&gt; client
*/
private Map<Tuple<String, String>, AmazonS3Client> clients = new HashMap<>();
private Map<Tuple<String, Tuple<String, EncryptionMaterials>>, AmazonS3Client> clients = new HashMap<>();
@Inject
public InternalAwsS3Service(Settings settings) {
@ -57,7 +62,9 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
EncryptionMaterials clientSideEncryptionMaterials) {
if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region
if (region != null) {
@ -69,11 +76,14 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
}
}
return getClient(endpoint, protocol, account, key, maxRetries);
return getClient(endpoint, protocol, account, key, maxRetries, clientSideEncryptionMaterials);
}
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) {
Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
EncryptionMaterials clientSideEncryptionMaterials) {
Tuple<String, EncryptionMaterials> tempTuple = new Tuple<>(account, clientSideEncryptionMaterials);
Tuple<String, Tuple<String, EncryptionMaterials>> clientDescriptor = new Tuple<>(endpoint, tempTuple);
AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) {
return client;
@ -123,7 +133,18 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
new StaticCredentialsProvider(new BasicAWSCredentials(account, key))
);
}
client = new AmazonS3Client(credentials, clientConfiguration);
if (clientSideEncryptionMaterials != null) {
EncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider(clientSideEncryptionMaterials);
CryptoConfiguration cryptoConfiguration = new CryptoConfiguration();
client = new AmazonS3EncryptionClient(
credentials,
encryptionMaterialsProvider,
clientConfiguration,
cryptoConfiguration);
} else {
client = new AmazonS3Client(credentials, clientConfiguration);
}
if (endpoint != null) {
client.setEndpoint(endpoint);

25
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
View File

@ -131,31 +131,10 @@ public class DefaultS3OutputStream extends S3OutputStream {
}
md.setContentLength(length);
InputStream inputStream = is;
// We try to compute a MD5 while reading it
MessageDigest messageDigest;
try {
messageDigest = MessageDigest.getInstance("MD5");
inputStream = new DigestInputStream(is, messageDigest);
} catch (NoSuchAlgorithmException impossible) {
// Every implementation of the Java platform is required to support MD5 (see MessageDigest)
throw new RuntimeException(impossible);
}
PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, inputStream, md)
PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, is, md)
.withStorageClass(blobStore.getStorageClass())
.withCannedAcl(blobStore.getCannedACL());
PutObjectResult putObjectResult = blobStore.client().putObject(putRequest);
String localMd5 = Base64.encodeAsString(messageDigest.digest());
String remoteMd5 = putObjectResult.getContentMd5();
if (!localMd5.equals(remoteMd5)) {
logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5);
throw new AmazonS3Exception("MD5 local [" + localMd5 +
"], remote [" + remoteMd5 +
"] are not equal...");
}
blobStore.client().putObject(putRequest);
}
private void initializeMultipart() {

7
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.cloud.aws.blobstore;
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CreateBucketRequest;
@ -70,6 +71,12 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
this.region = region;
this.serverSideEncryption = serverSideEncryption;
this.bufferSize = bufferSize;
if (client instanceof AmazonS3EncryptionClient && this.bufferSize.getBytes() % 16 > 0) {
throw new BlobStoreException("Detected client-side encryption " +
"and a buffer_size for the S3 storage not a multiple of the cipher block size (16)");
}
this.cannedACL = initCannedACL(cannedACL);
this.numberOfRetries = maxRetries;
this.storageClass = initStorageClass(storageClass);

6
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -128,6 +128,9 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_SYMMETRIC_KEY);
// Register S3 single repository settings
settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
@ -144,6 +147,9 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY);
}
/**

109
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.repositories.s3;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import com.amazonaws.util.Base64;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
@ -37,7 +39,15 @@ import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.security.KeyFactory;
import java.security.KeyPair;
import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Locale;
import java.util.function.Function;
@ -140,6 +150,21 @@ public class S3Repository extends BlobStoreRepository {
* repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory.
*/
Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope);
/**
* repositories.s3.client_symmetric_key: Specifies the Base64-encoded AES symmetric-key (128, 192 or 256 bits)
*/
Setting<String> CLIENT_SYMMETRIC_KEY = Setting.simpleString("repositories.s3.client_symmetric_key", Property.NodeScope);
/**
* repositories.s3.client_public_key: Specifies the Base64-encoded RSA public key
*/
Setting<String> CLIENT_PUBLIC_KEY = Setting.simpleString("repositories.s3.client_public_key", Property.NodeScope);
/**
* repositories.s3.client_private_key: Specifies the Base64-encoded RSA private key
*/
Setting<String> CLIENT_PRIVATE_KEY = Setting.simpleString("repositories.s3.client_private_key", Property.NodeScope);
}
/**
@ -223,6 +248,24 @@ public class S3Repository extends BlobStoreRepository {
* @see Repositories#BASE_PATH_SETTING
*/
Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
/**
* base_path
* @see Repositories#CLIENT_SYMMETRIC_KEY
*/
Setting<String> CLIENT_SYMMETRIC_KEY = Setting.simpleString("client_symmetric_key", Property.NodeScope);
/**
* base_path
* @see Repositories#CLIENT_PUBLIC_KEY
*/
Setting<String> CLIENT_PUBLIC_KEY = Setting.simpleString("client_public_key", Property.NodeScope);
/**
* base_path
* @see Repositories#CLIENT_PRIVATE_KEY
*/
Setting<String> CLIENT_PRIVATE_KEY = Setting.simpleString("client_private_key", Property.NodeScope);
}
private final S3BlobStore blobStore;
@ -281,8 +324,24 @@ public class S3Repository extends BlobStoreRepository {
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
// parse and validate the client side encryption setting
String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY);
String publicKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY);
EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name);
blobStore = new S3BlobStore(
settings,
s3Service.client(endpoint, protocol, region, key, secret, maxRetries, clientSideEncryptionMaterials),
bucket,
region,
serverSideEncryption,
bufferSize,
maxRetries,
cannedACL,
storageClass
);
String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
if (Strings.hasLength(basePath)) {
@ -294,6 +353,52 @@ public class S3Repository extends BlobStoreRepository {
} else {
this.basePath = BlobPath.cleanPath();
}
}
/**
* Init and verify initClientSideEncryption settings
*/
private EncryptionMaterials initClientSideEncryption(String symmetricKey, String publicKey, String privateKey, RepositoryName name) {
EncryptionMaterials clientSideEncryptionMaterials = null;
if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) {
throw new RepositoryException(name.name(), "Client-side encryption: You can't specify a symmetric key AND a public/private key pair");
}
if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) {
try {
// Check crypto
if (Cipher.getMaxAllowedKeyLength("AES") < 256) {
throw new RepositoryException(name.name(), "Client-side encryption: Please install the Java Cryptography Extension");
}
// Transform the keys in a EncryptionMaterials
if (Strings.isNullOrEmpty(symmetricKey) == false) {
clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES"));
} else {
if (Strings.isNullOrEmpty(publicKey) || Strings.isNullOrEmpty(privateKey)) {
String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key";
throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing");
}
clientSideEncryptionMaterials = new EncryptionMaterials(new KeyPair(
KeyFactory.getInstance("RSA").generatePublic(new X509EncodedKeySpec(Base64.decode(publicKey))),
KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(Base64.decode(privateKey)))));
}
} catch (IllegalArgumentException e) {
throw new RepositoryException(name.name(), "Client-side encryption: Error decoding your keys: " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
throw new RepositoryException(name.name(), e.getMessage());
} catch (InvalidKeySpecException e) {
throw new RepositoryException(name.name(), e.getMessage());
}
}
return clientSideEncryptionMaterials;
}
/**

8
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
View File

@ -97,6 +97,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
.put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class")
.put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl")
.put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath")
.build();
/**
@ -125,6 +126,9 @@ public class RepositoryS3SettingsTests extends ESTestCase {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
}
/**
@ -153,6 +157,9 @@ public class RepositoryS3SettingsTests extends ESTestCase {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
}
/**
@ -329,6 +336,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
throws IOException {
Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder()
.put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)

5
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.cloud.aws;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -51,8 +52,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, clientSideEncryptionMaterials));
}
private AmazonS3 cachedWrapper(AmazonS3 client) {

81
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -24,6 +24,8 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.util.Base64;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@ -33,6 +35,7 @@ import org.elasticsearch.cloud.aws.AbstractAwsTestCase;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.repositories.RepositoryVerificationException;
import org.elasticsearch.snapshots.SnapshotMissingException;
@ -42,7 +45,13 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import org.junit.Before;
import javax.crypto.KeyGenerator;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.hamcrest.Matchers.equalTo;
@ -200,6 +209,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
S3Repository.Repositories.REGION_SETTING.get(settings),
S3Repository.Repositories.KEY_SETTING.get(settings),
S3Repository.Repositories.SECRET_SETTING.get(settings),
null,
null);
String bucketName = bucket.get("bucket");
@ -399,6 +409,75 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
}
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
public void testClientSideEncryption() throws NoSuchAlgorithmException {
KeyGenerator keyGenerator1 = KeyGenerator.getInstance("AES");
keyGenerator1.init(128);
String symmetricEncryptionKeyBase64 = Base64.encodeAsString(keyGenerator1.generateKey().getEncoded());
KeyPairGenerator keyGenerator2 = KeyPairGenerator.getInstance("RSA");
keyGenerator2.initialize(512, new SecureRandom());
KeyPair keyPair = keyGenerator2.generateKeyPair();
String publicEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPublic().getEncoded());
String privateEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPrivate().getEncoded());
Client client = client();
try {
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(Settings.settingsBuilder()
.put("base_path", basePath)
.put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64)
.put("client_side_encryption_key.public", publicEncryptionKeyBase64)
.put("client_side_encryption_key.private", privateEncryptionKeyBase64)
.put("chunk_size", randomIntBetween(1000, 10000))
).get();
fail("Symmetric and public/private key pairs are exclusive options. An exception should be thrown.");
} catch (RepositoryException e) {
}
List<Settings.Builder> allSettings = Arrays.asList(
Settings.settingsBuilder()
.put("base_path", basePath)
.put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64)
.put("chunk_size", randomIntBetween(1000, 10000)),
Settings.settingsBuilder()
.put("base_path", basePath)
.put("client_side_encryption_key.public", publicEncryptionKeyBase64)
.put("client_side_encryption_key.private", privateEncryptionKeyBase64)
.put("chunk_size", randomIntBetween(1000, 10000))
);
for (Settings.Builder settings : allSettings) {
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(settings).get();
// Create the index and index some data
createIndex("test-idx-1");
for (int i = 0; i < 100; i++) {
index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
}
refresh();
// Take the snapshot
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
// Restore
cluster().wipeIndices("test-idx-1");
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet();
ensureGreen();
assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
// Clean, the test will bbe run with different settings
cluster().wipeIndices("test-idx-1");
wipeRepositories();
cleanRepositoryFiles(basePath);
}
}
private void assertRepositoryIsOperational(Client client, String repository) {
createIndex("test-idx-1");
ensureGreen();
@ -475,7 +554,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null, null);
try {
ObjectListing prevListing = null;
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html