honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge pull request #16843 from xuzha/s3-encryption" 

This reverts commit 37a183d9ed, reversing
changes made to 08903f1ed8.
This commit is contained in:
Jason Tedor 2016-03-24 17:11:02 -04:00
parent 37a183d9ed
commit 7f0134e725
10 changed files with 35 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/plugins/repository-s3.asciidoc
View File

@ -236,16 +236,6 @@ The following settings are supported:
currently supported by the plugin. For more information about the currently supported by the plugin. For more information about the
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
`client_symmetric_key`::
Sets the keys to use to encrypt your snapshots. You can specify either a symmetric key or a public/private key pair.
No encryption by default. This sets a Base64-encoded AES symmetric-key (128, 192 or 256 bits)
`client_public_key`::
Sets the a base64-encoded RSA public key
`client_private_key`::
Sets the a base64-encoded RSA private key
The S3 repositories use the same credentials as the rest of the AWS services The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details. provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

3
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.cloud.aws;
import com.amazonaws.Protocol; import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Setting.Property;
@ -154,5 +153,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope); Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
} }
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials); AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
} }

33
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View File

@ -31,11 +31,6 @@ import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider;
import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
@ -54,7 +49,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
/** /**
* (acceskey, endpoint) -&gt; client * (acceskey, endpoint) -&gt; client
*/ */
private Map<Tuple<String, Tuple<String, EncryptionMaterials>>, AmazonS3Client> clients = new HashMap<>(); private Map<Tuple<String, String>, AmazonS3Client> clients = new HashMap<>();
@Inject @Inject
public InternalAwsS3Service(Settings settings) { public InternalAwsS3Service(Settings settings) {
@ -62,9 +57,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
} }
@Override @Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
EncryptionMaterials clientSideEncryptionMaterials) {
if (Strings.isNullOrEmpty(endpoint)) { if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region // We need to set the endpoint based on the region
if (region != null) { if (region != null) {
@ -76,14 +69,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
} }
} }
return getClient(endpoint, protocol, account, key, maxRetries, clientSideEncryptionMaterials); return getClient(endpoint, protocol, account, key, maxRetries);
} }
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries, private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) {
EncryptionMaterials clientSideEncryptionMaterials) { Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
Tuple<String, EncryptionMaterials> tempTuple = new Tuple<>(account, clientSideEncryptionMaterials);
Tuple<String, Tuple<String, EncryptionMaterials>> clientDescriptor = new Tuple<>(endpoint, tempTuple);
AmazonS3Client client = clients.get(clientDescriptor); AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) { if (client != null) {
return client; return client;
@ -133,18 +123,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
new StaticCredentialsProvider(new BasicAWSCredentials(account, key)) new StaticCredentialsProvider(new BasicAWSCredentials(account, key))
); );
} }
client = new AmazonS3Client(credentials, clientConfiguration);
if (clientSideEncryptionMaterials != null) {
EncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider(clientSideEncryptionMaterials);
CryptoConfiguration cryptoConfiguration = new CryptoConfiguration();
client = new AmazonS3EncryptionClient(
credentials,
encryptionMaterialsProvider,
clientConfiguration,
cryptoConfiguration);
} else {
client = new AmazonS3Client(credentials, clientConfiguration);
}
if (endpoint != null) { if (endpoint != null) {
client.setEndpoint(endpoint); client.setEndpoint(endpoint);

25
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
View File

@ -131,10 +131,31 @@ public class DefaultS3OutputStream extends S3OutputStream {
} }
md.setContentLength(length); md.setContentLength(length);
PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, is, md) InputStream inputStream = is;
// We try to compute a MD5 while reading it
MessageDigest messageDigest;
try {
messageDigest = MessageDigest.getInstance("MD5");
inputStream = new DigestInputStream(is, messageDigest);
} catch (NoSuchAlgorithmException impossible) {
// Every implementation of the Java platform is required to support MD5 (see MessageDigest)
throw new RuntimeException(impossible);
}
PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, inputStream, md)
.withStorageClass(blobStore.getStorageClass()) .withStorageClass(blobStore.getStorageClass())
.withCannedAcl(blobStore.getCannedACL()); .withCannedAcl(blobStore.getCannedACL());
blobStore.client().putObject(putRequest); PutObjectResult putObjectResult = blobStore.client().putObject(putRequest);
String localMd5 = Base64.encodeAsString(messageDigest.digest());
String remoteMd5 = putObjectResult.getContentMd5();
if (!localMd5.equals(remoteMd5)) {
logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5);
throw new AmazonS3Exception("MD5 local [" + localMd5 +
"], remote [" + remoteMd5 +
"] are not equal...");
}
} }
private void initializeMultipart() { private void initializeMultipart() {

7
plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.cloud.aws.blobstore;
import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.CreateBucketRequest;
@ -71,12 +70,6 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
this.region = region; this.region = region;
this.serverSideEncryption = serverSideEncryption; this.serverSideEncryption = serverSideEncryption;
this.bufferSize = bufferSize; this.bufferSize = bufferSize;
if (client instanceof AmazonS3EncryptionClient && this.bufferSize.getBytes() % 16 > 0) {
throw new BlobStoreException("Detected client-side encryption " +
"and a buffer_size for the S3 storage not a multiple of the cipher block size (16)");
}
this.cannedACL = initCannedACL(cannedACL); this.cannedACL = initCannedACL(cannedACL);
this.numberOfRetries = maxRetries; this.numberOfRetries = maxRetries;
this.storageClass = initStorageClass(storageClass); this.storageClass = initStorageClass(storageClass);

6
plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View File

@ -128,9 +128,6 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING); settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING); settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING); settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_SYMMETRIC_KEY);
// Register S3 single repository settings // Register S3 single repository settings
settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING); settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
@ -147,9 +144,6 @@ public class S3RepositoryPlugin extends Plugin {
settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING); settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING); settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING); settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY);
} }
/** /**

109
plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View File

@ -20,8 +20,6 @@
package org.elasticsearch.repositories.s3; package org.elasticsearch.repositories.s3;
import com.amazonaws.Protocol; import com.amazonaws.Protocol;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import com.amazonaws.util.Base64;
import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3; import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
import org.elasticsearch.cloud.aws.blobstore.S3BlobStore; import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
@ -39,15 +37,7 @@ import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException; import java.io.IOException;
import java.security.KeyFactory;
import java.security.KeyPair;
import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Locale; import java.util.Locale;
import java.util.function.Function; import java.util.function.Function;
@ -150,21 +140,6 @@ public class S3Repository extends BlobStoreRepository {
* repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory. * repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory.
*/ */
Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope); Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope);
/**
* repositories.s3.client_symmetric_key: Specifies the Base64-encoded AES symmetric-key (128, 192 or 256 bits)
*/
Setting<String> CLIENT_SYMMETRIC_KEY = Setting.simpleString("repositories.s3.client_symmetric_key", Property.NodeScope);
/**
* repositories.s3.client_public_key: Specifies the Base64-encoded RSA public key
*/
Setting<String> CLIENT_PUBLIC_KEY = Setting.simpleString("repositories.s3.client_public_key", Property.NodeScope);
/**
* repositories.s3.client_private_key: Specifies the Base64-encoded RSA private key
*/
Setting<String> CLIENT_PRIVATE_KEY = Setting.simpleString("repositories.s3.client_private_key", Property.NodeScope);
} }
/** /**
@ -248,24 +223,6 @@ public class S3Repository extends BlobStoreRepository {
* @see Repositories#BASE_PATH_SETTING * @see Repositories#BASE_PATH_SETTING
*/ */
Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope); Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
/**
* base_path
* @see Repositories#CLIENT_SYMMETRIC_KEY
*/
Setting<String> CLIENT_SYMMETRIC_KEY = Setting.simpleString("client_symmetric_key", Property.NodeScope);
/**
* base_path
* @see Repositories#CLIENT_PUBLIC_KEY
*/
Setting<String> CLIENT_PUBLIC_KEY = Setting.simpleString("client_public_key", Property.NodeScope);
/**
* base_path
* @see Repositories#CLIENT_PRIVATE_KEY
*/
Setting<String> CLIENT_PRIVATE_KEY = Setting.simpleString("client_private_key", Property.NodeScope);
} }
private final S3BlobStore blobStore; private final S3BlobStore blobStore;
@ -324,24 +281,8 @@ public class S3Repository extends BlobStoreRepository {
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING); String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING); String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
// parse and validate the client side encryption setting blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries),
String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY); bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
String publicKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY);
EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name);
blobStore = new S3BlobStore(
settings,
s3Service.client(endpoint, protocol, region, key, secret, maxRetries, clientSideEncryptionMaterials),
bucket,
region,
serverSideEncryption,
bufferSize,
maxRetries,
cannedACL,
storageClass
);
String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING); String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
if (Strings.hasLength(basePath)) { if (Strings.hasLength(basePath)) {
@ -353,52 +294,6 @@ public class S3Repository extends BlobStoreRepository {
} else { } else {
this.basePath = BlobPath.cleanPath(); this.basePath = BlobPath.cleanPath();
} }
}
/**
* Init and verify initClientSideEncryption settings
*/
private EncryptionMaterials initClientSideEncryption(String symmetricKey, String publicKey, String privateKey, RepositoryName name) {
EncryptionMaterials clientSideEncryptionMaterials = null;
if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) {
throw new RepositoryException(name.name(), "Client-side encryption: You can't specify a symmetric key AND a public/private key pair");
}
if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) {
try {
// Check crypto
if (Cipher.getMaxAllowedKeyLength("AES") < 256) {
throw new RepositoryException(name.name(), "Client-side encryption: Please install the Java Cryptography Extension");
}
// Transform the keys in a EncryptionMaterials
if (Strings.isNullOrEmpty(symmetricKey) == false) {
clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES"));
} else {
if (Strings.isNullOrEmpty(publicKey) || Strings.isNullOrEmpty(privateKey)) {
String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key";
throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing");
}
clientSideEncryptionMaterials = new EncryptionMaterials(new KeyPair(
KeyFactory.getInstance("RSA").generatePublic(new X509EncodedKeySpec(Base64.decode(publicKey))),
KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(Base64.decode(privateKey)))));
}
} catch (IllegalArgumentException e) {
throw new RepositoryException(name.name(), "Client-side encryption: Error decoding your keys: " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
throw new RepositoryException(name.name(), e.getMessage());
} catch (InvalidKeySpecException e) {
throw new RepositoryException(name.name(), e.getMessage());
}
}
return clientSideEncryptionMaterials;
} }
/** /**

8
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
View File

@ -97,7 +97,6 @@ public class RepositoryS3SettingsTests extends ESTestCase {
.put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class") .put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class")
.put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl") .put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl")
.put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath") .put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath")
.build(); .build();
/** /**
@ -126,9 +125,6 @@ public class RepositoryS3SettingsTests extends ESTestCase {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
} }
/** /**
@ -157,9 +153,6 @@ public class RepositoryS3SettingsTests extends ESTestCase {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
} }
/** /**
@ -336,7 +329,6 @@ public class RepositoryS3SettingsTests extends ESTestCase {
private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage) private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
throws IOException { throws IOException {
Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES); Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder() RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder()
.put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer) .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)

5
plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View File

@ -20,7 +20,6 @@ package org.elasticsearch.cloud.aws;
import com.amazonaws.Protocol; import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@ -52,8 +51,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
@Override @Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, clientSideEncryptionMaterials)); return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
} }
private AmazonS3 cachedWrapper(AmazonS3 client) { private AmazonS3 cachedWrapper(AmazonS3 client) {

81
plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -24,8 +24,6 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.util.Base64;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@ -35,7 +33,6 @@ import org.elasticsearch.cloud.aws.AbstractAwsTestCase;
import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.repositories.RepositoryVerificationException;
import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotMissingException;
@ -45,13 +42,7 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import javax.crypto.KeyGenerator;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
@ -209,7 +200,6 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
S3Repository.Repositories.REGION_SETTING.get(settings), S3Repository.Repositories.REGION_SETTING.get(settings),
S3Repository.Repositories.KEY_SETTING.get(settings), S3Repository.Repositories.KEY_SETTING.get(settings),
S3Repository.Repositories.SECRET_SETTING.get(settings), S3Repository.Repositories.SECRET_SETTING.get(settings),
null,
null); null);
String bucketName = bucket.get("bucket"); String bucketName = bucket.get("bucket");
@ -409,75 +399,6 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
} }
} }
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
public void testClientSideEncryption() throws NoSuchAlgorithmException {
KeyGenerator keyGenerator1 = KeyGenerator.getInstance("AES");
keyGenerator1.init(128);
String symmetricEncryptionKeyBase64 = Base64.encodeAsString(keyGenerator1.generateKey().getEncoded());
KeyPairGenerator keyGenerator2 = KeyPairGenerator.getInstance("RSA");
keyGenerator2.initialize(512, new SecureRandom());
KeyPair keyPair = keyGenerator2.generateKeyPair();
String publicEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPublic().getEncoded());
String privateEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPrivate().getEncoded());
Client client = client();
try {
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(Settings.settingsBuilder()
.put("base_path", basePath)
.put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64)
.put("client_side_encryption_key.public", publicEncryptionKeyBase64)
.put("client_side_encryption_key.private", privateEncryptionKeyBase64)
.put("chunk_size", randomIntBetween(1000, 10000))
).get();
fail("Symmetric and public/private key pairs are exclusive options. An exception should be thrown.");
} catch (RepositoryException e) {
}
List<Settings.Builder> allSettings = Arrays.asList(
Settings.settingsBuilder()
.put("base_path", basePath)
.put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64)
.put("chunk_size", randomIntBetween(1000, 10000)),
Settings.settingsBuilder()
.put("base_path", basePath)
.put("client_side_encryption_key.public", publicEncryptionKeyBase64)
.put("client_side_encryption_key.private", privateEncryptionKeyBase64)
.put("chunk_size", randomIntBetween(1000, 10000))
);
for (Settings.Builder settings : allSettings) {
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(settings).get();
// Create the index and index some data
createIndex("test-idx-1");
for (int i = 0; i < 100; i++) {
index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
}
refresh();
// Take the snapshot
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
// Restore
cluster().wipeIndices("test-idx-1");
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet();
ensureGreen();
assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
// Clean, the test will bbe run with different settings
cluster().wipeIndices("test-idx-1");
wipeRepositories();
cleanRepositoryFiles(basePath);
}
}
private void assertRepositoryIsOperational(Client client, String repository) { private void assertRepositoryIsOperational(Client client, String repository) {
createIndex("test-idx-1"); createIndex("test-idx-1");
ensureGreen(); ensureGreen();
@ -554,7 +475,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
// We check that settings has been set in elasticsearch.yml integration test file // We check that settings has been set in elasticsearch.yml integration test file
// as described in README // as described in README
assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null, null); AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
try { try {
ObjectListing prevListing = null; ObjectListing prevListing = null;
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html