From 0f4defa49affb10966f39ac5536bdf03f0205ac0 Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Fri, 9 Feb 2024 23:25:17 +0100 Subject: [PATCH] NIFI-12771 Restored Client-side KMS encryption strategy for S3 This closes #8388 Signed-off-by: David Handermann v --- .../ClientSideCEncryptionStrategy.java | 4 +- .../ClientSideKMSEncryptionStrategy.java | 62 +++++++++++++++++++ .../StandardS3EncryptionService.java | 1 + .../TestS3EncryptionStrategies.java | 38 +++++++++--- 4 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideKMSEncryptionStrategy.java diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideCEncryptionStrategy.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideCEncryptionStrategy.java index 0e69c87d82..aeaa5d6cc9 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideCEncryptionStrategy.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideCEncryptionStrategy.java @@ -60,10 +60,10 @@ public class ClientSideCEncryptionStrategy implements S3EncryptionStrategy { final StaticEncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider(new EncryptionMaterials(symmetricKey)); final CryptoConfigurationV2 cryptoConfig = new CryptoConfigurationV2(); - cryptoConfig.setAwsKmsRegion(Region.getRegion(Regions.fromName(kmsRegion))); + // A placeholder KMS Region needs to be set due to bug https://github.com/aws/aws-sdk-java/issues/2530 + cryptoConfig.setAwsKmsRegion(Region.getRegion(Regions.DEFAULT_REGION)); final AmazonS3EncryptionClientV2Builder builder = AmazonS3EncryptionClientV2.encryptionBuilder() - .disableChunkedEncoding() .withCryptoConfiguration(cryptoConfig) .withEncryptionMaterialsProvider(encryptionMaterialsProvider); clientBuilder.accept(builder); diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideKMSEncryptionStrategy.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideKMSEncryptionStrategy.java new file mode 100644 index 0000000000..59187196ff --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/ClientSideKMSEncryptionStrategy.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.aws.s3.encryption; + +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Builder; +import com.amazonaws.services.s3.AmazonS3EncryptionClientV2; +import com.amazonaws.services.s3.AmazonS3EncryptionClientV2Builder; +import com.amazonaws.services.s3.model.CryptoConfigurationV2; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; +import org.apache.commons.lang3.StringUtils; + +import java.util.function.Consumer; + +/** + * This strategy uses KMS key id to perform client-side encryption. Use this strategy when you want the client to perform the encryption, + * (thus incurring the cost of processing) and manage the key in a KMS instance. + * + * See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro + * + */ +public class ClientSideKMSEncryptionStrategy implements S3EncryptionStrategy { + /** + * Create an encryption client. + * + * @param clientBuilder A consumer that is responsible for configuring the client builder + * @param kmsRegion AWS KMS region + * @param keyIdOrMaterial KMS key id + * @return AWS S3 client + */ + @Override + public AmazonS3 createEncryptionClient(final Consumer> clientBuilder, final String kmsRegion, final String keyIdOrMaterial) { + final KMSEncryptionMaterialsProvider encryptionMaterialsProvider = new KMSEncryptionMaterialsProvider(keyIdOrMaterial); + + final CryptoConfigurationV2 cryptoConfig = new CryptoConfigurationV2(); + if (StringUtils.isNotBlank(kmsRegion)) { + cryptoConfig.setAwsKmsRegion(Region.getRegion(Regions.fromName(kmsRegion))); + } + + final AmazonS3EncryptionClientV2Builder builder = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(encryptionMaterialsProvider); + clientBuilder.accept(builder); + return builder.build(); + } +} diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/StandardS3EncryptionService.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/StandardS3EncryptionService.java index a311e7e368..c47911abd5 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/StandardS3EncryptionService.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/encryption/StandardS3EncryptionService.java @@ -62,6 +62,7 @@ public class StandardS3EncryptionService extends AbstractControllerService imple STRATEGY_NAME_SSE_S3, new ServerSideS3EncryptionStrategy(), STRATEGY_NAME_SSE_KMS, new ServerSideKMSEncryptionStrategy(), STRATEGY_NAME_SSE_C, new ServerSideCEncryptionStrategy(), + STRATEGY_NAME_CSE_KMS, new ClientSideKMSEncryptionStrategy(), STRATEGY_NAME_CSE_C, new ClientSideCEncryptionStrategy() ); diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/encryption/TestS3EncryptionStrategies.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/encryption/TestS3EncryptionStrategies.java index 7c4b4b0dc1..b7d2e760b6 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/encryption/TestS3EncryptionStrategies.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/encryption/TestS3EncryptionStrategies.java @@ -35,6 +35,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; public class TestS3EncryptionStrategies { + private static final String REGION = Regions.DEFAULT_REGION.getName(); + private static final String KEY_ID = "key-id"; + private String randomKeyMaterial = ""; private ObjectMetadata metadata = null; @@ -63,8 +66,30 @@ public class TestS3EncryptionStrategies { // This shows that the strategy builds a client: assertNotNull(strategy.createEncryptionClient(builder -> { - builder.withRegion(Regions.DEFAULT_REGION.name()); - }, Regions.DEFAULT_REGION.getName(), randomKeyMaterial)); + builder.withRegion(REGION); + }, null, randomKeyMaterial)); + + // This shows that the strategy does not modify the metadata or any of the requests: + assertNull(metadata.getSSEAlgorithm()); + assertNull(putObjectRequest.getSSEAwsKeyManagementParams()); + assertNull(putObjectRequest.getSSECustomerKey()); + + assertNull(initUploadRequest.getSSEAwsKeyManagementParams()); + assertNull(initUploadRequest.getSSECustomerKey()); + + assertNull(getObjectRequest.getSSECustomerKey()); + + assertNull(uploadPartRequest.getSSECustomerKey()); + } + + @Test + public void testClientSideKMSEncryptionStrategy() { + S3EncryptionStrategy strategy = new ClientSideKMSEncryptionStrategy(); + + // This shows that the strategy builds a client: + assertNotNull(strategy.createEncryptionClient(builder -> { + builder.withRegion(REGION); + }, REGION, KEY_ID)); // This shows that the strategy does not modify the metadata or any of the requests: assertNull(metadata.getSSEAlgorithm()); @@ -117,15 +142,14 @@ public class TestS3EncryptionStrategies { assertNull(strategy.createEncryptionClient(null, null, null)); // This shows that the strategy sets the SSE KMS key id as expected: - String randomKeyId = "mock-key-id"; - strategy.configurePutObjectRequest(putObjectRequest, metadata, randomKeyId); - assertEquals(randomKeyId, putObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId()); + strategy.configurePutObjectRequest(putObjectRequest, metadata, KEY_ID); + assertEquals(KEY_ID, putObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId()); assertNull(putObjectRequest.getSSECustomerKey()); assertNull(metadata.getSSEAlgorithm()); // Same for InitiateMultipartUploadRequest: - strategy.configureInitiateMultipartUploadRequest(initUploadRequest, metadata, randomKeyId); - assertEquals(randomKeyId, initUploadRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId()); + strategy.configureInitiateMultipartUploadRequest(initUploadRequest, metadata, KEY_ID); + assertEquals(KEY_ID, initUploadRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId()); assertNull(initUploadRequest.getSSECustomerKey()); assertNull(metadata.getSSEAlgorithm()); }