Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9520 update AWS SDK and Kinesis Client Library versions; Allow PutS3Object to use all available StorageClasses 

This closes #5632.

Signed-off-by: Peter Turcsanyi <turcsanyi@apache.org>
This commit is contained in:
Chris Sampson 2022-01-05 12:32:23 +00:00 committed by Peter Turcsanyi
parent dc42012755
commit bd1b2b4d69
4 changed files with 306 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java
View File

@ -35,10 +35,13 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.ObjectTagging;
import com.amazonaws.services.s3.model.Tag;
@ -140,6 +143,10 @@ public class PutS3Object extends AbstractS3Processor {
public static final String CONTENT_DISPOSITION_INLINE = "inline";
public static final String CONTENT_DISPOSITION_ATTACHMENT = "attachment";
private static final Set<String> STORAGE_CLASSES = Collections.unmodifiableSortedSet(new TreeSet<>(
Arrays.stream(StorageClass.values()).map(StorageClass::name).collect(Collectors.toSet())
));
public static final PropertyDescriptor EXPIRATION_RULE_ID = new PropertyDescriptor.Builder()
.name("Expiration Time Rule")
.required(false)
@ -183,9 +190,7 @@ public class PutS3Object extends AbstractS3Processor {
public static final PropertyDescriptor STORAGE_CLASS = new PropertyDescriptor.Builder()
.name("Storage Class")
.required(true)
.allowableValues(StorageClass.Standard.name(), StorageClass.IntelligentTiering.name(), StorageClass.StandardInfrequentAccess.name(),
StorageClass.OneZoneInfrequentAccess.name(), StorageClass.Glacier.name(), StorageClass.DeepArchive.name(),
StorageClass.ReducedRedundancy.name(), StorageClass.Outposts.name())
.allowableValues(STORAGE_CLASSES)
.defaultValue(StorageClass.Standard.name())
.build();

639
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeAWSGatewayApiCommon.java
View File

File diff suppressed because it is too large Load Diff

12
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeAmazonGatewayApiMock.java
View File

@ -19,6 +19,8 @@ package org.apache.nifi.processors.aws.wag;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.http.AmazonHttpClient;
import com.amazonaws.http.apache.client.impl.SdkHttpClient;
import com.amazonaws.internal.TokenBucket;
import com.amazonaws.metrics.RequestMetricCollector;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.client.methods.HttpUriRequest;
@ -36,6 +38,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -47,7 +50,6 @@ import static org.mockito.Mockito.times;
public class TestInvokeAmazonGatewayApiMock {
private TestRunner runner = null;
private InvokeAWSGatewayApi mockGetApi = null;
private SdkHttpClient mockSdkClient = null;
@Before
@ -55,8 +57,8 @@ public class TestInvokeAmazonGatewayApiMock {
mockSdkClient = Mockito.mock(SdkHttpClient.class);
ClientConfiguration clientConfig = new ClientConfiguration();
mockGetApi = new InvokeAWSGatewayApi(
new AmazonHttpClient(clientConfig, mockSdkClient, null));
InvokeAWSGatewayApi mockGetApi = new InvokeAWSGatewayApi(
new AmazonHttpClient(clientConfig, mockSdkClient, RequestMetricCollector.NONE, new TokenBucket()));
runner = TestRunners.newTestRunner(mockGetApi);
runner.setValidateExpressionUsage(false);
@ -133,7 +135,7 @@ public class TestInvokeAmazonGatewayApiMock {
final Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/plain-text");
attributes.put("Foo", "Bar");
runner.enqueue("Hello".getBytes("UTF-8"), attributes);
runner.enqueue("Hello".getBytes(StandardCharsets.UTF_8), attributes);
// execute
runner.assertValid();
runner.run(1);
@ -184,7 +186,7 @@ public class TestInvokeAmazonGatewayApiMock {
final Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/plain-text");
attributes.put("Foo", "Bar");
runner.enqueue("Hello".getBytes("UTF-8"), attributes);
runner.enqueue("Hello".getBytes(StandardCharsets.UTF_8), attributes);
// execute
runner.assertValid();
runner.run(1);

6
nifi-nar-bundles/nifi-aws-bundle/pom.xml
View File

@ -27,9 +27,9 @@
<properties>
<!-- keep AWS 1.x until NIFI-8287 -->
<aws-java-sdk-version>1.11.1016</aws-java-sdk-version>
<!-- Do not upgrade to 1.14.x+ until https://github.com/awslabs/amazon-kinesis-client/issues/796 is fixed -->
<aws-kinesis-client-library-version>1.13.3</aws-kinesis-client-library-version>
<aws-java-sdk-version>1.12.133</aws-java-sdk-version>
<!-- keep KCL 1.x until NIFI-8531 (blocked by NIFI-8287) -->
<aws-kinesis-client-library-version>1.14.7</aws-kinesis-client-library-version>
</properties>
<dependencyManagement>