From 77de51df19b768ea1180d52e0e6106629161add0 Mon Sep 17 00:00:00 2001 From: Venkatesh Sellappa Date: Sat, 7 Nov 2015 21:09:18 -0500 Subject: [PATCH] NIFI-1025 Updated aws-sdk and joda-time to resolve AWS issues with Java 1.8 update 60. - Refactored tests - created AbstractS3Test for common utility methods - Corrected incorrect unit test in TestDeleteS3Object, and adjusted processor documentation to reflect behavior - moved aws dependency management to root pom This closes #107 Tested, Reviewed and Amended by Tony Kurc () --- .../processors/aws/s3/DeleteS3Object.java | 3 +- .../processors/aws/s3/AbstractS3Test.java | 151 ++++++++++++++++++ .../processors/aws/s3/TestDeleteS3Object.java | 120 +++++--------- .../processors/aws/s3/TestFetchS3Object.java | 61 +++++-- .../processors/aws/s3/TestPutS3Object.java | 42 +++-- nifi-nar-bundles/nifi-aws-bundle/pom.xml | 10 -- pom.xml | 7 +- 7 files changed, 277 insertions(+), 117 deletions(-) create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java index 836e0d88ea..056ff01ad3 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java @@ -41,7 +41,7 @@ import org.apache.nifi.processor.util.StandardValidators; @SeeAlso({PutS3Object.class}) @Tags({"Amazon", "S3", "AWS", "Archive", "Delete"}) @CapabilityDescription("Deletes FlowFiles on an Amazon S3 Bucket. " + - "And the FlowFiles are checked if exists or not before deleting.") + "If attempting to delete a file that does not exist, FlowFile is routed to success.") public class DeleteS3Object extends AbstractS3Processor { public static final PropertyDescriptor VERSION_ID = new PropertyDescriptor.Builder() @@ -80,6 +80,7 @@ public class DeleteS3Object extends AbstractS3Processor { try { if (versionId == null) { final DeleteObjectRequest r = new DeleteObjectRequest(bucket, key); + // This call returns success if object doesn't exist s3.deleteObject(r); } else { final DeleteVersionRequest r = new DeleteVersionRequest(bucket, key, versionId); diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java new file mode 100644 index 0000000000..167c16b2ad --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.aws.s3; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; + +import org.apache.nifi.util.file.FileUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; + +import static org.junit.Assert.fail; + +import com.amazonaws.auth.PropertiesCredentials; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.DeleteBucketRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * Base class for S3 tests. Establishes a bucket and helper methods for creating test scenarios + */ +public abstract class AbstractS3Test { + protected final static String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties"; + protected final static String BUCKET_NAME = "test-bucket-00000000-0000-0000-0000-123456789021"; + protected final static String SAMPLE_FILE_RESOURCE_NAME = "/hello.txt"; + protected final static String REGION = "eu-west-1"; + + // Static so multiple Tests can use same client + protected static AmazonS3Client client; + + @BeforeClass + public static void oneTimeSetup() { + // Creates a client and bucket for this test + + final FileInputStream fis; + try { + fis = new FileInputStream(CREDENTIALS_FILE); + } catch (FileNotFoundException e1) { + fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e1.getLocalizedMessage()); + return; + } + try { + final PropertiesCredentials credentials = new PropertiesCredentials(fis); + client = new AmazonS3Client(credentials); + + if (client.doesBucketExist(BUCKET_NAME)) { + fail("Bucket " + BUCKET_NAME + " exists. Choose a different bucket name to continue test"); + } + + CreateBucketRequest request = new CreateBucketRequest(BUCKET_NAME, REGION); + client.createBucket(request); + + } catch (final AmazonS3Exception e) { + fail("Can't create the key " + BUCKET_NAME + ": " + e.getLocalizedMessage()); + } catch (final IOException e) { + fail("Caught IOException preparing tests: " + e.getLocalizedMessage()); + } finally { + FileUtils.closeQuietly(fis); + } + + if (!client.doesBucketExist(BUCKET_NAME)) { + fail("Setup incomplete, tests will fail"); + } + } + + @AfterClass + public static void oneTimeTearDown() { + // Empty the bucket before deleting it. + try { + ObjectListing objectListing = client.listObjects(BUCKET_NAME); + + while (true) { + for (Iterator iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) { + S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next(); + client.deleteObject(BUCKET_NAME, objectSummary.getKey()); + } + + if (objectListing.isTruncated()) { + objectListing = client.listNextBatchOfObjects(objectListing); + } else { + break; + } + } + + DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME); + client.deleteBucket(dbr); + } catch (final AmazonS3Exception e) { + System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString()); + } + + if (client.doesBucketExist(BUCKET_NAME)) { + Assert.fail("Incomplete teardown, subsequent tests might fail"); + } + + } + + protected void putTestFile(String key, File file) throws AmazonS3Exception { + PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, file); + + client.putObject(putRequest); + } + + protected Path getResourcePath(String resourceName) { + Path path = null; + + try { + path = Paths.get(getClass().getResource(resourceName).toURI()); + } catch (URISyntaxException e) { + Assert.fail("Resource: " + resourceName + " does not exist" + e.getLocalizedMessage()); + } + + return path; + } + + protected File getFileFromResourceName(String resourceName) { + URI uri = null; + try { + uri = this.getClass().getResource(resourceName).toURI(); + } catch (URISyntaxException e) { + Assert.fail("Cannot proceed without File : " + resourceName); + } + + return new File(uri); + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java index 509a31fe1c..643aa11c57 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java @@ -16,77 +16,33 @@ */ package org.apache.nifi.processors.aws.s3; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.net.URL; import java.util.HashMap; import java.util.Map; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -import com.amazonaws.auth.PropertiesCredentials; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.DeleteBucketRequest; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; - +import org.junit.Ignore; +import org.junit.Test; @Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created") -public class TestDeleteS3Object { - - private static final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties"; - - // When you want to test this, you should create a bucket on Amazon S3 as follows. - private static final String TEST_REGION = "ap-northeast-1"; - private static final String TEST_BUCKET = "test-bucket-00000000-0000-0000-0000-1234567890123"; - - @BeforeClass - public static void oneTimeSetUp() { - // Creates a new bucket for this test - try { - PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE)); - AmazonS3Client client = new AmazonS3Client(credentials); - CreateBucketRequest request = new CreateBucketRequest(TEST_BUCKET, TEST_REGION); - client.createBucket(request); - } catch (final AmazonS3Exception e) { - System.out.println("Can't create the key " + TEST_BUCKET + ":" + e.toString()); - } catch (final IOException e) { - System.out.println(CREDENTIALS_FILE + " doesn't exist."); - } - } - - @AfterClass - public static void oneTimeTearDown() throws IOException { - // Delete a bucket for this test - PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE)); - AmazonS3Client client = new AmazonS3Client(credentials); - DeleteBucketRequest dbr = new DeleteBucketRequest(TEST_BUCKET); - client.deleteBucket(dbr); - } +public class TestDeleteS3Object extends AbstractS3Test { @Test public void testSimpleDelete() throws IOException { // Prepares for this test - uploadTestFile("hello.txt"); + putTestFile("delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object()); - DeleteS3Object deleter = new DeleteS3Object(); - final TestRunner runner = TestRunners.newTestRunner(deleter); runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(DeleteS3Object.REGION, TEST_REGION); - runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET); - runner.setProperty(DeleteS3Object.KEY, "hello.txt"); + runner.setProperty(DeleteS3Object.REGION, REGION); + runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME); final Map attrs = new HashMap<>(); - attrs.put("filename", "hello.txt"); + attrs.put("filename", "delete-me"); runner.enqueue(new byte[0], attrs); + runner.run(1); runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1); @@ -95,18 +51,39 @@ public class TestDeleteS3Object { @Test public void testDeleteFolder() throws IOException { // Prepares for this test - uploadTestFile("folder/1.txt"); + putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object()); - DeleteS3Object deleter = new DeleteS3Object(); - final TestRunner runner = TestRunners.newTestRunner(deleter); runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(DeleteS3Object.REGION, TEST_REGION); - runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET); - runner.setProperty(DeleteS3Object.KEY, "folder/1.txt"); + runner.setProperty(DeleteS3Object.REGION, REGION); + runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME); final Map attrs = new HashMap<>(); - attrs.put("filename", "hello.txt"); + attrs.put("filename", "folder/delete-me"); runner.enqueue(new byte[0], attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1); + } + + @Test + public void testDeleteFolderNoExpressionLanguage() throws IOException { + // Prepares for this test + putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object()); + + runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); + runner.setProperty(DeleteS3Object.REGION, REGION); + runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME); + runner.setProperty(DeleteS3Object.KEY, "folder/delete-me"); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "a-different-name"); + runner.enqueue(new byte[0], attrs); + runner.run(1); runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1); @@ -114,28 +91,19 @@ public class TestDeleteS3Object { @Test public void testTryToDeleteNotExistingFile() throws IOException { - DeleteS3Object deleter = new DeleteS3Object(); - final TestRunner runner = TestRunners.newTestRunner(deleter); + final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object()); + runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(DeleteS3Object.REGION, TEST_REGION); - runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET); - runner.setProperty(DeleteS3Object.BUCKET, "no-such-a-key"); + runner.setProperty(DeleteS3Object.REGION, REGION); + runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME); final Map attrs = new HashMap<>(); attrs.put("filename", "no-such-a-file"); runner.enqueue(new byte[0], attrs); + runner.run(1); - runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_FAILURE, 1); + runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1); } - // Uploads a test file - private void uploadTestFile(String key) throws IOException { - PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE)); - AmazonS3Client client = new AmazonS3Client(credentials); - URL fileURL = this.getClass().getClassLoader().getResource("hello.txt"); - File file = new File(fileURL.getPath()); - PutObjectRequest putRequest = new PutObjectRequest(TEST_BUCKET, key, file); - PutObjectResult result = client.putObject(putRequest); - } } diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java index 0321514946..e7d3e8b7b6 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java @@ -18,7 +18,6 @@ package org.apache.nifi.processors.aws.s3; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,32 +29,70 @@ import org.junit.Ignore; import org.junit.Test; @Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created") -public class TestFetchS3Object { - - private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties"; - +public class TestFetchS3Object extends AbstractS3Test { @Test - public void testGet() throws IOException { + public void testSimpleGet() throws IOException { + putTestFile("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object()); - runner.setProperty(FetchS3Object.BUCKET, "anonymous-test-bucket-00000000"); + runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(FetchS3Object.KEY, "folder/1.txt"); + runner.setProperty(FetchS3Object.REGION, REGION); + runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME); final Map attrs = new HashMap<>(); - attrs.put("start", "0"); - + attrs.put("filename", "test-file"); runner.enqueue(new byte[0], attrs); + runner.run(1); runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1); + } + + @Test + public void testTryToFetchNotExistingFile() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object()); + + runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); + runner.setProperty(FetchS3Object.REGION, REGION); + runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "no-such-a-file"); + runner.enqueue(new byte[0], attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1); + } + + @Test + public void testContentsOfFileRetrieved() throws IOException { + String key = "folder/1.txt"; + putTestFile(key, getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object()); + + runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); + runner.setProperty(FetchS3Object.REGION, REGION); + runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME); + + final Map attrs = new HashMap<>(); + attrs.put("filename", key); + runner.enqueue(new byte[0], attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1); + final List ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS); final MockFlowFile out = ffs.iterator().next(); - final byte[] expectedBytes = Files.readAllBytes(Paths.get("src/test/resources/hello.txt")); + final byte[] expectedBytes = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); out.assertContentEquals(new String(expectedBytes)); + for (final Map.Entry entry : out.getAttributes().entrySet()) { System.out.println(entry.getKey() + " : " + entry.getValue()); } } - } diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java index de7816db70..4dac0d4f9b 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java @@ -17,7 +17,6 @@ package org.apache.nifi.processors.aws.s3; import java.io.IOException; -import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -30,22 +29,22 @@ import org.junit.Test; import com.amazonaws.services.s3.model.StorageClass; @Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created") -public class TestPutS3Object { - - private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties"; +public class TestPutS3Object extends AbstractS3Test { @Test public void testSimplePut() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new PutS3Object()); + runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012"); - runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly"); + runner.setProperty(PutS3Object.REGION, REGION); + runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); + Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); for (int i = 0; i < 3; i++) { final Map attrs = new HashMap<>(); attrs.put("filename", String.valueOf(i) + ".txt"); - runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); } runner.run(3); @@ -55,14 +54,17 @@ public class TestPutS3Object { @Test public void testPutInFolder() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new PutS3Object()); - runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012"); + runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly"); + runner.setProperty(PutS3Object.REGION, REGION); + runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); + Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); final Map attrs = new HashMap<>(); attrs.put("filename", "folder/1.txt"); - runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + runner.run(); runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1); @@ -71,14 +73,18 @@ public class TestPutS3Object { @Test public void testStorageClass() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new PutS3Object()); - runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012"); + runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); + runner.setProperty(PutS3Object.REGION, REGION); + runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); runner.setProperty(PutS3Object.STORAGE_CLASS, StorageClass.ReducedRedundancy.name()); + Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); final Map attrs = new HashMap<>(); attrs.put("filename", "folder/2.txt"); - runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + runner.run(); runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1); @@ -87,16 +93,18 @@ public class TestPutS3Object { @Test public void testPermissions() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new PutS3Object()); - runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012"); + runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE); - runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST, "28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5"); + runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); + runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST,"28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5"); + runner.setProperty(PutS3Object.REGION, REGION); final Map attrs = new HashMap<>(); attrs.put("filename", "folder/4.txt"); - runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + runner.run(); runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1); } - -} +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-aws-bundle/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/pom.xml index a43bb80b94..2470e411f2 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-aws-bundle/pom.xml @@ -30,14 +30,4 @@ nifi-aws-nar - - - - com.amazonaws - aws-java-sdk - 1.9.24 - - - - diff --git a/pom.xml b/pom.xml index ded7257b02..08a227fbdb 100644 --- a/pom.xml +++ b/pom.xml @@ -492,7 +492,7 @@ joda-time joda-time - 2.8 + 2.8.2 com.yammer.metrics @@ -968,6 +968,11 @@ derby 10.11.1.1 + + com.amazonaws + aws-java-sdk + 1.10.32 +