NIFI-1025 Updated aws-sdk and joda-time to resolve AWS issues with Java 1.8 update 60.

- Refactored tests - created AbstractS3Test for common utility methods
 - Corrected incorrect unit test in TestDeleteS3Object, and adjusted processor documentation to reflect behavior
 - moved aws dependency management to root pom

This closes #107

Tested, Reviewed and Amended by Tony Kurc (<tkurc@apache.org>)
This commit is contained in:
Venkatesh Sellappa 2015-11-07 21:09:18 -05:00 committed by Tony Kurc
parent 99629646fe
commit 77de51df19
7 changed files with 277 additions and 117 deletions

View File

@ -41,7 +41,7 @@ import org.apache.nifi.processor.util.StandardValidators;
@SeeAlso({PutS3Object.class})
@Tags({"Amazon", "S3", "AWS", "Archive", "Delete"})
@CapabilityDescription("Deletes FlowFiles on an Amazon S3 Bucket. " +
"And the FlowFiles are checked if exists or not before deleting.")
"If attempting to delete a file that does not exist, FlowFile is routed to success.")
public class DeleteS3Object extends AbstractS3Processor {
public static final PropertyDescriptor VERSION_ID = new PropertyDescriptor.Builder()
@ -80,6 +80,7 @@ public class DeleteS3Object extends AbstractS3Processor {
try {
if (versionId == null) {
final DeleteObjectRequest r = new DeleteObjectRequest(bucket, key);
// This call returns success if object doesn't exist
s3.deleteObject(r);
} else {
final DeleteVersionRequest r = new DeleteVersionRequest(bucket, key, versionId);

View File

@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.aws.s3;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import org.apache.nifi.util.file.FileUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import static org.junit.Assert.fail;
import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.DeleteBucketRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectSummary;
/**
* Base class for S3 tests. Establishes a bucket and helper methods for creating test scenarios
*/
public abstract class AbstractS3Test {
protected final static String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
protected final static String BUCKET_NAME = "test-bucket-00000000-0000-0000-0000-123456789021";
protected final static String SAMPLE_FILE_RESOURCE_NAME = "/hello.txt";
protected final static String REGION = "eu-west-1";
// Static so multiple Tests can use same client
protected static AmazonS3Client client;
@BeforeClass
public static void oneTimeSetup() {
// Creates a client and bucket for this test
final FileInputStream fis;
try {
fis = new FileInputStream(CREDENTIALS_FILE);
} catch (FileNotFoundException e1) {
fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e1.getLocalizedMessage());
return;
}
try {
final PropertiesCredentials credentials = new PropertiesCredentials(fis);
client = new AmazonS3Client(credentials);
if (client.doesBucketExist(BUCKET_NAME)) {
fail("Bucket " + BUCKET_NAME + " exists. Choose a different bucket name to continue test");
}
CreateBucketRequest request = new CreateBucketRequest(BUCKET_NAME, REGION);
client.createBucket(request);
} catch (final AmazonS3Exception e) {
fail("Can't create the key " + BUCKET_NAME + ": " + e.getLocalizedMessage());
} catch (final IOException e) {
fail("Caught IOException preparing tests: " + e.getLocalizedMessage());
} finally {
FileUtils.closeQuietly(fis);
}
if (!client.doesBucketExist(BUCKET_NAME)) {
fail("Setup incomplete, tests will fail");
}
}
@AfterClass
public static void oneTimeTearDown() {
// Empty the bucket before deleting it.
try {
ObjectListing objectListing = client.listObjects(BUCKET_NAME);
while (true) {
for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
client.deleteObject(BUCKET_NAME, objectSummary.getKey());
}
if (objectListing.isTruncated()) {
objectListing = client.listNextBatchOfObjects(objectListing);
} else {
break;
}
}
DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
client.deleteBucket(dbr);
} catch (final AmazonS3Exception e) {
System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
}
if (client.doesBucketExist(BUCKET_NAME)) {
Assert.fail("Incomplete teardown, subsequent tests might fail");
}
}
protected void putTestFile(String key, File file) throws AmazonS3Exception {
PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, file);
client.putObject(putRequest);
}
protected Path getResourcePath(String resourceName) {
Path path = null;
try {
path = Paths.get(getClass().getResource(resourceName).toURI());
} catch (URISyntaxException e) {
Assert.fail("Resource: " + resourceName + " does not exist" + e.getLocalizedMessage());
}
return path;
}
protected File getFileFromResourceName(String resourceName) {
URI uri = null;
try {
uri = this.getClass().getResource(resourceName).toURI();
} catch (URISyntaxException e) {
Assert.fail("Cannot proceed without File : " + resourceName);
}
return new File(uri);
}
}

View File

@ -16,77 +16,33 @@
*/
package org.apache.nifi.processors.aws.s3;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.DeleteBucketRequest;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Ignore;
import org.junit.Test;
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
public class TestDeleteS3Object {
private static final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
// When you want to test this, you should create a bucket on Amazon S3 as follows.
private static final String TEST_REGION = "ap-northeast-1";
private static final String TEST_BUCKET = "test-bucket-00000000-0000-0000-0000-1234567890123";
@BeforeClass
public static void oneTimeSetUp() {
// Creates a new bucket for this test
try {
PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
AmazonS3Client client = new AmazonS3Client(credentials);
CreateBucketRequest request = new CreateBucketRequest(TEST_BUCKET, TEST_REGION);
client.createBucket(request);
} catch (final AmazonS3Exception e) {
System.out.println("Can't create the key " + TEST_BUCKET + ":" + e.toString());
} catch (final IOException e) {
System.out.println(CREDENTIALS_FILE + " doesn't exist.");
}
}
@AfterClass
public static void oneTimeTearDown() throws IOException {
// Delete a bucket for this test
PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
AmazonS3Client client = new AmazonS3Client(credentials);
DeleteBucketRequest dbr = new DeleteBucketRequest(TEST_BUCKET);
client.deleteBucket(dbr);
}
public class TestDeleteS3Object extends AbstractS3Test {
@Test
public void testSimpleDelete() throws IOException {
// Prepares for this test
uploadTestFile("hello.txt");
putTestFile("delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
DeleteS3Object deleter = new DeleteS3Object();
final TestRunner runner = TestRunners.newTestRunner(deleter);
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
runner.setProperty(DeleteS3Object.KEY, "hello.txt");
runner.setProperty(DeleteS3Object.REGION, REGION);
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "hello.txt");
attrs.put("filename", "delete-me");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
@ -95,18 +51,39 @@ public class TestDeleteS3Object {
@Test
public void testDeleteFolder() throws IOException {
// Prepares for this test
uploadTestFile("folder/1.txt");
putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
DeleteS3Object deleter = new DeleteS3Object();
final TestRunner runner = TestRunners.newTestRunner(deleter);
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
runner.setProperty(DeleteS3Object.KEY, "folder/1.txt");
runner.setProperty(DeleteS3Object.REGION, REGION);
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "hello.txt");
attrs.put("filename", "folder/delete-me");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
}
@Test
public void testDeleteFolderNoExpressionLanguage() throws IOException {
// Prepares for this test
putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(DeleteS3Object.REGION, REGION);
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
runner.setProperty(DeleteS3Object.KEY, "folder/delete-me");
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "a-different-name");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
@ -114,28 +91,19 @@ public class TestDeleteS3Object {
@Test
public void testTryToDeleteNotExistingFile() throws IOException {
DeleteS3Object deleter = new DeleteS3Object();
final TestRunner runner = TestRunners.newTestRunner(deleter);
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
runner.setProperty(DeleteS3Object.BUCKET, "no-such-a-key");
runner.setProperty(DeleteS3Object.REGION, REGION);
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "no-such-a-file");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_FAILURE, 1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
}
// Uploads a test file
private void uploadTestFile(String key) throws IOException {
PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
AmazonS3Client client = new AmazonS3Client(credentials);
URL fileURL = this.getClass().getClassLoader().getResource("hello.txt");
File file = new File(fileURL.getPath());
PutObjectRequest putRequest = new PutObjectRequest(TEST_BUCKET, key, file);
PutObjectResult result = client.putObject(putRequest);
}
}

View File

@ -18,7 +18,6 @@ package org.apache.nifi.processors.aws.s3;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -30,32 +29,70 @@ import org.junit.Ignore;
import org.junit.Test;
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
public class TestFetchS3Object {
private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
public class TestFetchS3Object extends AbstractS3Test {
@Test
public void testGet() throws IOException {
public void testSimpleGet() throws IOException {
putTestFile("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
runner.setProperty(FetchS3Object.BUCKET, "anonymous-test-bucket-00000000");
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(FetchS3Object.KEY, "folder/1.txt");
runner.setProperty(FetchS3Object.REGION, REGION);
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("start", "0");
attrs.put("filename", "test-file");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
}
@Test
public void testTryToFetchNotExistingFile() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(FetchS3Object.REGION, REGION);
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "no-such-a-file");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1);
}
@Test
public void testContentsOfFileRetrieved() throws IOException {
String key = "folder/1.txt";
putTestFile(key, getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(FetchS3Object.REGION, REGION);
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", key);
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
final MockFlowFile out = ffs.iterator().next();
final byte[] expectedBytes = Files.readAllBytes(Paths.get("src/test/resources/hello.txt"));
final byte[] expectedBytes = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME));
out.assertContentEquals(new String(expectedBytes));
for (final Map.Entry<String, String> entry : out.getAttributes().entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue());
}
}
}

View File

@ -17,7 +17,6 @@
package org.apache.nifi.processors.aws.s3;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
@ -30,22 +29,22 @@ import org.junit.Test;
import com.amazonaws.services.s3.model.StorageClass;
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
public class TestPutS3Object {
private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
public class TestPutS3Object extends AbstractS3Test {
@Test
public void testSimplePut() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
runner.setProperty(PutS3Object.REGION, REGION);
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
for (int i = 0; i < 3; i++) {
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", String.valueOf(i) + ".txt");
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
}
runner.run(3);
@ -55,14 +54,17 @@ public class TestPutS3Object {
@Test
public void testPutInFolder() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
runner.setProperty(PutS3Object.REGION, REGION);
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "folder/1.txt");
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
@ -71,14 +73,18 @@ public class TestPutS3Object {
@Test
public void testStorageClass() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(PutS3Object.REGION, REGION);
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
runner.setProperty(PutS3Object.STORAGE_CLASS, StorageClass.ReducedRedundancy.name());
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "folder/2.txt");
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
@ -87,16 +93,18 @@ public class TestPutS3Object {
@Test
public void testPermissions() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST, "28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST,"28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
runner.setProperty(PutS3Object.REGION, REGION);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "folder/4.txt");
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
}
}
}

View File

@ -30,14 +30,4 @@
<module>nifi-aws-nar</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.9.24</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

View File

@ -492,7 +492,7 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.8</version>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>com.yammer.metrics</groupId>
@ -968,6 +968,11 @@
<artifactId>derby</artifactId>
<version>10.11.1.1</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.10.32</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>