mirror of
https://github.com/apache/nifi.git
synced 2025-02-20 17:36:50 +00:00
NIFI-1025 Updated aws-sdk and joda-time to resolve AWS issues with Java 1.8 update 60.
- Refactored tests - created AbstractS3Test for common utility methods - Corrected incorrect unit test in TestDeleteS3Object, and adjusted processor documentation to reflect behavior - moved aws dependency management to root pom This closes #107 Tested, Reviewed and Amended by Tony Kurc (<tkurc@apache.org>)
This commit is contained in:
parent
99629646fe
commit
77de51df19
@ -41,7 +41,7 @@ import org.apache.nifi.processor.util.StandardValidators;
|
||||
@SeeAlso({PutS3Object.class})
|
||||
@Tags({"Amazon", "S3", "AWS", "Archive", "Delete"})
|
||||
@CapabilityDescription("Deletes FlowFiles on an Amazon S3 Bucket. " +
|
||||
"And the FlowFiles are checked if exists or not before deleting.")
|
||||
"If attempting to delete a file that does not exist, FlowFile is routed to success.")
|
||||
public class DeleteS3Object extends AbstractS3Processor {
|
||||
|
||||
public static final PropertyDescriptor VERSION_ID = new PropertyDescriptor.Builder()
|
||||
@ -80,6 +80,7 @@ public class DeleteS3Object extends AbstractS3Processor {
|
||||
try {
|
||||
if (versionId == null) {
|
||||
final DeleteObjectRequest r = new DeleteObjectRequest(bucket, key);
|
||||
// This call returns success if object doesn't exist
|
||||
s3.deleteObject(r);
|
||||
} else {
|
||||
final DeleteVersionRequest r = new DeleteVersionRequest(bucket, key, versionId);
|
||||
|
@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.aws.s3;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.nifi.util.file.FileUtils;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import com.amazonaws.auth.PropertiesCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.model.AmazonS3Exception;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.DeleteBucketRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectListing;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
|
||||
/**
|
||||
* Base class for S3 tests. Establishes a bucket and helper methods for creating test scenarios
|
||||
*/
|
||||
public abstract class AbstractS3Test {
|
||||
protected final static String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
|
||||
protected final static String BUCKET_NAME = "test-bucket-00000000-0000-0000-0000-123456789021";
|
||||
protected final static String SAMPLE_FILE_RESOURCE_NAME = "/hello.txt";
|
||||
protected final static String REGION = "eu-west-1";
|
||||
|
||||
// Static so multiple Tests can use same client
|
||||
protected static AmazonS3Client client;
|
||||
|
||||
@BeforeClass
|
||||
public static void oneTimeSetup() {
|
||||
// Creates a client and bucket for this test
|
||||
|
||||
final FileInputStream fis;
|
||||
try {
|
||||
fis = new FileInputStream(CREDENTIALS_FILE);
|
||||
} catch (FileNotFoundException e1) {
|
||||
fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e1.getLocalizedMessage());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
final PropertiesCredentials credentials = new PropertiesCredentials(fis);
|
||||
client = new AmazonS3Client(credentials);
|
||||
|
||||
if (client.doesBucketExist(BUCKET_NAME)) {
|
||||
fail("Bucket " + BUCKET_NAME + " exists. Choose a different bucket name to continue test");
|
||||
}
|
||||
|
||||
CreateBucketRequest request = new CreateBucketRequest(BUCKET_NAME, REGION);
|
||||
client.createBucket(request);
|
||||
|
||||
} catch (final AmazonS3Exception e) {
|
||||
fail("Can't create the key " + BUCKET_NAME + ": " + e.getLocalizedMessage());
|
||||
} catch (final IOException e) {
|
||||
fail("Caught IOException preparing tests: " + e.getLocalizedMessage());
|
||||
} finally {
|
||||
FileUtils.closeQuietly(fis);
|
||||
}
|
||||
|
||||
if (!client.doesBucketExist(BUCKET_NAME)) {
|
||||
fail("Setup incomplete, tests will fail");
|
||||
}
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void oneTimeTearDown() {
|
||||
// Empty the bucket before deleting it.
|
||||
try {
|
||||
ObjectListing objectListing = client.listObjects(BUCKET_NAME);
|
||||
|
||||
while (true) {
|
||||
for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
|
||||
S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
|
||||
client.deleteObject(BUCKET_NAME, objectSummary.getKey());
|
||||
}
|
||||
|
||||
if (objectListing.isTruncated()) {
|
||||
objectListing = client.listNextBatchOfObjects(objectListing);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
|
||||
client.deleteBucket(dbr);
|
||||
} catch (final AmazonS3Exception e) {
|
||||
System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
|
||||
}
|
||||
|
||||
if (client.doesBucketExist(BUCKET_NAME)) {
|
||||
Assert.fail("Incomplete teardown, subsequent tests might fail");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected void putTestFile(String key, File file) throws AmazonS3Exception {
|
||||
PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, file);
|
||||
|
||||
client.putObject(putRequest);
|
||||
}
|
||||
|
||||
protected Path getResourcePath(String resourceName) {
|
||||
Path path = null;
|
||||
|
||||
try {
|
||||
path = Paths.get(getClass().getResource(resourceName).toURI());
|
||||
} catch (URISyntaxException e) {
|
||||
Assert.fail("Resource: " + resourceName + " does not exist" + e.getLocalizedMessage());
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
protected File getFileFromResourceName(String resourceName) {
|
||||
URI uri = null;
|
||||
try {
|
||||
uri = this.getClass().getResource(resourceName).toURI();
|
||||
} catch (URISyntaxException e) {
|
||||
Assert.fail("Cannot proceed without File : " + resourceName);
|
||||
}
|
||||
|
||||
return new File(uri);
|
||||
}
|
||||
}
|
@ -16,77 +16,33 @@
|
||||
*/
|
||||
package org.apache.nifi.processors.aws.s3;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.amazonaws.auth.PropertiesCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.model.AmazonS3Exception;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.DeleteBucketRequest;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.PutObjectResult;
|
||||
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
|
||||
public class TestDeleteS3Object {
|
||||
|
||||
private static final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
|
||||
|
||||
// When you want to test this, you should create a bucket on Amazon S3 as follows.
|
||||
private static final String TEST_REGION = "ap-northeast-1";
|
||||
private static final String TEST_BUCKET = "test-bucket-00000000-0000-0000-0000-1234567890123";
|
||||
|
||||
@BeforeClass
|
||||
public static void oneTimeSetUp() {
|
||||
// Creates a new bucket for this test
|
||||
try {
|
||||
PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
|
||||
AmazonS3Client client = new AmazonS3Client(credentials);
|
||||
CreateBucketRequest request = new CreateBucketRequest(TEST_BUCKET, TEST_REGION);
|
||||
client.createBucket(request);
|
||||
} catch (final AmazonS3Exception e) {
|
||||
System.out.println("Can't create the key " + TEST_BUCKET + ":" + e.toString());
|
||||
} catch (final IOException e) {
|
||||
System.out.println(CREDENTIALS_FILE + " doesn't exist.");
|
||||
}
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void oneTimeTearDown() throws IOException {
|
||||
// Delete a bucket for this test
|
||||
PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
|
||||
AmazonS3Client client = new AmazonS3Client(credentials);
|
||||
DeleteBucketRequest dbr = new DeleteBucketRequest(TEST_BUCKET);
|
||||
client.deleteBucket(dbr);
|
||||
}
|
||||
public class TestDeleteS3Object extends AbstractS3Test {
|
||||
|
||||
@Test
|
||||
public void testSimpleDelete() throws IOException {
|
||||
// Prepares for this test
|
||||
uploadTestFile("hello.txt");
|
||||
putTestFile("delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
|
||||
|
||||
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
|
||||
|
||||
DeleteS3Object deleter = new DeleteS3Object();
|
||||
final TestRunner runner = TestRunners.newTestRunner(deleter);
|
||||
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
|
||||
runner.setProperty(DeleteS3Object.KEY, "hello.txt");
|
||||
runner.setProperty(DeleteS3Object.REGION, REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "hello.txt");
|
||||
attrs.put("filename", "delete-me");
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
|
||||
@ -95,18 +51,39 @@ public class TestDeleteS3Object {
|
||||
@Test
|
||||
public void testDeleteFolder() throws IOException {
|
||||
// Prepares for this test
|
||||
uploadTestFile("folder/1.txt");
|
||||
putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
|
||||
|
||||
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
|
||||
|
||||
DeleteS3Object deleter = new DeleteS3Object();
|
||||
final TestRunner runner = TestRunners.newTestRunner(deleter);
|
||||
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
|
||||
runner.setProperty(DeleteS3Object.KEY, "folder/1.txt");
|
||||
runner.setProperty(DeleteS3Object.REGION, REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "hello.txt");
|
||||
attrs.put("filename", "folder/delete-me");
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteFolderNoExpressionLanguage() throws IOException {
|
||||
// Prepares for this test
|
||||
putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
|
||||
|
||||
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
|
||||
|
||||
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(DeleteS3Object.REGION, REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
|
||||
runner.setProperty(DeleteS3Object.KEY, "folder/delete-me");
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "a-different-name");
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
|
||||
@ -114,28 +91,19 @@ public class TestDeleteS3Object {
|
||||
|
||||
@Test
|
||||
public void testTryToDeleteNotExistingFile() throws IOException {
|
||||
DeleteS3Object deleter = new DeleteS3Object();
|
||||
final TestRunner runner = TestRunners.newTestRunner(deleter);
|
||||
final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
|
||||
|
||||
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, "no-such-a-key");
|
||||
runner.setProperty(DeleteS3Object.REGION, REGION);
|
||||
runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "no-such-a-file");
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_FAILURE, 1);
|
||||
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
|
||||
}
|
||||
|
||||
// Uploads a test file
|
||||
private void uploadTestFile(String key) throws IOException {
|
||||
PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
|
||||
AmazonS3Client client = new AmazonS3Client(credentials);
|
||||
URL fileURL = this.getClass().getClassLoader().getResource("hello.txt");
|
||||
File file = new File(fileURL.getPath());
|
||||
PutObjectRequest putRequest = new PutObjectRequest(TEST_BUCKET, key, file);
|
||||
PutObjectResult result = client.putObject(putRequest);
|
||||
}
|
||||
}
|
||||
|
@ -18,7 +18,6 @@ package org.apache.nifi.processors.aws.s3;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -30,32 +29,70 @@ import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
|
||||
public class TestFetchS3Object {
|
||||
|
||||
private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
|
||||
|
||||
public class TestFetchS3Object extends AbstractS3Test {
|
||||
@Test
|
||||
public void testGet() throws IOException {
|
||||
public void testSimpleGet() throws IOException {
|
||||
putTestFile("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
|
||||
|
||||
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
|
||||
runner.setProperty(FetchS3Object.BUCKET, "anonymous-test-bucket-00000000");
|
||||
|
||||
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(FetchS3Object.KEY, "folder/1.txt");
|
||||
runner.setProperty(FetchS3Object.REGION, REGION);
|
||||
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("start", "0");
|
||||
|
||||
attrs.put("filename", "test-file");
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTryToFetchNotExistingFile() throws IOException {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
|
||||
|
||||
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(FetchS3Object.REGION, REGION);
|
||||
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "no-such-a-file");
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContentsOfFileRetrieved() throws IOException {
|
||||
String key = "folder/1.txt";
|
||||
putTestFile(key, getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
|
||||
|
||||
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
|
||||
|
||||
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(FetchS3Object.REGION, REGION);
|
||||
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", key);
|
||||
runner.enqueue(new byte[0], attrs);
|
||||
|
||||
runner.run(1);
|
||||
|
||||
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
|
||||
|
||||
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
|
||||
final MockFlowFile out = ffs.iterator().next();
|
||||
|
||||
final byte[] expectedBytes = Files.readAllBytes(Paths.get("src/test/resources/hello.txt"));
|
||||
final byte[] expectedBytes = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME));
|
||||
out.assertContentEquals(new String(expectedBytes));
|
||||
|
||||
for (final Map.Entry<String, String> entry : out.getAttributes().entrySet()) {
|
||||
System.out.println(entry.getKey() + " : " + entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -17,7 +17,6 @@
|
||||
package org.apache.nifi.processors.aws.s3;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@ -30,22 +29,22 @@ import org.junit.Test;
|
||||
import com.amazonaws.services.s3.model.StorageClass;
|
||||
|
||||
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
|
||||
public class TestPutS3Object {
|
||||
|
||||
private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
|
||||
public class TestPutS3Object extends AbstractS3Test {
|
||||
|
||||
@Test
|
||||
public void testSimplePut() throws IOException {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
|
||||
|
||||
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
|
||||
runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
|
||||
runner.setProperty(PutS3Object.REGION, REGION);
|
||||
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", String.valueOf(i) + ".txt");
|
||||
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
|
||||
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
|
||||
}
|
||||
runner.run(3);
|
||||
|
||||
@ -55,14 +54,17 @@ public class TestPutS3Object {
|
||||
@Test
|
||||
public void testPutInFolder() throws IOException {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
|
||||
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
|
||||
|
||||
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
|
||||
runner.setProperty(PutS3Object.REGION, REGION);
|
||||
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
|
||||
|
||||
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "folder/1.txt");
|
||||
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
|
||||
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
|
||||
|
||||
runner.run();
|
||||
|
||||
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
|
||||
@ -71,14 +73,18 @@ public class TestPutS3Object {
|
||||
@Test
|
||||
public void testStorageClass() throws IOException {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
|
||||
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
|
||||
|
||||
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(PutS3Object.REGION, REGION);
|
||||
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
|
||||
runner.setProperty(PutS3Object.STORAGE_CLASS, StorageClass.ReducedRedundancy.name());
|
||||
|
||||
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "folder/2.txt");
|
||||
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
|
||||
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
|
||||
|
||||
runner.run();
|
||||
|
||||
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
|
||||
@ -87,16 +93,18 @@ public class TestPutS3Object {
|
||||
@Test
|
||||
public void testPermissions() throws IOException {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
|
||||
runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
|
||||
|
||||
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
|
||||
runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST, "28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
|
||||
runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
|
||||
runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST,"28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
|
||||
runner.setProperty(PutS3Object.REGION, REGION);
|
||||
|
||||
final Map<String, String> attrs = new HashMap<>();
|
||||
attrs.put("filename", "folder/4.txt");
|
||||
runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
|
||||
runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
|
||||
|
||||
runner.run();
|
||||
|
||||
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -30,14 +30,4 @@
|
||||
<module>nifi-aws-nar</module>
|
||||
</modules>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk</artifactId>
|
||||
<version>1.9.24</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
</project>
|
||||
|
7
pom.xml
7
pom.xml
@ -492,7 +492,7 @@
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
<version>2.8</version>
|
||||
<version>2.8.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.yammer.metrics</groupId>
|
||||
@ -968,6 +968,11 @@
|
||||
<artifactId>derby</artifactId>
|
||||
<version>10.11.1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk</artifactId>
|
||||
<version>1.10.32</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
<dependencies>
|
||||
|
Loading…
x
Reference in New Issue
Block a user