iSharkFly-Docs/java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #14360 from panos-kakos/TASK-22456

Browse Source 
TASK 22456
This commit is contained in:
Kasra Madadipouya 2023-08-06 15:59:02 +02:00 committed by GitHub
commit fdfc98264d
23 changed files with 831 additions and 873 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
aws-modules/aws-s3-v2/README.md
View File

@ -1,2 +0,0 @@
## Relevant Articles
- [Listing All AWS S3 Objects in a Bucket Using Java](https://www.baeldung.com/java-aws-s3-list-bucket-objects)

50
aws-modules/aws-s3-v2/pom.xml
View File

@ -1,50 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.baeldung.s3</groupId>
<artifactId>aws-s3-v2</artifactId>
<version>1.0-SNAPSHOT</version>
<name>aws-s3-v2</name>
<packaging>jar</packaging>
<parent>
<groupId>com.baeldung</groupId>
<artifactId>aws-modules</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<aws.java.sdk.version>2.20.52</aws.java.sdk.version>
<maven.compiler.plugin.version>3.6.1</maven.compiler.plugin.version>
</properties>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>${aws.java.sdk.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

22
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java
View File

@ -1,22 +0,0 @@
package com.baeldung.s3.listobjects;
import software.amazon.awssdk.regions.Region;
public class Main {
private static String AWS_BUCKET = "baeldung-tutorial-s3";
public static Region AWS_REGION = Region.EU_CENTRAL_1;
public static void main(String[] args) {
S3Service s3Service = new S3Service(AWS_REGION);
s3Service.putObject(AWS_BUCKET, FileGenerator.generateFiles(1000, 1));
s3Service.listBuckets();
s3Service.listObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500);
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/");
s3Service.cleanup();
}
}

129
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java
View File

@ -1,129 +0,0 @@
package com.baeldung.s3.listobjects;
import java.util.List;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
class S3Service {
private S3Client s3Client;
public S3Service(Region awsRegion) {
init(awsRegion);
}
public S3Service(S3Client s3Client) {
this.s3Client = s3Client;
}
public void init(Region awsRegion) {
this.s3Client = S3Client.builder()
.region(awsRegion)
.credentialsProvider(ProfileCredentialsProvider.create("default"))
.build();
}
public void listObjectsInBucket(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
List<S3Object> contents = listObjectsV2Response.contents();
System.out.println("Number of objects in the bucket: " + contents.stream().count());
contents.stream().forEach(System.out::println);
}
public void listAllObjectsInBucket(String bucketName) {
String nextContinuationToken = null;
long totalObjects = 0;
do {
ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder()
.bucket(bucketName)
.continuationToken(nextContinuationToken);
ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
nextContinuationToken = response.nextContinuationToken();
totalObjects += response.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
} while (nextContinuationToken != null);
System.out.println("Number of objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.prefix(prefix)
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listBuckets() {
// List all buckets
ListBucketsResponse listBucketsResponse = s3Client.listBuckets();
// Display the bucket names
List<Bucket> buckets = listBucketsResponse.buckets();
System.out.println("Buckets:");
for (Bucket bucket : buckets) {
System.out.println(bucket.name());
}
}
public void putObject(String bucketName, List<File> files) {
try {
files.stream().forEach(file -> {
s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(),
RequestBody.fromByteBuffer(file.getContent()));
System.out.println("Uploaded file: " + file.getName());
});
} catch (S3Exception e) {
System.err.println("Upload failed");
e.printStackTrace();
}
}
public void cleanup() {
this.s3Client.close();
}
}

14
aws-modules/aws-s3-v2/src/main/resources/logback.xml
View File

@ -1,14 +0,0 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
<!-- Change the log level for software.amazon.awssdk -->
<logger name="software.amazon.awssdk" level="info" />
</configuration>

1
aws-modules/aws-s3/README.md
View File

@ -8,3 +8,4 @@ This module contains articles about Simple Storage Service (S3) on AWS
- [Multipart Uploads in Amazon S3 with Java](https://www.baeldung.com/aws-s3-multipart-upload)
- [Using the JetS3t Java Client With Amazon S3](https://www.baeldung.com/jets3t-amazon-s3)
- [Check if a Specified Key Exists in a Given S3 Bucket Using Java](https://www.baeldung.com/java-aws-s3-check-specified-key-exists)
- [Listing All AWS S3 Objects in a Bucket Using Java](https://www.baeldung.com/java-aws-s3-list-bucket-objects)

12
aws-modules/aws-s3/pom.xml
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>aws-s3</artifactId>
<version>0.1.0-SNAPSHOT</version>
@ -16,10 +16,11 @@
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>${aws-java-sdk.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>${aws.java.sdk.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
@ -60,6 +61,7 @@
</build>
<properties>
<aws.java.sdk.version>2.20.52</aws.java.sdk.version>
<commons-codec-version>1.10.L001</commons-codec-version>
<jets3t-version>0.9.4.0006L</jets3t-version>
</properties>

42
aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java
View File

@ -1,42 +0,0 @@
package com.baeldung.s3;
import org.apache.http.HttpStatus;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.AmazonS3;
public class AWSS3ObjectUtils {
private AmazonS3 s3Client;
public AWSS3ObjectUtils(AmazonS3 s3client) {
this.s3Client = s3client;
}
public boolean doesObjectExistByDefaultMethod(String bucket, String key) {
return s3Client.doesObjectExist(bucket, key);
}
public boolean doesObjectExistByListObjects(String bucket, String key) {
return s3Client.listObjects(bucket)
.getObjectSummaries()
.stream()
.filter(s3ObjectSummary -> s3ObjectSummary.getKey()
.equals(key))
.findFirst()
.isPresent();
}
public boolean doesObjectExistByMetaData(String bucket, String key) {
try {
s3Client.getObjectMetadata(bucket, key);
return true;
} catch (AmazonServiceException e) {
if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
return false;
} else {
throw e;
}
}
}
}

93
aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java
View File

@ -1,93 +0,0 @@
package com.baeldung.s3;
import java.io.File;
import java.net.URL;
import java.util.List;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsResult;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;
public class AWSS3Service {
private final AmazonS3 s3client;
public AWSS3Service() {
this(new AmazonS3Client() {
});
}
public AWSS3Service(AmazonS3 s3client) {
this.s3client = s3client;
}
//is bucket exist?
public boolean doesBucketExist(String bucketName) {
return s3client.doesBucketExist(bucketName);
}
//create a bucket
public Bucket createBucket(String bucketName) {
return s3client.createBucket(bucketName);
}
//list all buckets
public List<Bucket> listBuckets() {
return s3client.listBuckets();
}
//delete a bucket
public void deleteBucket(String bucketName) {
s3client.deleteBucket(bucketName);
}
//uploading object
public PutObjectResult putObject(String bucketName, String key, File file) {
return s3client.putObject(bucketName, key, file);
}
//uploading object and getting url
public URL getObjectURL(String bucketName, String key, File file) {
s3client.putObject(bucketName, key, file);
return s3client.getUrl(bucketName, key);
}
//listing objects
public ObjectListing listObjects(String bucketName) {
return s3client.listObjects(bucketName);
}
//get an object
public S3Object getObject(String bucketName, String objectKey) {
return s3client.getObject(bucketName, objectKey);
}
//copying an object
public CopyObjectResult copyObject(
String sourceBucketName,
String sourceKey,
String destinationBucketName,
String destinationKey
) {
return s3client.copyObject(
sourceBucketName,
sourceKey,
destinationBucketName,
destinationKey
);
}
//deleting an object
public void deleteObject(String bucketName, String objectKey) {
s3client.deleteObject(bucketName, objectKey);
}
//deleting multiple Objects
public DeleteObjectsResult deleteObjects(DeleteObjectsRequest delObjReq) {
return s3client.deleteObjects(delObjReq);
}
}

42
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java → aws-modules/aws-s3/src/main/java/com/baeldung/s3/File.java
View File

@ -1,21 +1,21 @@
package com.baeldung.s3.listobjects;
import java.nio.ByteBuffer;
public class File {
private String name;
private ByteBuffer content;
public File(String name, ByteBuffer content) {
this.name = name;
this.content = content;
}
public String getName() {
return name;
}
public ByteBuffer getContent() {
return content;
}
}
package com.baeldung.s3;
import java.nio.ByteBuffer;
public class File {
private String name;
private ByteBuffer content;
public File(String name, ByteBuffer content) {
this.name = name;
this.content = content;
}
public String getName() {
return name;
}
public ByteBuffer getContent() {
return content;
}
}

54
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java → aws-modules/aws-s3/src/main/java/com/baeldung/s3/FileGenerator.java
View File

@ -1,27 +1,27 @@
package com.baeldung.s3.listobjects;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class FileGenerator {
public static List<File> generateFiles(int fileCount, int fileSize) {
List<File> files = new ArrayList<>();
for (int i = 0; i < fileCount; i++) {
String fileName = "file_" + i + ".txt";
ByteBuffer fileContent = generateRandomBytes(fileSize);
files.add(new File(fileName, fileContent));
}
return files;
}
private static ByteBuffer generateRandomBytes(int size) {
byte[] array = new byte[size];
new Random().nextBytes(array);
return ByteBuffer.wrap(array);
}
}
package com.baeldung.s3;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class FileGenerator {
public static List<File> generateFiles(int fileCount, int fileSize) {
List<File> files = new ArrayList<>();
for (int i = 0; i < fileCount; i++) {
String fileName = "file_" + i + ".txt";
ByteBuffer fileContent = generateRandomBytes(fileSize);
files.add(new File(fileName, fileContent));
}
return files;
}
private static ByteBuffer generateRandomBytes(int size) {
byte[] array = new byte[size];
new Random().nextBytes(array);
return ByteBuffer.wrap(array);
}
}

115
aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java
View File

@ -1,56 +1,101 @@
package com.baeldung.s3;
import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.model.*;
import java.io.File;
import java.util.concurrent.Executors;
public class MultipartUpload {
public static void main(String[] args) throws Exception {
public static void main(String[] args) {
String existingBucketName = "baeldung-bucket";
String keyName = "my-picture.jpg";
String filePath = "documents/my-picture.jpg";
AmazonS3 amazonS3 = AmazonS3ClientBuilder
.standard()
.withCredentials(new DefaultAWSCredentialsProviderChain())
.withRegion(Regions.DEFAULT_REGION)
.build();
ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create();
Region region = Region.US_EAST_1;
S3Client s3 = S3Client.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.build();
int maxUploadThreads = 5;
// Initiate a multipart upload
CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder()
.bucket(existingBucketName)
.key(keyName)
.build();
TransferManager tm = TransferManagerBuilder
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold((long) (5 * 1024 * 1024))
.withExecutorFactory(() -> Executors.newFixedThreadPool(maxUploadThreads))
.build();
CreateMultipartUploadResponse createResponse = s3.createMultipartUpload(createRequest);
ProgressListener progressListener =
progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
String uploadId = createResponse.uploadId();
PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath));
// Prepare the parts to be uploaded
List<CompletedPart> completedParts = new ArrayList<>();
int partNumber = 1;
ByteBuffer buffer = ByteBuffer.allocate(5 * 1024 * 1024); // Set your preferred part size (5 MB in this example)
request.setGeneralProgressListener(progressListener);
// Read the file and upload each part
try (RandomAccessFile file = new RandomAccessFile(filePath, "r")) {
long fileSize = file.length();
long position = 0;
Upload upload = tm.upload(request);
while (position < fileSize) {
file.seek(position);
int bytesRead = file.getChannel().read(buffer);
try {
upload.waitForCompletion();
System.out.println("Upload complete.");
} catch (AmazonClientException e) {
System.out.println("Error occurred while uploading file");
buffer.flip();
UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
.bucket(existingBucketName)
.key(keyName)
.uploadId(uploadId)
.partNumber(partNumber)
.contentLength((long) bytesRead)
.build();
UploadPartResponse response = s3.uploadPart(uploadPartRequest, RequestBody.fromByteBuffer(buffer));
completedParts.add(CompletedPart.builder()
.partNumber(partNumber)
.eTag(response.eTag())
.build());
buffer.clear();
position += bytesRead;
partNumber++;
}
} catch (IOException e) {
e.printStackTrace();
}
// Complete the multipart upload
CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder()
.parts(completedParts)
.build();
CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder()
.bucket(existingBucketName)
.key(keyName)
.uploadId(uploadId)
.multipartUpload(completedUpload)
.build();
CompleteMultipartUploadResponse completeResponse = s3.completeMultipartUpload(completeRequest);
// Print the object's URL
String objectUrl = s3.utilities().getUrl(GetUrlRequest.builder()
.bucket(existingBucketName)
.key(keyName)
.build())
.toExternalForm();
System.out.println("Uploaded object URL: " + objectUrl);
}
}

123
aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java
View File

@ -1,104 +1,69 @@
package com.baeldung.s3;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import software.amazon.awssdk.regions.Region;
public class S3Application {
private static final AWSCredentials credentials;
private static String bucketName;
private static String AWS_BUCKET = "baeldung-tutorial-s3";
public static Region AWS_REGION = Region.EU_CENTRAL_1;
static {
//put your accesskey and secretkey here
credentials = new BasicAWSCredentials(
"<AWS accesskey>",
"<AWS secretkey>"
);
}
public static void main(String[] args) throws IOException {
//set-up the client
AmazonS3 s3client = AmazonS3ClientBuilder
.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.withRegion(Regions.US_EAST_2)
.build();
AWSS3Service awsService = new AWSS3Service(s3client);
bucketName = "baeldung-bucket";
public static void main(String[] args) {
S3Service s3Service = new S3Service(AWS_REGION);
//creating a bucket
if(awsService.doesBucketExist(bucketName)) {
s3Service.createBucket(AWS_BUCKET);
//check if a bucket exists
if(s3Service.doesBucketExist(AWS_BUCKET)) {
System.out.println("Bucket name is not available."
+ " Try again with a different Bucket name.");
+ " Try again with a different Bucket name.");
return;
}
awsService.createBucket(bucketName);
//list all the buckets
for(Bucket s : awsService.listBuckets() ) {
System.out.println(s.getName());
}
//deleting bucket
awsService.deleteBucket("baeldung-bucket-test2");
//uploading object
awsService.putObject(
bucketName,
"Document/hello.txt",
new File("/Users/user/Document/hello.txt")
);
//uploading object and getting url
awsService.getObjectURL(bucketName, "Document/hello.txt", new File("/Users/user/Document/hello.txt"));
s3Service.putObjects(AWS_BUCKET, FileGenerator.generateFiles(1000, 1));
//list all the buckets
s3Service.listBuckets();
s3Service.listObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500);
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/");
//deleting bucket
s3Service.deleteBucket("baeldung-bucket-test2");
//uploading object
s3Service.putObject(
AWS_BUCKET,
"Document/hello.txt",
new File("/Users/user/Document/hello.txt")
);
//listing objects
ObjectListing objectListing = awsService.listObjects(bucketName);
for(S3ObjectSummary os : objectListing.getObjectSummaries()) {
System.out.println(os.getKey());
}
s3Service.listObjects(AWS_BUCKET);
//downloading an object
S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt");
S3ObjectInputStream inputStream = s3object.getObjectContent();
FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt"));
s3Service.getObject(AWS_BUCKET, "Document/hello.txt");
//copying an object
awsService.copyObject(
"baeldung-bucket",
"picture/pic.png",
"baeldung-bucket2",
"Document/picture.png"
s3Service.copyObject(
"baeldung-bucket",
"picture/pic.png",
"baeldung-bucket2",
"Document/picture.png"
);
//deleting an object
awsService.deleteObject(bucketName, "Document/hello.txt");
s3Service.deleteObject(AWS_BUCKET, "Document/hello.txt");
//deleting multiple objects
String objkeyArr[] = {
"Document/hello2.txt",
"Document/picture.png"
};
DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket")
.withKeys(objkeyArr);
awsService.deleteObjects(delObjReq);
List<String> objKeyList = List.of("Document/hello2.txt", "Document/picture.png");
s3Service.deleteObjects(AWS_BUCKET, objKeyList);
}
}

323
aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java Normal file
View File

@ -0,0 +1,323 @@
package com.baeldung.s3;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Bucket;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
class S3Service {
private S3Client s3Client;
public S3Service(Region awsRegion) {
init(awsRegion);
}
public S3Service(S3Client s3Client) {
this.s3Client = s3Client;
}
public void init(Region awsRegion) {
this.s3Client = S3Client.builder()
.region(awsRegion)
.credentialsProvider(ProfileCredentialsProvider.create("default"))
.build();
}
//is bucket exist?
public boolean doesBucketExist(String bucketName) {
HeadBucketRequest headBucketRequest = HeadBucketRequest.builder()
.bucket(bucketName)
.build();
try {
s3Client.headBucket(headBucketRequest);
return true;
} catch (NoSuchBucketException e) {
return false;
}
}
//create a bucket
public void createBucket(String bucketName) {
CreateBucketRequest bucketRequest = CreateBucketRequest.builder()
.bucket(bucketName)
.build();
s3Client.createBucket(bucketRequest);
}
//delete a bucket
public void deleteBucket(String bucketName) {
try {
DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder()
.bucket(bucketName)
.build();
s3Client.deleteBucket(deleteBucketRequest);
System.out.println("Successfully deleted bucket : " + bucketName);
} catch (S3Exception e) {
System.err.println(e.getMessage());
System.exit(1);
}
}
//uploading object
public PutObjectResponse putObject(String bucketName, String key, java.io.File file) {
PutObjectRequest request = PutObjectRequest.builder()
.bucket(bucketName)
.key(key)
.build();
return s3Client.putObject(request, Path.of(file.toURI()) );
}
//listing objects
public void listObjects(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
for(S3Object os : listObjectsV2Response.contents()) {
System.out.println(os.key());
}
}
//get an object
public void getObject(String bucketName, String objectKey) {
try {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(objectKey)
.build();
ResponseBytes<GetObjectResponse> responseResponseBytes = s3Client.getObjectAsBytes(objectRequest);
byte[] data = responseResponseBytes.asByteArray();
// Write the data to a local file.
java.io.File myFile = new java.io.File("/Users/user/Desktop/hello.txt" );
OutputStream os = new FileOutputStream(myFile);
os.write(data);
System.out.println("Successfully obtained bytes from an S3 object");
os.close();
} catch (IOException ex) {
ex.printStackTrace();
} catch (S3Exception e) {
System.err.println(e.awsErrorDetails().errorMessage());
System.exit(1);
}
}
//copying an object
public CopyObjectResponse copyObject(
String sourceBucketName,
String sourceKey,
String destinationBucketName,
String destinationKey
) {
CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destinationBucketName)
.destinationKey(destinationKey)
.build();
return s3Client.copyObject(copyObjectRequest);
}
//deleting an object
public void deleteObject(String bucketName, String objectKey) {
DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
.bucket(bucketName)
.key(objectKey)
.build();
s3Client.deleteObject(deleteObjectRequest);
}
//deleting multiple Objects
public void deleteObjects(String bucketName, List<String> keys ) {
ArrayList<ObjectIdentifier> toDelete = new ArrayList<>();
for(String objKey : keys) {
toDelete.add(ObjectIdentifier.builder()
.key(objKey)
.build());
}
DeleteObjectsRequest deleteObjectRequest = DeleteObjectsRequest.builder()
.bucket(bucketName)
.delete(Delete.builder()
.objects(toDelete).build())
.build();
s3Client.deleteObjects(deleteObjectRequest);
}
public boolean doesObjectExistByListObjects(String bucketName, String key) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
return listObjectsV2Response.contents()
.stream()
.filter(s3ObjectSummary -> s3ObjectSummary.getValueForField("key", String.class)
.equals(key))
.findFirst()
.isPresent();
}
public boolean doesObjectExistByDefaultMethod(String bucket, String key) {
try {
HeadObjectRequest headObjectRequest = HeadObjectRequest.builder()
.bucket(bucket)
.key(key)
.build();
s3Client.headObject(headObjectRequest);
System.out.println("Object exists");
return true;
} catch (S3Exception e) {
if (e.statusCode() == 404) {
System.out.println("Object does not exist");
return false;
} else {
throw e;
}
}
}
public void listBuckets() {
// List all buckets
ListBucketsResponse listBucketsResponse = s3Client.listBuckets();
// Display the bucket names
List<Bucket> buckets = listBucketsResponse.buckets();
System.out.println("Buckets:");
for (Bucket bucket : buckets) {
System.out.println(bucket.name());
}
}
public void listObjectsInBucket(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
List<S3Object> contents = listObjectsV2Response.contents();
System.out.println("Number of objects in the bucket: " + contents.stream().count());
contents.stream().forEach(System.out::println);
}
public void listAllObjectsInBucket(String bucketName) {
String nextContinuationToken = null;
long totalObjects = 0;
do {
ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder()
.bucket(bucketName)
.continuationToken(nextContinuationToken);
ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
nextContinuationToken = response.nextContinuationToken();
totalObjects += response.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
} while (nextContinuationToken != null);
System.out.println("Number of objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.prefix(prefix)
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void putObjects(String bucketName, List<File> files) {
try {
files.stream().forEach(file -> {
s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(),
RequestBody.fromByteBuffer(file.getContent()));
System.out.println("Uploaded file: " + file.getName());
});
} catch (S3Exception e) {
System.err.println("Upload failed");
e.printStackTrace();
}
}
public void cleanup() {
this.s3Client.close();
}
}

3
aws-modules/aws-s3/src/main/resources/logback.xml
View File

@ -10,4 +10,7 @@
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
<!-- Change the log level for software.amazon.awssdk -->
<logger name="software.amazon.awssdk" level="info" />
</configuration>

49
aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java
View File

@ -1,5 +1,13 @@
package com.baeldung.jets3t;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -10,8 +18,8 @@ import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.StorageObject;
import org.jets3t.service.security.AWSCredentials;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.*;
import java.nio.file.Files;
@ -19,14 +27,13 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.stream.Collectors;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static junit.framework.TestCase.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class JetS3tLiveTest {
private Log log = LogFactory.getLog(JetS3tLiveTest.class);
private final Log log = LogFactory.getLog(JetS3tLiveTest.class);
private static final String BucketName = "baeldung-barfoo";
private static final String TestString = "test string";
@ -35,7 +42,7 @@ public class JetS3tLiveTest {
private static S3Service s3Service;
@BeforeClass
@BeforeAll
public static void connectS3() throws Exception {
// Replace with your keys
@ -50,7 +57,7 @@ public class JetS3tLiveTest {
}
@Test
public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception {
void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception {
// List buckets, get a count
S3Bucket[] myBuckets = s3Service.listAllBuckets();
@ -89,7 +96,7 @@ public class JetS3tLiveTest {
}
@Test
public void givenString_Uploaded_StringInfoIsAvailable() throws Exception {
void givenString_Uploaded_StringInfoIsAvailable() throws Exception {
// Create a bucket
S3Bucket bucket = createBucket();
@ -120,7 +127,7 @@ public class JetS3tLiveTest {
}
@Test
public void givenStringUploaded_StringIsDownloaded() throws Exception {
void givenStringUploaded_StringIsDownloaded() throws Exception {
// Get a bucket
S3Bucket bucket = createBucket();
@ -135,7 +142,7 @@ public class JetS3tLiveTest {
String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n"));
// Verify
assertTrue(TestString.equals(downloadedString));
assertEquals(TestString, downloadedString);
// Clean up for next test
@ -144,7 +151,7 @@ public class JetS3tLiveTest {
}
@Test
public void givenBinaryFileUploaded_FileIsDownloaded() throws Exception {
void givenBinaryFileUploaded_FileIsDownloaded() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
@ -169,7 +176,7 @@ public class JetS3tLiveTest {
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/newtest.jpg");
assertTrue(origMD5.equals(newMD5));
assertEquals(origMD5, newMD5);
// Clean up
deleteObject("test.jpg");
@ -186,7 +193,7 @@ public class JetS3tLiveTest {
@Test
public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception {
void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
@ -233,7 +240,7 @@ public class JetS3tLiveTest {
}
@Test
public void whenFileCopied_CopyIsSame() throws Exception {
void whenFileCopied_CopyIsSame() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
@ -260,7 +267,7 @@ public class JetS3tLiveTest {
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/testcopy.jpg");
assertTrue(origMD5.equals(newMD5));
assertEquals(origMD5, newMD5);
// Clean up
deleteObject("test.jpg");
@ -271,7 +278,7 @@ public class JetS3tLiveTest {
@Test
public void whenFileRenamed_NewNameIsSame() throws Exception {
void whenFileRenamed_NewNameIsSame() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
@ -297,7 +304,7 @@ public class JetS3tLiveTest {
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/spidey.jpg");
assertTrue(origMD5.equals(newMD5));
assertEquals(origMD5, newMD5);
// Clean up
deleteObject("test.jpg");
@ -307,7 +314,7 @@ public class JetS3tLiveTest {
}
@Test
public void whenFileMoved_NewInstanceIsSame() throws Exception {
void whenFileMoved_NewInstanceIsSame() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
@ -338,7 +345,7 @@ public class JetS3tLiveTest {
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/spidey.jpg");
assertTrue(origMD5.equals(newMD5));
assertEquals(origMD5, newMD5);
// Clean up
deleteBucket();

49
aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java
View File

@ -1,49 +0,0 @@
package com.baeldung.s3;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.Before;
import org.junit.Test;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
/**
* Required defined environment variables AWS_ACCESS_KEY_ID & AWS_ACCESS_KEY to access S3.
* Required S3 bucket and key that exist.
*/
public class AWSS3ObjectManualTest {
private static final String BUCKET = "your-bucket";
private static final String KEY_THAT_EXIST = "your-key-that-exist";
private AWSS3ObjectUtils s3ObjectUtils;
@Before
public void setUp() {
AmazonS3 client = AmazonS3ClientBuilder.standard()
.withRegion(Regions.DEFAULT_REGION)
.withCredentials(new EnvironmentVariableCredentialsProvider())
.build();
s3ObjectUtils = new AWSS3ObjectUtils(client);
}
@Test
public void whenVerifyIfObjectExistByDefaultMethod_thenCorrect() {
assertTrue(s3ObjectUtils.doesObjectExistByDefaultMethod(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist");
}
@Test
public void whenVerifyIfObjectExistByListObjects_thenCorrect() {
assertTrue(s3ObjectUtils.doesObjectExistByListObjects(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist");
}
@Test
public void whenVerifyIfObjectExistByMetaData_thenCorrect() {
assertTrue(s3ObjectUtils.doesObjectExistByMetaData(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist");
}
}

113
aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java
View File

@ -1,113 +0,0 @@
package com.baeldung.s3;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import org.junit.Before;
import org.junit.Test;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsResult;
import com.amazonaws.services.s3.model.PutObjectResult;
public class AWSS3ServiceIntegrationTest {
private static final String BUCKET_NAME = "bucket_name";
private static final String KEY_NAME = "key_name";
private static final String BUCKET_NAME2 = "bucket_name2";
private static final String KEY_NAME2 = "key_name2";
private AmazonS3 s3;
private AWSS3Service service;
@Before
public void setUp() {
s3 = mock(AmazonS3.class);
service = new AWSS3Service(s3);
}
@Test
public void whenInitializingAWSS3Service_thenNotNull() {
assertThat(new AWSS3Service()).isNotNull();
}
@Test
public void whenVerifyingIfS3BucketExist_thenCorrect() {
service.doesBucketExist(BUCKET_NAME);
verify(s3).doesBucketExist(BUCKET_NAME);
}
@Test
public void whenVerifyingCreationOfS3Bucket_thenCorrect() {
service.createBucket(BUCKET_NAME);
verify(s3).createBucket(BUCKET_NAME);
}
@Test
public void whenVerifyingListBuckets_thenCorrect() {
service.listBuckets();
verify(s3).listBuckets();
}
@Test
public void whenDeletingBucket_thenCorrect() {
service.deleteBucket(BUCKET_NAME);
verify(s3).deleteBucket(BUCKET_NAME);
}
@Test
public void whenVerifyingPutObject_thenCorrect() {
File file = mock(File.class);
PutObjectResult result = mock(PutObjectResult.class);
when(s3.putObject(anyString(), anyString(), (File) any())).thenReturn(result);
assertThat(service.putObject(BUCKET_NAME, KEY_NAME, file)).isEqualTo(result);
verify(s3).putObject(BUCKET_NAME, KEY_NAME, file);
}
@Test
public void whenVerifyingListObjects_thenCorrect() {
service.listObjects(BUCKET_NAME);
verify(s3).listObjects(BUCKET_NAME);
}
@Test
public void whenVerifyingGetObject_thenCorrect() {
service.getObject(BUCKET_NAME, KEY_NAME);
verify(s3).getObject(BUCKET_NAME, KEY_NAME);
}
@Test
public void whenVerifyingCopyObject_thenCorrect() {
CopyObjectResult result = mock(CopyObjectResult.class);
when(s3.copyObject(anyString(), anyString(), anyString(), anyString())).thenReturn(result);
assertThat(service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result);
verify(s3).copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2);
}
@Test
public void whenVerifyingDeleteObject_thenCorrect() {
service.deleteObject(BUCKET_NAME, KEY_NAME);
verify(s3).deleteObject(BUCKET_NAME, KEY_NAME);
}
@Test
public void whenVerifyingDeleteObjects_thenCorrect() {
DeleteObjectsRequest request = mock(DeleteObjectsRequest.class);
DeleteObjectsResult result = mock(DeleteObjectsResult.class);
when(s3.deleteObjects((DeleteObjectsRequest)any())).thenReturn(result);
assertThat(service.deleteObjects(request)).isEqualTo(result);
verify(s3).deleteObjects(request);
}
}

61
aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java
View File

@ -1,61 +0,0 @@
package com.baeldung.s3;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.util.concurrent.Executors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class MultipartUploadLiveTest {
private static final String BUCKET_NAME = "bucket_name";
private static final String KEY_NAME = "picture.jpg";
private AmazonS3 amazonS3;
private TransferManager tm;
private ProgressListener progressListener;
@Before
public void setup() {
amazonS3 = mock(AmazonS3.class);
tm = TransferManagerBuilder
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold((long) (5 * 1024 * 1025))
.withExecutorFactory(() -> Executors.newFixedThreadPool(5))
.build();
progressListener =
progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
}
@Test
public void whenUploadingFileWithTransferManager_thenVerifyUploadRequested() {
File file = mock(File.class);
PutObjectResult s3Result = mock(PutObjectResult.class);
when(amazonS3.putObject(anyString(), anyString(), (File) any())).thenReturn(s3Result);
when(file.getName()).thenReturn(KEY_NAME);
PutObjectRequest request = new PutObjectRequest(BUCKET_NAME, KEY_NAME, file);
request.setGeneralProgressListener(progressListener);
Upload upload = tm.upload(request);
assertThat(upload).isNotNull();
verify(amazonS3).putObject(request);
}
}

137
aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java Normal file
View File

@ -0,0 +1,137 @@
package com.baeldung.s3;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import java.util.Collections;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
class S3ServiceIntegrationTest {
private static final String BUCKET_NAME = "bucket_name";
private static final String KEY_NAME = "key_name";
private static final String BUCKET_NAME2 = "bucket_name2";
private static final String KEY_NAME2 = "key_name2";
@Mock
private S3Client s3Client;
private S3Service s3Service;
private final String AWS_BUCKET = "baeldung-tutorial-s3";
@BeforeEach
public void setup() {
MockitoAnnotations.openMocks(this);
s3Service = new S3Service(s3Client);
}
@AfterEach
public void cleanup() {
s3Service.cleanup();
Mockito.verify(s3Client).close();
}
@Test
void whenInitializingAWSS3Service_thenNotNull() {
assertThat(new S3Service(s3Client)).isNotNull();
}
@Test
void whenVerifyingIfS3BucketExist_thenCorrect() {
HeadBucketRequest headBucketRequest = HeadBucketRequest.builder()
.bucket(BUCKET_NAME)
.build();
s3Service.doesBucketExist(BUCKET_NAME);
verify(s3Client).headBucket(headBucketRequest);
}
@Test
void whenVerifyingCreationOfS3Bucket_thenCorrect() {
CreateBucketRequest bucketRequest = CreateBucketRequest.builder()
.bucket(BUCKET_NAME)
.build();
s3Service.createBucket(BUCKET_NAME);
verify(s3Client).createBucket(bucketRequest);
}
@Test
void whenVerifyingListBuckets_thenCorrect() {
when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build());
s3Service.listBuckets();
Mockito.verify(s3Client).listBuckets();
}
@Test
void whenDeletingBucket_thenCorrect() {
DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder()
.bucket(BUCKET_NAME)
.build();
s3Service.deleteBucket(BUCKET_NAME);
verify(s3Client).deleteBucket(deleteBucketRequest);
}
@Test
void whenVerifyingListObjects_thenCorrect() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listObjects(AWS_BUCKET);
verify(s3Client).listObjectsV2(request);
}
@Test
void whenVerifyingCopyObject_thenCorrect() {
CopyObjectRequest request = CopyObjectRequest.builder()
.sourceBucket(BUCKET_NAME)
.sourceKey(KEY_NAME)
.destinationBucket(BUCKET_NAME2)
.destinationKey(KEY_NAME2)
.build();
CopyObjectResponse result = CopyObjectResponse.builder().build();
when(s3Client.copyObject(request)).thenReturn(result);
assertThat(s3Service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result);
verify(s3Client).copyObject(request);
}
@Test
void whenVerifyingDeleteObject_thenCorrect() {
DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
.bucket(BUCKET_NAME)
.key(KEY_NAME)
.build();
s3Service.deleteObject(BUCKET_NAME, KEY_NAME);
verify(s3Client).deleteObject(deleteObjectRequest);
}
}

218
aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java → aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceLiveTest.java
View File

@ -1,109 +1,109 @@
package com.baeldung.s3.listobjects;
import com.baeldung.s3.listobjects.S3Service;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import java.util.Arrays;
import java.util.Collections;
import static org.mockito.Mockito.when;
class S3ServiceLiveTest {
@Mock
private S3Client s3Client;
private S3Service s3Service;
private String AWS_BUCKET = "baeldung-tutorial-s3";
@BeforeEach
public void setup() {
MockitoAnnotations.openMocks(this);
s3Service = new S3Service(s3Client);
}
@AfterEach
public void cleanup() {
s3Service.cleanup();
Mockito.verify(s3Client).close();
}
@Test
public void givenBucketName_whenListObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
public void givenBucketName_whenListAllObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
public void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() {
int pageSize = 10;
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("object1").build();
S3Object s3Object2 = S3Object.builder().key("object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
public void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() {
int pageSize = 1;
String prefix = "folder/";
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("folder/object1").build();
S3Object s3Object2 = S3Object.builder().key("folder/object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
public void whenListBuckets_thenReturnBucketList() {
when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build());
s3Service.listBuckets();
Mockito.verify(s3Client).listBuckets();
}
}
package com.baeldung.s3;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import java.util.Arrays;
import java.util.Collections;
class S3ServiceLiveTest {
@Mock
private S3Client s3Client;
private S3Service s3Service;
private String AWS_BUCKET = "baeldung-tutorial-s3";
@BeforeEach
public void setup() {
MockitoAnnotations.openMocks(this);
s3Service = new S3Service(s3Client);
}
@AfterEach
public void cleanup() {
s3Service.cleanup();
Mockito.verify(s3Client).close();
}
@Test
void givenBucketName_whenListObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
void givenBucketName_whenListAllObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() {
int pageSize = 10;
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("object1").build();
S3Object s3Object2 = S3Object.builder().key("object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() {
int pageSize = 1;
String prefix = "folder/";
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("folder/object1").build();
S3Object s3Object2 = S3Object.builder().key("folder/object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
void whenListBuckets_thenReturnBucketList() {
when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build());
s3Service.listBuckets();
Mockito.verify(s3Client).listBuckets();
}
}

51
aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java Normal file
View File

@ -0,0 +1,51 @@
package com.baeldung.s3;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import software.amazon.awssdk.services.s3.S3Client;
/**
* Required defined environment variables AWS_ACCESS_KEY_ID & AWS_ACCESS_KEY to access S3.
* Required S3 bucket and key that exist.
*/
class S3ServiceManualTest {
private static final String BUCKET_NAME = "bucket_name";
private static final String KEY_NAME = "key_name";
@Mock
private S3Client s3Client;
private S3Service s3Service;
@BeforeEach
public void setup() {
MockitoAnnotations.openMocks(this);
s3Service = new S3Service(s3Client);
}
@AfterEach
public void cleanup() {
s3Service.cleanup();
Mockito.verify(s3Client).close();
}
@Test
void whenVerifyIfObjectExistByDefaultMethod_thenCorrect() {
assertTrue(s3Service.doesObjectExistByDefaultMethod(BUCKET_NAME, KEY_NAME), "Key: " + KEY_NAME + " doesn't exist");
}
@Test
void whenVerifyIfObjectExistByListObjects_thenCorrect() {
assertTrue(s3Service.doesObjectExistByListObjects(BUCKET_NAME, KEY_NAME), "Key: " + KEY_NAME + " doesn't exist");
}
}

1
aws-modules/pom.xml
View File

@ -19,7 +19,6 @@
<module>aws-miscellaneous</module>
<module>aws-reactive</module>
<module>aws-s3</module>
<module>aws-s3-v2</module>
</modules>
<properties>