From f7bd212ec5bb0949b6e9bdc12d9ecf28afb025d9 Mon Sep 17 00:00:00 2001 From: panos-kakos Date: Wed, 5 Jul 2023 08:38:36 +0300 Subject: [PATCH] [JAVA-22456] Upgraded aws-s3 module to aws-sdk 2.20.52 --- aws-modules/aws-s3/pom.xml | 12 +- .../com/baeldung/s3/AWSS3ObjectUtils.java | 42 --- .../java/com/baeldung/s3/AWSS3Service.java | 93 ------- .../java/com/baeldung/s3/MultipartUpload.java | 115 ++++++--- .../java/com/baeldung/s3/S3Application.java | 114 +++------ .../main/java/com/baeldung/s3/S3Service.java | 241 ++++++++++++++++++ .../com/baeldung/jets3t/JetS3tLiveTest.java | 49 ++-- .../baeldung/s3/AWSS3ObjectManualTest.java | 49 ---- .../s3/AWSS3ServiceIntegrationTest.java | 113 -------- .../baeldung/s3/MultipartUploadLiveTest.java | 61 ----- .../baeldung/s3/S3ServiceIntegrationTest.java | 137 ++++++++++ .../com/baeldung/s3/S3ServiceManualTest.java | 51 ++++ 12 files changed, 579 insertions(+), 498 deletions(-) delete mode 100644 aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java delete mode 100644 aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java create mode 100644 aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java delete mode 100644 aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java delete mode 100644 aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java delete mode 100644 aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java create mode 100644 aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java create mode 100644 aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java diff --git a/aws-modules/aws-s3/pom.xml b/aws-modules/aws-s3/pom.xml index 0ea99588d8..157aeb671d 100644 --- a/aws-modules/aws-s3/pom.xml +++ b/aws-modules/aws-s3/pom.xml @@ -1,7 +1,7 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 aws-s3 0.1.0-SNAPSHOT @@ -16,10 +16,11 @@ - com.amazonaws - aws-java-sdk - ${aws-java-sdk.version} + software.amazon.awssdk + s3 + ${aws.java.sdk.version} + commons-io commons-io @@ -60,6 +61,7 @@ + 2.20.52 1.10.L001 0.9.4.0006L diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java deleted file mode 100644 index 1c31218ff9..0000000000 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3ObjectUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.baeldung.s3; - -import org.apache.http.HttpStatus; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; - -public class AWSS3ObjectUtils { - - private AmazonS3 s3Client; - - public AWSS3ObjectUtils(AmazonS3 s3client) { - this.s3Client = s3client; - } - - public boolean doesObjectExistByDefaultMethod(String bucket, String key) { - return s3Client.doesObjectExist(bucket, key); - } - - public boolean doesObjectExistByListObjects(String bucket, String key) { - return s3Client.listObjects(bucket) - .getObjectSummaries() - .stream() - .filter(s3ObjectSummary -> s3ObjectSummary.getKey() - .equals(key)) - .findFirst() - .isPresent(); - } - - public boolean doesObjectExistByMetaData(String bucket, String key) { - try { - s3Client.getObjectMetadata(bucket, key); - return true; - } catch (AmazonServiceException e) { - if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { - return false; - } else { - throw e; - } - } - } -} \ No newline at end of file diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java deleted file mode 100644 index 64ed043f98..0000000000 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/AWSS3Service.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.baeldung.s3; - -import java.io.File; -import java.net.URL; -import java.util.List; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; - -public class AWSS3Service { - private final AmazonS3 s3client; - - public AWSS3Service() { - this(new AmazonS3Client() { - }); - } - - public AWSS3Service(AmazonS3 s3client) { - this.s3client = s3client; - } - - //is bucket exist? - public boolean doesBucketExist(String bucketName) { - return s3client.doesBucketExist(bucketName); - } - - //create a bucket - public Bucket createBucket(String bucketName) { - return s3client.createBucket(bucketName); - } - - //list all buckets - public List listBuckets() { - return s3client.listBuckets(); - } - - //delete a bucket - public void deleteBucket(String bucketName) { - s3client.deleteBucket(bucketName); - } - - //uploading object - public PutObjectResult putObject(String bucketName, String key, File file) { - return s3client.putObject(bucketName, key, file); - } - //uploading object and getting url - public URL getObjectURL(String bucketName, String key, File file) { - s3client.putObject(bucketName, key, file); - return s3client.getUrl(bucketName, key); - } - - //listing objects - public ObjectListing listObjects(String bucketName) { - return s3client.listObjects(bucketName); - } - - //get an object - public S3Object getObject(String bucketName, String objectKey) { - return s3client.getObject(bucketName, objectKey); - } - - //copying an object - public CopyObjectResult copyObject( - String sourceBucketName, - String sourceKey, - String destinationBucketName, - String destinationKey - ) { - return s3client.copyObject( - sourceBucketName, - sourceKey, - destinationBucketName, - destinationKey - ); - } - - //deleting an object - public void deleteObject(String bucketName, String objectKey) { - s3client.deleteObject(bucketName, objectKey); - } - - //deleting multiple Objects - public DeleteObjectsResult deleteObjects(DeleteObjectsRequest delObjReq) { - return s3client.deleteObjects(delObjReq); - } -} diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java index 711046c112..de45238994 100644 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/MultipartUpload.java @@ -1,56 +1,101 @@ package com.baeldung.s3; -import com.amazonaws.AmazonClientException; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.event.ProgressListener; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.*; -import java.io.File; -import java.util.concurrent.Executors; public class MultipartUpload { - public static void main(String[] args) throws Exception { + public static void main(String[] args) { String existingBucketName = "baeldung-bucket"; String keyName = "my-picture.jpg"; String filePath = "documents/my-picture.jpg"; - AmazonS3 amazonS3 = AmazonS3ClientBuilder - .standard() - .withCredentials(new DefaultAWSCredentialsProviderChain()) - .withRegion(Regions.DEFAULT_REGION) - .build(); + ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(); + Region region = Region.US_EAST_1; + S3Client s3 = S3Client.builder() + .region(region) + .credentialsProvider(credentialsProvider) + .build(); - int maxUploadThreads = 5; + // Initiate a multipart upload + CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .build(); - TransferManager tm = TransferManagerBuilder - .standard() - .withS3Client(amazonS3) - .withMultipartUploadThreshold((long) (5 * 1024 * 1024)) - .withExecutorFactory(() -> Executors.newFixedThreadPool(maxUploadThreads)) - .build(); + CreateMultipartUploadResponse createResponse = s3.createMultipartUpload(createRequest); - ProgressListener progressListener = - progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred()); + String uploadId = createResponse.uploadId(); - PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath)); + // Prepare the parts to be uploaded + List completedParts = new ArrayList<>(); + int partNumber = 1; + ByteBuffer buffer = ByteBuffer.allocate(5 * 1024 * 1024); // Set your preferred part size (5 MB in this example) - request.setGeneralProgressListener(progressListener); + // Read the file and upload each part + try (RandomAccessFile file = new RandomAccessFile(filePath, "r")) { + long fileSize = file.length(); + long position = 0; - Upload upload = tm.upload(request); + while (position < fileSize) { + file.seek(position); + int bytesRead = file.getChannel().read(buffer); - try { - upload.waitForCompletion(); - System.out.println("Upload complete."); - } catch (AmazonClientException e) { - System.out.println("Error occurred while uploading file"); + buffer.flip(); + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength((long) bytesRead) + .build(); + + UploadPartResponse response = s3.uploadPart(uploadPartRequest, RequestBody.fromByteBuffer(buffer)); + + completedParts.add(CompletedPart.builder() + .partNumber(partNumber) + .eTag(response.eTag()) + .build()); + + buffer.clear(); + position += bytesRead; + partNumber++; + } + } catch (IOException e) { e.printStackTrace(); } + + // Complete the multipart upload + CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .uploadId(uploadId) + .multipartUpload(completedUpload) + .build(); + + CompleteMultipartUploadResponse completeResponse = s3.completeMultipartUpload(completeRequest); + + // Print the object's URL + String objectUrl = s3.utilities().getUrl(GetUrlRequest.builder() + .bucket(existingBucketName) + .key(keyName) + .build()) + .toExternalForm(); + + System.out.println("Uploaded object URL: " + objectUrl); } } \ No newline at end of file diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java index 39a9b0050f..1ce2f9cbb5 100644 --- a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java @@ -1,104 +1,60 @@ package com.baeldung.s3; import java.io.File; -import java.io.IOException; +import java.util.List; -import org.apache.commons.io.FileUtils; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.regions.Region; public class S3Application { - private static final AWSCredentials credentials; - private static String bucketName; + private static String AWS_BUCKET = "baeldung-tutorial-s3"; + public static Region AWS_REGION = Region.EU_CENTRAL_1; - static { - //put your accesskey and secretkey here - credentials = new BasicAWSCredentials( - "", - "" - ); - } - - public static void main(String[] args) throws IOException { - //set-up the client - AmazonS3 s3client = AmazonS3ClientBuilder - .standard() - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .withRegion(Regions.US_EAST_2) - .build(); - - AWSS3Service awsService = new AWSS3Service(s3client); - - bucketName = "baeldung-bucket"; + public static void main(String[] args) { + S3Service s3Service = new S3Service(AWS_REGION); //creating a bucket - if(awsService.doesBucketExist(bucketName)) { + s3Service.createBucket(AWS_BUCKET); + + //check if a bucket exists + if(s3Service.doesBucketExist(AWS_BUCKET)) { System.out.println("Bucket name is not available." - + " Try again with a different Bucket name."); + + " Try again with a different Bucket name."); return; } - awsService.createBucket(bucketName); - - //list all the buckets - for(Bucket s : awsService.listBuckets() ) { - System.out.println(s.getName()); - } - - //deleting bucket - awsService.deleteBucket("baeldung-bucket-test2"); - - //uploading object - awsService.putObject( - bucketName, - "Document/hello.txt", - new File("/Users/user/Document/hello.txt") - ); - //uploading object and getting url - awsService.getObjectURL(bucketName, "Document/hello.txt", new File("/Users/user/Document/hello.txt")); + //list all the buckets + s3Service.listBuckets(); + + //deleting bucket + s3Service.deleteBucket("baeldung-bucket-test2"); + + //uploading object + s3Service.putObject( + AWS_BUCKET, + "Document/hello.txt", + new File("/Users/user/Document/hello.txt") + ); //listing objects - ObjectListing objectListing = awsService.listObjects(bucketName); - for(S3ObjectSummary os : objectListing.getObjectSummaries()) { - System.out.println(os.getKey()); - } + s3Service.listObjects(AWS_BUCKET); //downloading an object - S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt"); - S3ObjectInputStream inputStream = s3object.getObjectContent(); - FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt")); - + s3Service.getObject(AWS_BUCKET, "Document/hello.txt"); + //copying an object - awsService.copyObject( - "baeldung-bucket", - "picture/pic.png", - "baeldung-bucket2", - "Document/picture.png" + s3Service.copyObject( + "baeldung-bucket", + "picture/pic.png", + "baeldung-bucket2", + "Document/picture.png" ); - + //deleting an object - awsService.deleteObject(bucketName, "Document/hello.txt"); + s3Service.deleteObject(AWS_BUCKET, "Document/hello.txt"); //deleting multiple objects - String objkeyArr[] = { - "Document/hello2.txt", - "Document/picture.png" - }; - - DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket") - .withKeys(objkeyArr); - awsService.deleteObjects(delObjReq); + List objKeyList = List.of("Document/hello2.txt", "Document/picture.png"); + s3Service.deleteObjects(AWS_BUCKET, objKeyList); } } diff --git a/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java new file mode 100644 index 0000000000..361e7e83b0 --- /dev/null +++ b/aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java @@ -0,0 +1,241 @@ +package com.baeldung.s3; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Bucket; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +class S3Service { + + private S3Client s3Client; + + public S3Service(Region awsRegion) { + init(awsRegion); + } + + public S3Service(S3Client s3Client) { + this.s3Client = s3Client; + } + + public void init(Region awsRegion) { + this.s3Client = S3Client.builder() + .region(awsRegion) + .credentialsProvider(ProfileCredentialsProvider.create("default")) + .build(); + } + + //is bucket exist? + public boolean doesBucketExist(String bucketName) { + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder() + .bucket(bucketName) + .build(); + + try { + s3Client.headBucket(headBucketRequest); + return true; + } catch (NoSuchBucketException e) { + return false; + } + } + + //create a bucket + public void createBucket(String bucketName) { + CreateBucketRequest bucketRequest = CreateBucketRequest.builder() + .bucket(bucketName) + .build(); + + s3Client.createBucket(bucketRequest); + } + + //delete a bucket + public void deleteBucket(String bucketName) { + try { + DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder() + .bucket(bucketName) + .build(); + + s3Client.deleteBucket(deleteBucketRequest); + System.out.println("Successfully deleted bucket : " + bucketName); + } catch (S3Exception e) { + System.err.println(e.getMessage()); + System.exit(1); + } + } + + //uploading object + public PutObjectResponse putObject(String bucketName, String key, java.io.File file) { + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + return s3Client.putObject(request, Path.of(file.toURI()) ); + } + + //listing objects + public void listObjects(String bucketName) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); + + for(S3Object os : listObjectsV2Response.contents()) { + System.out.println(os.key()); + } + } + + //get an object + public void getObject(String bucketName, String objectKey) { + try { + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + + ResponseBytes responseResponseBytes = s3Client.getObjectAsBytes(objectRequest); + + byte[] data = responseResponseBytes.asByteArray(); + + // Write the data to a local file. + java.io.File myFile = new java.io.File("/Users/user/Desktop/hello.txt" ); + OutputStream os = new FileOutputStream(myFile); + os.write(data); + System.out.println("Successfully obtained bytes from an S3 object"); + os.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } catch (S3Exception e) { + System.err.println(e.awsErrorDetails().errorMessage()); + System.exit(1); + } + } + + //copying an object + public CopyObjectResponse copyObject( + String sourceBucketName, + String sourceKey, + String destinationBucketName, + String destinationKey + ) { + CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() + .sourceBucket(sourceBucketName) + .sourceKey(sourceKey) + .destinationBucket(destinationBucketName) + .destinationKey(destinationKey) + .build(); + + return s3Client.copyObject(copyObjectRequest); + } + + //deleting an object + public void deleteObject(String bucketName, String objectKey) { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + + s3Client.deleteObject(deleteObjectRequest); + } + + //deleting multiple Objects + public void deleteObjects(String bucketName, List keys ) { + + ArrayList toDelete = new ArrayList<>(); + for(String objKey : keys) { + toDelete.add(ObjectIdentifier.builder() + .key(objKey) + .build()); + } + + DeleteObjectsRequest deleteObjectRequest = DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(Delete.builder() + .objects(toDelete).build()) + .build(); + + s3Client.deleteObjects(deleteObjectRequest); + } + + public boolean doesObjectExistByListObjects(String bucketName, String key) { + + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); + + return listObjectsV2Response.contents() + .stream() + .filter(s3ObjectSummary -> s3ObjectSummary.getValueForField("key", String.class) + .equals(key)) + .findFirst() + .isPresent(); + } + + public boolean doesObjectExistByDefaultMethod(String bucket, String key) { + try { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucket) + .key(key) + .build(); + + s3Client.headObject(headObjectRequest); + + System.out.println("Object exists"); + return true; + } catch (S3Exception e) { + if (e.statusCode() == 404) { + System.out.println("Object does not exist"); + return false; + } else { + throw e; + } + } + } + + public void listBuckets() { + // List all buckets + ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); + + // Display the bucket names + List buckets = listBucketsResponse.buckets(); + System.out.println("Buckets:"); + for (Bucket bucket : buckets) { + System.out.println(bucket.name()); + } + } + + public void cleanup() { + this.s3Client.close(); + } + +} \ No newline at end of file diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java index cf0b203e10..cad7da6cc9 100644 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/jets3t/JetS3tLiveTest.java @@ -1,5 +1,13 @@ package com.baeldung.jets3t; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -10,8 +18,8 @@ import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; import org.jets3t.service.model.StorageObject; import org.jets3t.service.security.AWSCredentials; -import org.junit.BeforeClass; -import org.junit.Test; + + import java.io.*; import java.nio.file.Files; @@ -19,14 +27,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.stream.Collectors; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static junit.framework.TestCase.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; + + + public class JetS3tLiveTest { - private Log log = LogFactory.getLog(JetS3tLiveTest.class); + private final Log log = LogFactory.getLog(JetS3tLiveTest.class); private static final String BucketName = "baeldung-barfoo"; private static final String TestString = "test string"; @@ -35,7 +42,7 @@ public class JetS3tLiveTest { private static S3Service s3Service; - @BeforeClass + @BeforeAll public static void connectS3() throws Exception { // Replace with your keys @@ -50,7 +57,7 @@ public class JetS3tLiveTest { } @Test - public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception { + void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception { // List buckets, get a count S3Bucket[] myBuckets = s3Service.listAllBuckets(); @@ -89,7 +96,7 @@ public class JetS3tLiveTest { } @Test - public void givenString_Uploaded_StringInfoIsAvailable() throws Exception { + void givenString_Uploaded_StringInfoIsAvailable() throws Exception { // Create a bucket S3Bucket bucket = createBucket(); @@ -120,7 +127,7 @@ public class JetS3tLiveTest { } @Test - public void givenStringUploaded_StringIsDownloaded() throws Exception { + void givenStringUploaded_StringIsDownloaded() throws Exception { // Get a bucket S3Bucket bucket = createBucket(); @@ -135,7 +142,7 @@ public class JetS3tLiveTest { String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n")); // Verify - assertTrue(TestString.equals(downloadedString)); + assertEquals(TestString, downloadedString); // Clean up for next test @@ -144,7 +151,7 @@ public class JetS3tLiveTest { } @Test - public void givenBinaryFileUploaded_FileIsDownloaded() throws Exception { + void givenBinaryFileUploaded_FileIsDownloaded() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -169,7 +176,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/newtest.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteObject("test.jpg"); @@ -186,7 +193,7 @@ public class JetS3tLiveTest { @Test - public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception { + void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -233,7 +240,7 @@ public class JetS3tLiveTest { } @Test - public void whenFileCopied_CopyIsSame() throws Exception { + void whenFileCopied_CopyIsSame() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -260,7 +267,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/testcopy.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteObject("test.jpg"); @@ -271,7 +278,7 @@ public class JetS3tLiveTest { @Test - public void whenFileRenamed_NewNameIsSame() throws Exception { + void whenFileRenamed_NewNameIsSame() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -297,7 +304,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/spidey.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteObject("test.jpg"); @@ -307,7 +314,7 @@ public class JetS3tLiveTest { } @Test - public void whenFileMoved_NewInstanceIsSame() throws Exception { + void whenFileMoved_NewInstanceIsSame() throws Exception { // get a bucket S3Bucket bucket = createBucket(); @@ -338,7 +345,7 @@ public class JetS3tLiveTest { // Get hashes and compare String origMD5 = getFileMD5("src/test/resources/test.jpg"); String newMD5 = getFileMD5("src/test/resources/spidey.jpg"); - assertTrue(origMD5.equals(newMD5)); + assertEquals(origMD5, newMD5); // Clean up deleteBucket(); diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java deleted file mode 100644 index 02cc22367a..0000000000 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ObjectManualTest.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.baeldung.s3; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.Before; -import org.junit.Test; - -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; - -/** - * Required defined environment variables AWS_ACCESS_KEY_ID & AWS_ACCESS_KEY to access S3. - * Required S3 bucket and key that exist. - */ - -public class AWSS3ObjectManualTest { - - private static final String BUCKET = "your-bucket"; - private static final String KEY_THAT_EXIST = "your-key-that-exist"; - private AWSS3ObjectUtils s3ObjectUtils; - - @Before - public void setUp() { - AmazonS3 client = AmazonS3ClientBuilder.standard() - .withRegion(Regions.DEFAULT_REGION) - .withCredentials(new EnvironmentVariableCredentialsProvider()) - .build(); - - s3ObjectUtils = new AWSS3ObjectUtils(client); - } - - @Test - public void whenVerifyIfObjectExistByDefaultMethod_thenCorrect() { - assertTrue(s3ObjectUtils.doesObjectExistByDefaultMethod(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist"); - - } - - @Test - public void whenVerifyIfObjectExistByListObjects_thenCorrect() { - assertTrue(s3ObjectUtils.doesObjectExistByListObjects(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist"); - } - - @Test - public void whenVerifyIfObjectExistByMetaData_thenCorrect() { - assertTrue(s3ObjectUtils.doesObjectExistByMetaData(BUCKET, KEY_THAT_EXIST), "Key: " + KEY_THAT_EXIST + " doesn't exist"); - } -} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java deleted file mode 100644 index d386704513..0000000000 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/AWSS3ServiceIntegrationTest.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.baeldung.s3; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.File; - -import org.junit.Before; -import org.junit.Test; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.PutObjectResult; - -public class AWSS3ServiceIntegrationTest { - - private static final String BUCKET_NAME = "bucket_name"; - private static final String KEY_NAME = "key_name"; - private static final String BUCKET_NAME2 = "bucket_name2"; - private static final String KEY_NAME2 = "key_name2"; - - private AmazonS3 s3; - private AWSS3Service service; - - @Before - public void setUp() { - s3 = mock(AmazonS3.class); - service = new AWSS3Service(s3); - } - - @Test - public void whenInitializingAWSS3Service_thenNotNull() { - assertThat(new AWSS3Service()).isNotNull(); - } - - @Test - public void whenVerifyingIfS3BucketExist_thenCorrect() { - service.doesBucketExist(BUCKET_NAME); - verify(s3).doesBucketExist(BUCKET_NAME); - } - - @Test - public void whenVerifyingCreationOfS3Bucket_thenCorrect() { - service.createBucket(BUCKET_NAME); - verify(s3).createBucket(BUCKET_NAME); - } - - @Test - public void whenVerifyingListBuckets_thenCorrect() { - service.listBuckets(); - verify(s3).listBuckets(); - } - - @Test - public void whenDeletingBucket_thenCorrect() { - service.deleteBucket(BUCKET_NAME); - verify(s3).deleteBucket(BUCKET_NAME); - } - - @Test - public void whenVerifyingPutObject_thenCorrect() { - File file = mock(File.class); - PutObjectResult result = mock(PutObjectResult.class); - when(s3.putObject(anyString(), anyString(), (File) any())).thenReturn(result); - - assertThat(service.putObject(BUCKET_NAME, KEY_NAME, file)).isEqualTo(result); - verify(s3).putObject(BUCKET_NAME, KEY_NAME, file); - } - - @Test - public void whenVerifyingListObjects_thenCorrect() { - service.listObjects(BUCKET_NAME); - verify(s3).listObjects(BUCKET_NAME); - } - - @Test - public void whenVerifyingGetObject_thenCorrect() { - service.getObject(BUCKET_NAME, KEY_NAME); - verify(s3).getObject(BUCKET_NAME, KEY_NAME); - } - - @Test - public void whenVerifyingCopyObject_thenCorrect() { - CopyObjectResult result = mock(CopyObjectResult.class); - when(s3.copyObject(anyString(), anyString(), anyString(), anyString())).thenReturn(result); - - assertThat(service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result); - verify(s3).copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2); - } - - @Test - public void whenVerifyingDeleteObject_thenCorrect() { - service.deleteObject(BUCKET_NAME, KEY_NAME); - verify(s3).deleteObject(BUCKET_NAME, KEY_NAME); - } - - @Test - public void whenVerifyingDeleteObjects_thenCorrect() { - DeleteObjectsRequest request = mock(DeleteObjectsRequest.class); - DeleteObjectsResult result = mock(DeleteObjectsResult.class); - when(s3.deleteObjects((DeleteObjectsRequest)any())).thenReturn(result); - - assertThat(service.deleteObjects(request)).isEqualTo(result); - verify(s3).deleteObjects(request); - } - -} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java deleted file mode 100644 index e2a34b1c39..0000000000 --- a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/MultipartUploadLiveTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.baeldung.s3; - -import com.amazonaws.event.ProgressListener; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.util.concurrent.Executors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class MultipartUploadLiveTest { - - private static final String BUCKET_NAME = "bucket_name"; - private static final String KEY_NAME = "picture.jpg"; - - private AmazonS3 amazonS3; - private TransferManager tm; - private ProgressListener progressListener; - - @Before - public void setup() { - amazonS3 = mock(AmazonS3.class); - tm = TransferManagerBuilder - .standard() - .withS3Client(amazonS3) - .withMultipartUploadThreshold((long) (5 * 1024 * 1025)) - .withExecutorFactory(() -> Executors.newFixedThreadPool(5)) - .build(); - progressListener = - progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred()); - } - - @Test - public void whenUploadingFileWithTransferManager_thenVerifyUploadRequested() { - File file = mock(File.class); - PutObjectResult s3Result = mock(PutObjectResult.class); - - when(amazonS3.putObject(anyString(), anyString(), (File) any())).thenReturn(s3Result); - when(file.getName()).thenReturn(KEY_NAME); - - PutObjectRequest request = new PutObjectRequest(BUCKET_NAME, KEY_NAME, file); - request.setGeneralProgressListener(progressListener); - - Upload upload = tm.upload(request); - - assertThat(upload).isNotNull(); - verify(amazonS3).putObject(request); - } -} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java new file mode 100644 index 0000000000..bf24bcaa43 --- /dev/null +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceIntegrationTest.java @@ -0,0 +1,137 @@ +package com.baeldung.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.util.Collections; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; + +class S3ServiceIntegrationTest { + + private static final String BUCKET_NAME = "bucket_name"; + private static final String KEY_NAME = "key_name"; + private static final String BUCKET_NAME2 = "bucket_name2"; + private static final String KEY_NAME2 = "key_name2"; + + @Mock + private S3Client s3Client; + + private S3Service s3Service; + + private final String AWS_BUCKET = "baeldung-tutorial-s3"; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + s3Service = new S3Service(s3Client); + } + + @AfterEach + public void cleanup() { + s3Service.cleanup(); + Mockito.verify(s3Client).close(); + } + + @Test + void whenInitializingAWSS3Service_thenNotNull() { + assertThat(new S3Service(s3Client)).isNotNull(); + } + + @Test + void whenVerifyingIfS3BucketExist_thenCorrect() { + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder() + .bucket(BUCKET_NAME) + .build(); + + s3Service.doesBucketExist(BUCKET_NAME); + verify(s3Client).headBucket(headBucketRequest); + } + + @Test + void whenVerifyingCreationOfS3Bucket_thenCorrect() { + CreateBucketRequest bucketRequest = CreateBucketRequest.builder() + .bucket(BUCKET_NAME) + .build(); + + s3Service.createBucket(BUCKET_NAME); + verify(s3Client).createBucket(bucketRequest); + } + + @Test + void whenVerifyingListBuckets_thenCorrect() { + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build()); + + s3Service.listBuckets(); + Mockito.verify(s3Client).listBuckets(); + } + + @Test + void whenDeletingBucket_thenCorrect() { + DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder() + .bucket(BUCKET_NAME) + .build(); + + s3Service.deleteBucket(BUCKET_NAME); + verify(s3Client).deleteBucket(deleteBucketRequest); + } + + + + @Test + void whenVerifyingListObjects_thenCorrect() { + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); + + when(s3Client.listObjectsV2(request)).thenReturn(response); + + s3Service.listObjects(AWS_BUCKET); + verify(s3Client).listObjectsV2(request); + } + + + @Test + void whenVerifyingCopyObject_thenCorrect() { + CopyObjectRequest request = CopyObjectRequest.builder() + .sourceBucket(BUCKET_NAME) + .sourceKey(KEY_NAME) + .destinationBucket(BUCKET_NAME2) + .destinationKey(KEY_NAME2) + .build(); + + CopyObjectResponse result = CopyObjectResponse.builder().build(); + + when(s3Client.copyObject(request)).thenReturn(result); + + assertThat(s3Service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result); + verify(s3Client).copyObject(request); + } + + @Test + void whenVerifyingDeleteObject_thenCorrect() { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(BUCKET_NAME) + .key(KEY_NAME) + .build(); + + s3Service.deleteObject(BUCKET_NAME, KEY_NAME); + verify(s3Client).deleteObject(deleteObjectRequest); + } +} diff --git a/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java new file mode 100644 index 0000000000..defe75e346 --- /dev/null +++ b/aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceManualTest.java @@ -0,0 +1,51 @@ +package com.baeldung.s3; + + +import static org.junit.jupiter.api.Assertions.assertTrue; + + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import software.amazon.awssdk.services.s3.S3Client; + +/** + * Required defined environment variables AWS_ACCESS_KEY_ID & AWS_ACCESS_KEY to access S3. + * Required S3 bucket and key that exist. + */ + +class S3ServiceManualTest { + + private static final String BUCKET_NAME = "bucket_name"; + private static final String KEY_NAME = "key_name"; + @Mock + private S3Client s3Client; + + private S3Service s3Service; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + s3Service = new S3Service(s3Client); + } + + @AfterEach + public void cleanup() { + s3Service.cleanup(); + Mockito.verify(s3Client).close(); + } + + @Test + void whenVerifyIfObjectExistByDefaultMethod_thenCorrect() { + assertTrue(s3Service.doesObjectExistByDefaultMethod(BUCKET_NAME, KEY_NAME), "Key: " + KEY_NAME + " doesn't exist"); + } + + @Test + void whenVerifyIfObjectExistByListObjects_thenCorrect() { + assertTrue(s3Service.doesObjectExistByListObjects(BUCKET_NAME, KEY_NAME), "Key: " + KEY_NAME + " doesn't exist"); + } +}