diff --git a/aws-modules/aws-s3-v2/pom.xml b/aws-modules/aws-s3-v2/pom.xml new file mode 100644 index 0000000000..888d98e654 --- /dev/null +++ b/aws-modules/aws-s3-v2/pom.xml @@ -0,0 +1,50 @@ + + 4.0.0 + com.baeldung.s3 + aws-s3-v2 + 1.0-SNAPSHOT + aws-s3-v2 + jar + + + com.baeldung + aws-modules + 1.0.0-SNAPSHOT + + + + UTF-8 + 2.20.52 + 3.6.1 + + + + + software.amazon.awssdk + s3 + ${aws.java.sdk.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + false + + + + package + + shade + + + + + + + diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java new file mode 100644 index 0000000000..f9fe7f6fb3 --- /dev/null +++ b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java @@ -0,0 +1,21 @@ +package com.baeldung.s3.listobjects; + +import java.nio.ByteBuffer; + +public class File { + private String name; + private ByteBuffer content; + + public File(String name, ByteBuffer content) { + this.name = name; + this.content = content; + } + + public String getName() { + return name; + } + + public ByteBuffer getContent() { + return content; + } +} diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java new file mode 100644 index 0000000000..7c640591d7 --- /dev/null +++ b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java @@ -0,0 +1,27 @@ +package com.baeldung.s3.listobjects; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class FileGenerator { + + public static List generateFiles(int fileCount, int fileSize) { + List files = new ArrayList<>(); + for (int i = 0; i < fileCount; i++) { + String fileName = "file_" + i + ".txt"; + ByteBuffer fileContent = generateRandomBytes(fileSize); + files.add(new File(fileName, fileContent)); + } + return files; + } + + private static ByteBuffer generateRandomBytes(int size) { + byte[] array = new byte[size]; + new Random().nextBytes(array); + return ByteBuffer.wrap(array); + } + + +} diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java new file mode 100644 index 0000000000..7b7c95a0d0 --- /dev/null +++ b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java @@ -0,0 +1,22 @@ +package com.baeldung.s3.listobjects; + +import software.amazon.awssdk.regions.Region; + +public class Main { + + private static String AWS_BUCKET = "baeldung-tutorial-s3"; + public static Region AWS_REGION = Region.EU_CENTRAL_1; + + public static void main(String[] args) { + S3Service s3Service = new S3Service(AWS_REGION); + + s3Service.putObject(AWS_BUCKET, FileGenerator.generateFiles(1000, 1)); + s3Service.listBuckets(); + s3Service.listObjectsInBucket(AWS_BUCKET); + s3Service.listAllObjectsInBucket(AWS_BUCKET); + s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500); + s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/"); + + s3Service.cleanup(); + } +} diff --git a/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java new file mode 100644 index 0000000000..132123f214 --- /dev/null +++ b/aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java @@ -0,0 +1,129 @@ +package com.baeldung.s3.listobjects; + +import java.util.List; + +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +class S3Service { + + private S3Client s3Client; + + public S3Service(Region awsRegion) { + init(awsRegion); + } + + public S3Service(S3Client s3Client) { + this.s3Client = s3Client; + } + + public void init(Region awsRegion) { + this.s3Client = S3Client.builder() + .region(awsRegion) + .credentialsProvider(ProfileCredentialsProvider.create("default")) + .build(); + } + + public void listObjectsInBucket(String bucketName) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request); + + List contents = listObjectsV2Response.contents(); + + System.out.println("Number of objects in the bucket: " + contents.stream().count()); + contents.stream().forEach(System.out::println); + } + + public void listAllObjectsInBucket(String bucketName) { + String nextContinuationToken = null; + long totalObjects = 0; + + do { + ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() + .bucket(bucketName) + .continuationToken(nextContinuationToken); + + ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build()); + nextContinuationToken = response.nextContinuationToken(); + + totalObjects += response.contents().stream() + .peek(System.out::println) + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + } while (nextContinuationToken != null); + System.out.println("Number of objects in the bucket: " + totalObjects); + } + + public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(pageSize) // Set the maxKeys parameter to control the page size + .build(); + + ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request); + long totalObjects = 0; + + for (ListObjectsV2Response page : listObjectsV2Iterable) { + long retrievedPageSize = page.contents().stream() + .peek(System.out::println) + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + totalObjects += retrievedPageSize; + System.out.println("Page size: " + retrievedPageSize); + } + System.out.println("Total objects in the bucket: " + totalObjects); + } + + public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) { + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(pageSize) // Set the maxKeys parameter to control the page size + .prefix(prefix) + .build(); + + ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request); + long totalObjects = 0; + + for (ListObjectsV2Response page : listObjectsV2Iterable) { + long retrievedPageSize = page.contents().stream() + .peek(System.out::println) + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + totalObjects += retrievedPageSize; + System.out.println("Page size: " + retrievedPageSize); + } + System.out.println("Total objects in the bucket: " + totalObjects); + } + + public void listBuckets() { + // List all buckets + ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); + + // Display the bucket names + List buckets = listBucketsResponse.buckets(); + System.out.println("Buckets:"); + for (Bucket bucket : buckets) { + System.out.println(bucket.name()); + } + } + + public void putObject(String bucketName, List files) { + try { + files.stream().forEach(file -> { + s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(), + RequestBody.fromByteBuffer(file.getContent())); + System.out.println("Uploaded file: " + file.getName()); + }); + } catch (S3Exception e) { + System.err.println("Upload failed"); + e.printStackTrace(); + } + } + + public void cleanup() { + this.s3Client.close(); + } +} \ No newline at end of file diff --git a/aws-modules/aws-s3-v2/src/main/resources/logback.xml b/aws-modules/aws-s3-v2/src/main/resources/logback.xml new file mode 100644 index 0000000000..91d3d0cfa6 --- /dev/null +++ b/aws-modules/aws-s3-v2/src/main/resources/logback.xml @@ -0,0 +1,14 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java b/aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java new file mode 100644 index 0000000000..40b30f8669 --- /dev/null +++ b/aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java @@ -0,0 +1,109 @@ +package com.baeldung.s3.listobjects; + +import com.baeldung.s3.listobjects.S3Service; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.util.Arrays; +import java.util.Collections; + +import static org.mockito.Mockito.when; + +class S3ServiceLiveTest { + + @Mock + private S3Client s3Client; + + private S3Service s3Service; + + private String AWS_BUCKET = "baeldung-tutorial-s3"; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + s3Service = new S3Service(s3Client); + } + + @AfterEach + public void cleanup() { + s3Service.cleanup(); + Mockito.verify(s3Client).close(); + } + + @Test + public void givenBucketName_whenListObjectsInBucket_thenReturnList() { + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); + + when(s3Client.listObjectsV2(request)).thenReturn(response); + + s3Service.listObjectsInBucket(AWS_BUCKET); + Mockito.verify(s3Client).listObjectsV2(request); + } + + @Test + public void givenBucketName_whenListAllObjectsInBucket_thenReturnList() { + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build(); + + when(s3Client.listObjectsV2(request)).thenReturn(response); + + s3Service.listAllObjectsInBucket(AWS_BUCKET); + Mockito.verify(s3Client).listObjectsV2(request); + } + + @Test + public void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() { + int pageSize = 10; + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build(); + + ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class); + + S3Object s3Object1 = S3Object.builder().key("object1").build(); + S3Object s3Object2 = S3Object.builder().key("object2").build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build(); + + when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable); + when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator()); + + s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize); + Mockito.verify(s3Client).listObjectsV2Paginator(request); + } + + @Test + public void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() { + int pageSize = 1; + String prefix = "folder/"; + ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build(); + + ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class); + + S3Object s3Object1 = S3Object.builder().key("folder/object1").build(); + S3Object s3Object2 = S3Object.builder().key("folder/object2").build(); + ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build(); + + when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable); + when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator()); + + s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix); + Mockito.verify(s3Client).listObjectsV2Paginator(request); + } + + @Test + public void whenListBuckets_thenReturnBucketList() { + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build()); + + s3Service.listBuckets(); + Mockito.verify(s3Client).listBuckets(); + } +} diff --git a/aws-modules/pom.xml b/aws-modules/pom.xml index 02473815b5..77872e3f8f 100644 --- a/aws-modules/pom.xml +++ b/aws-modules/pom.xml @@ -19,6 +19,7 @@ aws-miscellaneous aws-reactive aws-s3 + aws-s3-v2