iSharkFly-Docs
/
java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[JAVA-22456] Merge modules aws-s3 & aws-s3-v2

This commit is contained in:
panos-kakos 2023-07-05 08:50:30 +03:00
parent f7bd212ec5
commit 82584d9e02
12 changed files with 251 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
aws-modules/aws-s3-v2/README.md
View File

@ -1,2 +0,0 @@
## Relevant Articles
- [Listing All AWS S3 Objects in a Bucket Using Java](https://www.baeldung.com/java-aws-s3-list-bucket-objects)

50
aws-modules/aws-s3-v2/pom.xml
View File

@ -1,50 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.baeldung.s3</groupId>
<artifactId>aws-s3-v2</artifactId>
<version>1.0-SNAPSHOT</version>
<name>aws-s3-v2</name>
<packaging>jar</packaging>
<parent>
<groupId>com.baeldung</groupId>
<artifactId>aws-modules</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<aws.java.sdk.version>2.20.52</aws.java.sdk.version>
<maven.compiler.plugin.version>3.6.1</maven.compiler.plugin.version>
</properties>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>${aws.java.sdk.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

22
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java
View File

@ -1,22 +0,0 @@
package com.baeldung.s3.listobjects;
import software.amazon.awssdk.regions.Region;
public class Main {
private static String AWS_BUCKET = "baeldung-tutorial-s3";
public static Region AWS_REGION = Region.EU_CENTRAL_1;
public static void main(String[] args) {
S3Service s3Service = new S3Service(AWS_REGION);
s3Service.putObject(AWS_BUCKET, FileGenerator.generateFiles(1000, 1));
s3Service.listBuckets();
s3Service.listObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500);
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/");
s3Service.cleanup();
}
}

129
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java
View File

@ -1,129 +0,0 @@
package com.baeldung.s3.listobjects;
import java.util.List;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
class S3Service {
private S3Client s3Client;
public S3Service(Region awsRegion) {
init(awsRegion);
}
public S3Service(S3Client s3Client) {
this.s3Client = s3Client;
}
public void init(Region awsRegion) {
this.s3Client = S3Client.builder()
.region(awsRegion)
.credentialsProvider(ProfileCredentialsProvider.create("default"))
.build();
}
public void listObjectsInBucket(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
List<S3Object> contents = listObjectsV2Response.contents();
System.out.println("Number of objects in the bucket: " + contents.stream().count());
contents.stream().forEach(System.out::println);
}
public void listAllObjectsInBucket(String bucketName) {
String nextContinuationToken = null;
long totalObjects = 0;
do {
ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder()
.bucket(bucketName)
.continuationToken(nextContinuationToken);
ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
nextContinuationToken = response.nextContinuationToken();
totalObjects += response.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
} while (nextContinuationToken != null);
System.out.println("Number of objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.prefix(prefix)
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listBuckets() {
// List all buckets
ListBucketsResponse listBucketsResponse = s3Client.listBuckets();
// Display the bucket names
List<Bucket> buckets = listBucketsResponse.buckets();
System.out.println("Buckets:");
for (Bucket bucket : buckets) {
System.out.println(bucket.name());
}
}
public void putObject(String bucketName, List<File> files) {
try {
files.stream().forEach(file -> {
s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(),
RequestBody.fromByteBuffer(file.getContent()));
System.out.println("Uploaded file: " + file.getName());
});
} catch (S3Exception e) {
System.err.println("Upload failed");
e.printStackTrace();
}
}
public void cleanup() {
this.s3Client.close();
}
}

14
aws-modules/aws-s3-v2/src/main/resources/logback.xml
View File

@ -1,14 +0,0 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
<!-- Change the log level for software.amazon.awssdk -->
<logger name="software.amazon.awssdk" level="info" />
</configuration>

42
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java → aws-modules/aws-s3/src/main/java/com/baeldung/s3/File.java
View File

@ -1,21 +1,21 @@
package com.baeldung.s3.listobjects;
import java.nio.ByteBuffer;
public class File {
private String name;
private ByteBuffer content;
public File(String name, ByteBuffer content) {
this.name = name;
this.content = content;
}
public String getName() {
return name;
}
public ByteBuffer getContent() {
return content;
}
}
package com.baeldung.s3;
import java.nio.ByteBuffer;
public class File {
private String name;
private ByteBuffer content;
public File(String name, ByteBuffer content) {
this.name = name;
this.content = content;
}
public String getName() {
return name;
}
public ByteBuffer getContent() {
return content;
}
}

54
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java → aws-modules/aws-s3/src/main/java/com/baeldung/s3/FileGenerator.java
View File

@ -1,27 +1,27 @@
package com.baeldung.s3.listobjects;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class FileGenerator {
public static List<File> generateFiles(int fileCount, int fileSize) {
List<File> files = new ArrayList<>();
for (int i = 0; i < fileCount; i++) {
String fileName = "file_" + i + ".txt";
ByteBuffer fileContent = generateRandomBytes(fileSize);
files.add(new File(fileName, fileContent));
}
return files;
}
private static ByteBuffer generateRandomBytes(int size) {
byte[] array = new byte[size];
new Random().nextBytes(array);
return ByteBuffer.wrap(array);
}
}
package com.baeldung.s3;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class FileGenerator {
public static List<File> generateFiles(int fileCount, int fileSize) {
List<File> files = new ArrayList<>();
for (int i = 0; i < fileCount; i++) {
String fileName = "file_" + i + ".txt";
ByteBuffer fileContent = generateRandomBytes(fileSize);
files.add(new File(fileName, fileContent));
}
return files;
}
private static ByteBuffer generateRandomBytes(int size) {
byte[] array = new byte[size];
new Random().nextBytes(array);
return ByteBuffer.wrap(array);
}
}

9
aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java
View File

@ -23,8 +23,17 @@ public class S3Application {
return;
}
s3Service.putObjects(AWS_BUCKET, FileGenerator.generateFiles(1000, 1));
//list all the buckets
s3Service.listBuckets();
s3Service.listObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500);
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/");
//deleting bucket
s3Service.deleteBucket("baeldung-bucket-test2");

82
aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java
View File

@ -234,6 +234,88 @@ class S3Service {
}
}
public void listObjectsInBucket(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
List<S3Object> contents = listObjectsV2Response.contents();
System.out.println("Number of objects in the bucket: " + contents.stream().count());
contents.stream().forEach(System.out::println);
}
public void listAllObjectsInBucket(String bucketName) {
String nextContinuationToken = null;
long totalObjects = 0;
do {
ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder()
.bucket(bucketName)
.continuationToken(nextContinuationToken);
ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
nextContinuationToken = response.nextContinuationToken();
totalObjects += response.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
} while (nextContinuationToken != null);
System.out.println("Number of objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.prefix(prefix)
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void putObjects(String bucketName, List<File> files) {
try {
files.stream().forEach(file -> {
s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(),
RequestBody.fromByteBuffer(file.getContent()));
System.out.println("Uploaded file: " + file.getName());
});
} catch (S3Exception e) {
System.err.println("Upload failed");
e.printStackTrace();
}
}
public void cleanup() {
this.s3Client.close();
}

3
aws-modules/aws-s3/src/main/resources/logback.xml
View File

@ -10,4 +10,7 @@
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
<!-- Change the log level for software.amazon.awssdk -->
<logger name="software.amazon.awssdk" level="info" />
</configuration>

218
aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java → aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceLiveTest.java
View File

@ -1,109 +1,109 @@
package com.baeldung.s3.listobjects;
import com.baeldung.s3.listobjects.S3Service;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import java.util.Arrays;
import java.util.Collections;
import static org.mockito.Mockito.when;
class S3ServiceLiveTest {
@Mock
private S3Client s3Client;
private S3Service s3Service;
private String AWS_BUCKET = "baeldung-tutorial-s3";
@BeforeEach
public void setup() {
MockitoAnnotations.openMocks(this);
s3Service = new S3Service(s3Client);
}
@AfterEach
public void cleanup() {
s3Service.cleanup();
Mockito.verify(s3Client).close();
}
@Test
public void givenBucketName_whenListObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
public void givenBucketName_whenListAllObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
public void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() {
int pageSize = 10;
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("object1").build();
S3Object s3Object2 = S3Object.builder().key("object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
public void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() {
int pageSize = 1;
String prefix = "folder/";
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("folder/object1").build();
S3Object s3Object2 = S3Object.builder().key("folder/object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
public void whenListBuckets_thenReturnBucketList() {
when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build());
s3Service.listBuckets();
Mockito.verify(s3Client).listBuckets();
}
}
package com.baeldung.s3;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import java.util.Arrays;
import java.util.Collections;
class S3ServiceLiveTest {
@Mock
private S3Client s3Client;
private S3Service s3Service;
private String AWS_BUCKET = "baeldung-tutorial-s3";
@BeforeEach
public void setup() {
MockitoAnnotations.openMocks(this);
s3Service = new S3Service(s3Client);
}
@AfterEach
public void cleanup() {
s3Service.cleanup();
Mockito.verify(s3Client).close();
}
@Test
void givenBucketName_whenListObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
void givenBucketName_whenListAllObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
when(s3Client.listObjectsV2(request)).thenReturn(response);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
Mockito.verify(s3Client).listObjectsV2(request);
}
@Test
void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() {
int pageSize = 10;
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("object1").build();
S3Object s3Object2 = S3Object.builder().key("object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, pageSize);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() {
int pageSize = 1;
String prefix = "folder/";
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build();
ListObjectsV2Iterable mockIterable = Mockito.mock(ListObjectsV2Iterable.class);
S3Object s3Object1 = S3Object.builder().key("folder/object1").build();
S3Object s3Object2 = S3Object.builder().key("folder/object2").build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(s3Object1, s3Object2).build();
when(s3Client.listObjectsV2Paginator(request)).thenReturn(mockIterable);
when(mockIterable.iterator()).thenReturn(Arrays.asList(response).iterator());
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, pageSize, prefix);
Mockito.verify(s3Client).listObjectsV2Paginator(request);
}
@Test
void whenListBuckets_thenReturnBucketList() {
when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build());
s3Service.listBuckets();
Mockito.verify(s3Client).listBuckets();
}
}

1
aws-modules/pom.xml
View File

@ -19,7 +19,6 @@
<module>aws-miscellaneous</module>
<module>aws-reactive</module>
<module>aws-s3</module>
<module>aws-s3-v2</module>
</modules>
<properties>