iSharkFly-Docs
/
java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[JAVA-22456] Merge modules aws-s3 & aws-s3-v2

This commit is contained in:
panos-kakos 2023-07-05 08:50:30 +03:00
parent f7bd212ec5
commit 82584d9e02
12 changed files with 251 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
aws-modules/aws-s3-v2/README.md
View File

@ -1,2 +0,0 @@
## Relevant Articles
- [Listing All AWS S3 Objects in a Bucket Using Java](https://www.baeldung.com/java-aws-s3-list-bucket-objects)

50
aws-modules/aws-s3-v2/pom.xml
View File

@ -1,50 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.baeldung.s3</groupId>
<artifactId>aws-s3-v2</artifactId>
<version>1.0-SNAPSHOT</version>
<name>aws-s3-v2</name>
<packaging>jar</packaging>
<parent>
<groupId>com.baeldung</groupId>
<artifactId>aws-modules</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<aws.java.sdk.version>2.20.52</aws.java.sdk.version>
<maven.compiler.plugin.version>3.6.1</maven.compiler.plugin.version>
</properties>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>${aws.java.sdk.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

22
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/Main.java
View File

@ -1,22 +0,0 @@
package com.baeldung.s3.listobjects;
import software.amazon.awssdk.regions.Region;
public class Main {
private static String AWS_BUCKET = "baeldung-tutorial-s3";
public static Region AWS_REGION = Region.EU_CENTRAL_1;
public static void main(String[] args) {
S3Service s3Service = new S3Service(AWS_REGION);
s3Service.putObject(AWS_BUCKET, FileGenerator.generateFiles(1000, 1));
s3Service.listBuckets();
s3Service.listObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500);
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/");
s3Service.cleanup();
}
}

129
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/S3Service.java
View File

@ -1,129 +0,0 @@
package com.baeldung.s3.listobjects;
import java.util.List;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
class S3Service {
private S3Client s3Client;
public S3Service(Region awsRegion) {
init(awsRegion);
}
public S3Service(S3Client s3Client) {
this.s3Client = s3Client;
}
public void init(Region awsRegion) {
this.s3Client = S3Client.builder()
.region(awsRegion)
.credentialsProvider(ProfileCredentialsProvider.create("default"))
.build();
}
public void listObjectsInBucket(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
List<S3Object> contents = listObjectsV2Response.contents();
System.out.println("Number of objects in the bucket: " + contents.stream().count());
contents.stream().forEach(System.out::println);
}
public void listAllObjectsInBucket(String bucketName) {
String nextContinuationToken = null;
long totalObjects = 0;
do {
ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder()
.bucket(bucketName)
.continuationToken(nextContinuationToken);
ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
nextContinuationToken = response.nextContinuationToken();
totalObjects += response.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
} while (nextContinuationToken != null);
System.out.println("Number of objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.prefix(prefix)
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listBuckets() {
// List all buckets
ListBucketsResponse listBucketsResponse = s3Client.listBuckets();
// Display the bucket names
List<Bucket> buckets = listBucketsResponse.buckets();
System.out.println("Buckets:");
for (Bucket bucket : buckets) {
System.out.println(bucket.name());
}
}
public void putObject(String bucketName, List<File> files) {
try {
files.stream().forEach(file -> {
s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(),
RequestBody.fromByteBuffer(file.getContent()));
System.out.println("Uploaded file: " + file.getName());
});
} catch (S3Exception e) {
System.err.println("Upload failed");
e.printStackTrace();
}
}
public void cleanup() {
this.s3Client.close();
}
}

14
aws-modules/aws-s3-v2/src/main/resources/logback.xml
View File

@ -1,14 +0,0 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
<!-- Change the log level for software.amazon.awssdk -->
<logger name="software.amazon.awssdk" level="info" />
</configuration>

2
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/File.java → aws-modules/aws-s3/src/main/java/com/baeldung/s3/File.java
View File

@ -1,4 +1,4 @@
package com.baeldung.s3.listobjects;
package com.baeldung.s3;
import java.nio.ByteBuffer;

2
aws-modules/aws-s3-v2/src/main/java/com/baeldung/s3/listobjects/FileGenerator.java → aws-modules/aws-s3/src/main/java/com/baeldung/s3/FileGenerator.java
View File

@ -1,4 +1,4 @@
package com.baeldung.s3.listobjects;
package com.baeldung.s3;
import java.nio.ByteBuffer;
import java.util.ArrayList;

9
aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Application.java
View File

@ -23,8 +23,17 @@ public class S3Application {
return;
}
s3Service.putObjects(AWS_BUCKET, FileGenerator.generateFiles(1000, 1));
//list all the buckets
s3Service.listBuckets();
s3Service.listObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucket(AWS_BUCKET);
s3Service.listAllObjectsInBucketPaginated(AWS_BUCKET, 500);
s3Service.listAllObjectsInBucketPaginatedWithPrefix(AWS_BUCKET, 500, "backup/");
//deleting bucket
s3Service.deleteBucket("baeldung-bucket-test2");

82
aws-modules/aws-s3/src/main/java/com/baeldung/s3/S3Service.java
View File

@ -234,6 +234,88 @@ class S3Service {
}
}
public void listObjectsInBucket(String bucketName) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.build();
ListObjectsV2Response listObjectsV2Response = s3Client.listObjectsV2(listObjectsV2Request);
List<S3Object> contents = listObjectsV2Response.contents();
System.out.println("Number of objects in the bucket: " + contents.stream().count());
contents.stream().forEach(System.out::println);
}
public void listAllObjectsInBucket(String bucketName) {
String nextContinuationToken = null;
long totalObjects = 0;
do {
ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder()
.bucket(bucketName)
.continuationToken(nextContinuationToken);
ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
nextContinuationToken = response.nextContinuationToken();
totalObjects += response.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
} while (nextContinuationToken != null);
System.out.println("Number of objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginated(String bucketName, int pageSize) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void listAllObjectsInBucketPaginatedWithPrefix(String bucketName, int pageSize, String prefix) {
ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder()
.bucket(bucketName)
.maxKeys(pageSize) // Set the maxKeys parameter to control the page size
.prefix(prefix)
.build();
ListObjectsV2Iterable listObjectsV2Iterable = s3Client.listObjectsV2Paginator(listObjectsV2Request);
long totalObjects = 0;
for (ListObjectsV2Response page : listObjectsV2Iterable) {
long retrievedPageSize = page.contents().stream()
.peek(System.out::println)
.reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum);
totalObjects += retrievedPageSize;
System.out.println("Page size: " + retrievedPageSize);
}
System.out.println("Total objects in the bucket: " + totalObjects);
}
public void putObjects(String bucketName, List<File> files) {
try {
files.stream().forEach(file -> {
s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key(file.getName()).build(),
RequestBody.fromByteBuffer(file.getContent()));
System.out.println("Uploaded file: " + file.getName());
});
} catch (S3Exception e) {
System.err.println("Upload failed");
e.printStackTrace();
}
}
public void cleanup() {
this.s3Client.close();
}

3
aws-modules/aws-s3/src/main/resources/logback.xml
View File

@ -10,4 +10,7 @@
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
<!-- Change the log level for software.amazon.awssdk -->
<logger name="software.amazon.awssdk" level="info" />
</configuration>

18
aws-modules/aws-s3-v2/src/test/java/com/baeldung/s3/listobjects/S3ServiceLiveTest.java → aws-modules/aws-s3/src/test/java/com/baeldung/s3/S3ServiceLiveTest.java
View File

@ -1,12 +1,14 @@
package com.baeldung.s3.listobjects;
package com.baeldung.s3;
import static org.mockito.Mockito.when;
import com.baeldung.s3.listobjects.S3Service;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
@ -17,8 +19,6 @@ import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import java.util.Arrays;
import java.util.Collections;
import static org.mockito.Mockito.when;
class S3ServiceLiveTest {
@Mock
@ -41,7 +41,7 @@ class S3ServiceLiveTest {
}
@Test
public void givenBucketName_whenListObjectsInBucket_thenReturnList() {
void givenBucketName_whenListObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
@ -52,7 +52,7 @@ class S3ServiceLiveTest {
}
@Test
public void givenBucketName_whenListAllObjectsInBucket_thenReturnList() {
void givenBucketName_whenListAllObjectsInBucket_thenReturnList() {
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).build();
ListObjectsV2Response response = ListObjectsV2Response.builder().contents(Collections.emptyList()).build();
@ -63,7 +63,7 @@ class S3ServiceLiveTest {
}
@Test
public void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() {
void givenBucketNameAndPageSize_whenListAllObjectsInBucketPaginated_thenReturnPaginatedList() {
int pageSize = 10;
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).build();
@ -81,7 +81,7 @@ class S3ServiceLiveTest {
}
@Test
public void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() {
void givenBucketNamePageSizeAndPrefix_whenListAllObjectsInBucketPaginatedWithPrefix_thenReturnPaginatedList() {
int pageSize = 1;
String prefix = "folder/";
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(AWS_BUCKET).maxKeys(pageSize).prefix(prefix).build();
@ -100,7 +100,7 @@ class S3ServiceLiveTest {
}
@Test
public void whenListBuckets_thenReturnBucketList() {
void whenListBuckets_thenReturnBucketList() {
when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(Collections.emptyList()).build());
s3Service.listBuckets();

1
aws-modules/pom.xml
View File

@ -19,7 +19,6 @@
<module>aws-miscellaneous</module>
<module>aws-reactive</module>
<module>aws-s3</module>
<module>aws-s3-v2</module>
</modules>
<properties>