From ca6c1068e2823deecb59c57441905780b7bdfe3d Mon Sep 17 00:00:00 2001 From: Ulisses Lima Date: Fri, 16 Dec 2022 04:23:40 -0300 Subject: [PATCH] BAEL-5812 Stream Large byte[] to File with WebClient (#13119) * first draft * editor review 1 --- .../streamlargefile/StreamLargeFileApp.java | 12 ++++ .../client/LargeFileDownloadWebClient.java | 41 +++++++++++++ .../client/LimitedFileDownloadWebClient.java | 55 +++++++++++++++++ .../server/LargeFileController.java | 31 ++++++++++ .../streamlargefile/generate-sample-files.sh | 12 ++++ .../src/main/resources/streamlargefile/run.sh | 21 +++++++ .../LargeFileControllerLiveTest.java | 59 +++++++++++++++++++ 7 files changed, 231 insertions(+) create mode 100644 spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/StreamLargeFileApp.java create mode 100644 spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LargeFileDownloadWebClient.java create mode 100644 spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LimitedFileDownloadWebClient.java create mode 100644 spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/server/LargeFileController.java create mode 100755 spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/generate-sample-files.sh create mode 100755 spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/run.sh create mode 100644 spring-reactive-modules/spring-5-reactive-client-2/src/test/java/com/baeldung/streamlargefile/LargeFileControllerLiveTest.java diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/StreamLargeFileApp.java b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/StreamLargeFileApp.java new file mode 100644 index 0000000000..d664ac58e0 --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/StreamLargeFileApp.java @@ -0,0 +1,12 @@ +package com.baeldung.streamlargefile; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class StreamLargeFileApp { + + public static void main(String... args) { + SpringApplication.run(StreamLargeFileApp.class, args); + } +} diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LargeFileDownloadWebClient.java b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LargeFileDownloadWebClient.java new file mode 100644 index 0000000000..3288aa7787 --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LargeFileDownloadWebClient.java @@ -0,0 +1,41 @@ +package com.baeldung.streamlargefile.client; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferUtils; +import org.springframework.web.reactive.function.client.WebClient; + +import reactor.core.publisher.Flux; + +public class LargeFileDownloadWebClient { + + private LargeFileDownloadWebClient() { + } + + public static long fetch(WebClient client, String destination) throws IOException { + Flux flux = client.get() + .retrieve() + .bodyToFlux(DataBuffer.class); + + Path path = Paths.get(destination); + + DataBufferUtils.write(flux, path) + .block(); + + return Files.size(path); + } + + public static void main(String... args) throws IOException { + String baseUrl = args[0]; + String destination = args[1]; + + WebClient client = WebClient.create(baseUrl); + + long bytes = fetch(client, destination); + System.out.printf("downloaded %d bytes to %s", bytes, destination); + } +} diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LimitedFileDownloadWebClient.java b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LimitedFileDownloadWebClient.java new file mode 100644 index 0000000000..9b02a0d47b --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/client/LimitedFileDownloadWebClient.java @@ -0,0 +1,55 @@ +package com.baeldung.streamlargefile.client; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.springframework.web.reactive.function.client.ExchangeStrategies; +import org.springframework.web.reactive.function.client.WebClient; + +import reactor.core.publisher.Mono; + +public class LimitedFileDownloadWebClient { + + private LimitedFileDownloadWebClient() { + } + + public static long fetch(WebClient client, String destination) throws IOException { + Mono mono = client.get() + .retrieve() + .bodyToMono(byte[].class) + .onErrorMap(RuntimeException::new); + + byte[] bytes = mono.block(); + + Path path = Paths.get(destination); + Files.write(path, bytes); + + return bytes.length; + } + + public static void main(String... args) throws IOException { + String baseUrl = args[0]; + String destination = args[1]; + + WebClient client = WebClient.builder() + .baseUrl(baseUrl) + .exchangeStrategies(useMaxMemory()) + .build(); + + long bytes = fetch(client, destination); + System.out.printf("downloaded %d bytes to %s", bytes, destination); + } + + public static ExchangeStrategies useMaxMemory() { + long totalMemory = Runtime.getRuntime() + .maxMemory(); + + return ExchangeStrategies.builder() + .codecs(configurer -> + configurer.defaultCodecs() + .maxInMemorySize((int) totalMemory)) + .build(); + } +} diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/server/LargeFileController.java b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/server/LargeFileController.java new file mode 100644 index 0000000000..7fa27cced6 --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/main/java/com/baeldung/streamlargefile/server/LargeFileController.java @@ -0,0 +1,31 @@ +package com.baeldung.streamlargefile.server; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/large-file") +public class LargeFileController { + + public static final Path downloadPath = Paths.get("/tmp/large.dat"); + + @GetMapping("size") + Long getSize() throws IOException { + return Files.size(downloadPath); + } + + @GetMapping + ResponseEntity get() { + return ResponseEntity.ok() + .body(new FileSystemResource(downloadPath)); + } +} diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/generate-sample-files.sh b/spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/generate-sample-files.sh new file mode 100755 index 0000000000..bca9e35796 --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/generate-sample-files.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +generate() { + file="$1" + size="$2" + + fallocate -l "$size" "$file" + ls -lah "$file" +} + +generate /tmp/small.dat 128K +generate /tmp/large.dat 128M diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/run.sh b/spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/run.sh new file mode 100755 index 0000000000..72727643af --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/main/resources/streamlargefile/run.sh @@ -0,0 +1,21 @@ +#!/bin/bash +MYSELF="$(readlink -f "$0")" +MYDIR="${MYSELF%/*}" + +client="${1:-Large}" +url="${2:-http://localhost:8081/large-file}" +download_destination="${3:-/tmp/download.dat}" +xmx="${4:-32m}" + +module_dir="$(readlink -f "$MYDIR/../../../..")" + +echo "module: $module_dir" +cd $module_dir || exit + +echo "packaging..." +mvn clean package dependency:copy-dependencies + +echo "GET $url with $client client..." +java -Xmx$xmx -cp target/dependency/*:target/* \ +"com.baeldung.streamlargefile.client.${client}FileDownloadWebClient" \ +"$url" "$download_destination" diff --git a/spring-reactive-modules/spring-5-reactive-client-2/src/test/java/com/baeldung/streamlargefile/LargeFileControllerLiveTest.java b/spring-reactive-modules/spring-5-reactive-client-2/src/test/java/com/baeldung/streamlargefile/LargeFileControllerLiveTest.java new file mode 100644 index 0000000000..65ba97e283 --- /dev/null +++ b/spring-reactive-modules/spring-5-reactive-client-2/src/test/java/com/baeldung/streamlargefile/LargeFileControllerLiveTest.java @@ -0,0 +1,59 @@ +package com.baeldung.streamlargefile; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ClassPathResource; +import org.springframework.web.reactive.function.client.WebClient; + +import com.baeldung.streamlargefile.client.LargeFileDownloadWebClient; +import com.baeldung.streamlargefile.client.LimitedFileDownloadWebClient; +import com.baeldung.streamlargefile.server.LargeFileController; + +class LargeFileControllerLiveTest { + + private static final String BASE_URL = "http://localhost:8081/large-file"; + private static final String DOWNLOAD_DESTINATION = LargeFileController.downloadPath.resolveSibling("download.dat") + .toString(); + private static final Path downloadFile = LargeFileController.downloadPath; + private static final Runtime runtime = Runtime.getRuntime(); + private static final Long xmx = runtime.maxMemory(); + + private WebClient client = WebClient.create(BASE_URL); + + @BeforeAll + static void init() throws IOException { + if (!Files.exists(downloadFile)) { + ClassPathResource res = new ClassPathResource("streamlargefile/generate-sample-files.sh"); + + runtime.exec(res.getFile() + .getAbsolutePath()); + } + } + + @Test + void givenMemorySafeClient_whenFileLargerThanXmx_thenFileDownloaded() throws IOException { + if (xmx < Files.size(downloadFile)) { + long size = LargeFileDownloadWebClient.fetch(client, DOWNLOAD_DESTINATION); + assertTrue(size > xmx); + } + } + + @Test + void givenLimitedClient_whenXmxLargerThanFile_thenFileDownloaded() throws IOException { + WebClient client = WebClient.builder() + .baseUrl(BASE_URL) + .exchangeStrategies(LimitedFileDownloadWebClient.useMaxMemory()) + .build(); + + if (xmx > Files.size(downloadFile)) { + long size = LimitedFileDownloadWebClient.fetch(client, DOWNLOAD_DESTINATION); + assertTrue(size < xmx); + } + } +}