BAEL-5812 Stream Large byte[] to File with WebClient (#13119)
* first draft * editor review 1
This commit is contained in:
parent
3a3784e689
commit
ca6c1068e2
|
@ -0,0 +1,12 @@
|
||||||
|
package com.baeldung.streamlargefile;
|
||||||
|
|
||||||
|
import org.springframework.boot.SpringApplication;
|
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
|
||||||
|
@SpringBootApplication
|
||||||
|
public class StreamLargeFileApp {
|
||||||
|
|
||||||
|
public static void main(String... args) {
|
||||||
|
SpringApplication.run(StreamLargeFileApp.class, args);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,41 @@
|
||||||
|
package com.baeldung.streamlargefile.client;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
|
||||||
|
import org.springframework.core.io.buffer.DataBuffer;
|
||||||
|
import org.springframework.core.io.buffer.DataBufferUtils;
|
||||||
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
|
||||||
|
import reactor.core.publisher.Flux;
|
||||||
|
|
||||||
|
public class LargeFileDownloadWebClient {
|
||||||
|
|
||||||
|
private LargeFileDownloadWebClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static long fetch(WebClient client, String destination) throws IOException {
|
||||||
|
Flux<DataBuffer> flux = client.get()
|
||||||
|
.retrieve()
|
||||||
|
.bodyToFlux(DataBuffer.class);
|
||||||
|
|
||||||
|
Path path = Paths.get(destination);
|
||||||
|
|
||||||
|
DataBufferUtils.write(flux, path)
|
||||||
|
.block();
|
||||||
|
|
||||||
|
return Files.size(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String... args) throws IOException {
|
||||||
|
String baseUrl = args[0];
|
||||||
|
String destination = args[1];
|
||||||
|
|
||||||
|
WebClient client = WebClient.create(baseUrl);
|
||||||
|
|
||||||
|
long bytes = fetch(client, destination);
|
||||||
|
System.out.printf("downloaded %d bytes to %s", bytes, destination);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,55 @@
|
||||||
|
package com.baeldung.streamlargefile.client;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
|
||||||
|
import org.springframework.web.reactive.function.client.ExchangeStrategies;
|
||||||
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
|
||||||
|
import reactor.core.publisher.Mono;
|
||||||
|
|
||||||
|
public class LimitedFileDownloadWebClient {
|
||||||
|
|
||||||
|
private LimitedFileDownloadWebClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static long fetch(WebClient client, String destination) throws IOException {
|
||||||
|
Mono<byte[]> mono = client.get()
|
||||||
|
.retrieve()
|
||||||
|
.bodyToMono(byte[].class)
|
||||||
|
.onErrorMap(RuntimeException::new);
|
||||||
|
|
||||||
|
byte[] bytes = mono.block();
|
||||||
|
|
||||||
|
Path path = Paths.get(destination);
|
||||||
|
Files.write(path, bytes);
|
||||||
|
|
||||||
|
return bytes.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String... args) throws IOException {
|
||||||
|
String baseUrl = args[0];
|
||||||
|
String destination = args[1];
|
||||||
|
|
||||||
|
WebClient client = WebClient.builder()
|
||||||
|
.baseUrl(baseUrl)
|
||||||
|
.exchangeStrategies(useMaxMemory())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
long bytes = fetch(client, destination);
|
||||||
|
System.out.printf("downloaded %d bytes to %s", bytes, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ExchangeStrategies useMaxMemory() {
|
||||||
|
long totalMemory = Runtime.getRuntime()
|
||||||
|
.maxMemory();
|
||||||
|
|
||||||
|
return ExchangeStrategies.builder()
|
||||||
|
.codecs(configurer ->
|
||||||
|
configurer.defaultCodecs()
|
||||||
|
.maxInMemorySize((int) totalMemory))
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,31 @@
|
||||||
|
package com.baeldung.streamlargefile.server;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
|
||||||
|
import org.springframework.core.io.FileSystemResource;
|
||||||
|
import org.springframework.core.io.Resource;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/large-file")
|
||||||
|
public class LargeFileController {
|
||||||
|
|
||||||
|
public static final Path downloadPath = Paths.get("/tmp/large.dat");
|
||||||
|
|
||||||
|
@GetMapping("size")
|
||||||
|
Long getSize() throws IOException {
|
||||||
|
return Files.size(downloadPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping
|
||||||
|
ResponseEntity<Resource> get() {
|
||||||
|
return ResponseEntity.ok()
|
||||||
|
.body(new FileSystemResource(downloadPath));
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,12 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
generate() {
|
||||||
|
file="$1"
|
||||||
|
size="$2"
|
||||||
|
|
||||||
|
fallocate -l "$size" "$file"
|
||||||
|
ls -lah "$file"
|
||||||
|
}
|
||||||
|
|
||||||
|
generate /tmp/small.dat 128K
|
||||||
|
generate /tmp/large.dat 128M
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/bin/bash
|
||||||
|
MYSELF="$(readlink -f "$0")"
|
||||||
|
MYDIR="${MYSELF%/*}"
|
||||||
|
|
||||||
|
client="${1:-Large}"
|
||||||
|
url="${2:-http://localhost:8081/large-file}"
|
||||||
|
download_destination="${3:-/tmp/download.dat}"
|
||||||
|
xmx="${4:-32m}"
|
||||||
|
|
||||||
|
module_dir="$(readlink -f "$MYDIR/../../../..")"
|
||||||
|
|
||||||
|
echo "module: $module_dir"
|
||||||
|
cd $module_dir || exit
|
||||||
|
|
||||||
|
echo "packaging..."
|
||||||
|
mvn clean package dependency:copy-dependencies
|
||||||
|
|
||||||
|
echo "GET $url with $client client..."
|
||||||
|
java -Xmx$xmx -cp target/dependency/*:target/* \
|
||||||
|
"com.baeldung.streamlargefile.client.${client}FileDownloadWebClient" \
|
||||||
|
"$url" "$download_destination"
|
|
@ -0,0 +1,59 @@
|
||||||
|
package com.baeldung.streamlargefile;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.core.io.ClassPathResource;
|
||||||
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
|
||||||
|
import com.baeldung.streamlargefile.client.LargeFileDownloadWebClient;
|
||||||
|
import com.baeldung.streamlargefile.client.LimitedFileDownloadWebClient;
|
||||||
|
import com.baeldung.streamlargefile.server.LargeFileController;
|
||||||
|
|
||||||
|
class LargeFileControllerLiveTest {
|
||||||
|
|
||||||
|
private static final String BASE_URL = "http://localhost:8081/large-file";
|
||||||
|
private static final String DOWNLOAD_DESTINATION = LargeFileController.downloadPath.resolveSibling("download.dat")
|
||||||
|
.toString();
|
||||||
|
private static final Path downloadFile = LargeFileController.downloadPath;
|
||||||
|
private static final Runtime runtime = Runtime.getRuntime();
|
||||||
|
private static final Long xmx = runtime.maxMemory();
|
||||||
|
|
||||||
|
private WebClient client = WebClient.create(BASE_URL);
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void init() throws IOException {
|
||||||
|
if (!Files.exists(downloadFile)) {
|
||||||
|
ClassPathResource res = new ClassPathResource("streamlargefile/generate-sample-files.sh");
|
||||||
|
|
||||||
|
runtime.exec(res.getFile()
|
||||||
|
.getAbsolutePath());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void givenMemorySafeClient_whenFileLargerThanXmx_thenFileDownloaded() throws IOException {
|
||||||
|
if (xmx < Files.size(downloadFile)) {
|
||||||
|
long size = LargeFileDownloadWebClient.fetch(client, DOWNLOAD_DESTINATION);
|
||||||
|
assertTrue(size > xmx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void givenLimitedClient_whenXmxLargerThanFile_thenFileDownloaded() throws IOException {
|
||||||
|
WebClient client = WebClient.builder()
|
||||||
|
.baseUrl(BASE_URL)
|
||||||
|
.exchangeStrategies(LimitedFileDownloadWebClient.useMaxMemory())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
if (xmx > Files.size(downloadFile)) {
|
||||||
|
long size = LimitedFileDownloadWebClient.fetch(client, DOWNLOAD_DESTINATION);
|
||||||
|
assertTrue(size < xmx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue