honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize GoogleCloudStorageHttpHandler (#49677) (#49707) 

Removing a lot of needless buffering and array creation
to reduce the significant memory usage of tests using this.
The incoming stream from the `exchange` is already buffered
so there is no point in adding a ton of additional buffers
everywhere.
This commit is contained in:
Armin Braun 2019-11-29 11:17:47 +01:00 committed by GitHub
parent a59b7e07f1
commit 90e9d61f2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 49 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java vendored
View File

@ -161,7 +161,7 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
// Batch https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
final String uri = "/storage/v1/b/" + bucket + "/o/";
final StringBuilder batch = new StringBuilder();
for (String line : Streams.readAllLines(new BufferedInputStream(wrappedRequest))) {
for (String line : Streams.readAllLines(wrappedRequest)) {
if (line.length() == 0 || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) {
batch.append(line).append('\n');
} else if (line.startsWith("DELETE")) {
@ -226,8 +226,13 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
final int start = getContentRangeStart(range);
final int end = getContentRangeEnd(range);
final ByteArrayOutputStream out = new ByteArrayOutputStream();
long bytesRead = Streams.copy(wrappedRequest, out);
final ByteArrayOutputStream out = new ByteArrayOutputStream() {
@Override
public byte[] toByteArray() {
return buf;
}
};
long bytesRead = Streams.copy(wrappedRequest, out, new byte[128]);
int length = Math.max(end + 1, limit != null ? limit : 0);
if ((int) bytesRead > length) {
throw new AssertionError("Requesting more bytes than available for blob");
@ -267,56 +272,62 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort();
}
private static final Pattern NAME_PATTERN = Pattern.compile("\"name\":\"([^\"]*)\"");
public static Optional<Tuple<String, BytesArray>> parseMultipartRequestBody(final InputStream requestBody) throws IOException {
Tuple<String, BytesArray> content = null;
try (BufferedInputStream in = new BufferedInputStream(new GZIPInputStream(requestBody))) {
String name = null;
int read;
ByteArrayOutputStream out = new ByteArrayOutputStream() {
@Override
public byte[] toByteArray() {
return buf;
}
};
while ((read = in.read()) != -1) {
out.reset();
boolean markAndContinue = false;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
do { // search next consecutive {carriage return, new line} chars and stop
if ((char) read == '\r') {
int next = in.read();
if (next != -1) {
if (next == '\n') {
break;
}
out.write(read);
out.write(next);
continue;
do { // search next consecutive {carriage return, new line} chars and stop
if ((char) read == '\r') {
int next = in.read();
if (next != -1) {
if (next == '\n') {
break;
}
}
out.write(read);
} while ((read = in.read()) != -1);
final String line = new String(out.toByteArray(), UTF_8);
if (line.length() == 0 || line.equals("\r\n") || line.startsWith("--")
|| line.toLowerCase(Locale.ROOT).startsWith("content")) {
markAndContinue = true;
} else if (line.startsWith("{\"bucket\":")) {
markAndContinue = true;
Matcher matcher = Pattern.compile("\"name\":\"([^\"]*)\"").matcher(line);
if (matcher.find()) {
name = matcher.group(1);
out.write(read);
out.write(next);
continue;
}
}
if (markAndContinue) {
in.mark(Integer.MAX_VALUE);
continue;
out.write(read);
} while ((read = in.read()) != -1);
final String bucketPrefix = "{\"bucket\":";
final String start = new String(out.toByteArray(), 0, Math.min(out.size(), bucketPrefix.length()), UTF_8);
if (start.length() == 0 || start.equals("\r\n") || start.startsWith("--")
|| start.toLowerCase(Locale.ROOT).startsWith("content")) {
markAndContinue = true;
} else if (start.startsWith(bucketPrefix)) {
markAndContinue = true;
final String line = new String(out.toByteArray(), bucketPrefix.length(), out.size() - bucketPrefix.length(), UTF_8);
Matcher matcher = NAME_PATTERN.matcher(line);
if (matcher.find()) {
name = matcher.group(1);
}
}
if (markAndContinue) {
in.mark(Integer.MAX_VALUE);
continue;
}
if (name != null) {
in.reset();
try (ByteArrayOutputStream binary = new ByteArrayOutputStream()) {
while ((read = in.read()) != -1) {
binary.write(read);
}
binary.flush();
byte[] tmp = binary.toByteArray();
// removes the trailing end "\r\n--__END_OF_PART__--\r\n" which is 23 bytes long
content = Tuple.tuple(name, new BytesArray(Arrays.copyOf(tmp, tmp.length - 23)));
out.reset();
while ((read = in.read()) != -1) {
out.write(read);
}
// removes the trailing end "\r\n--__END_OF_PART__--\r\n" which is 23 bytes long
content = Tuple.tuple(name, new BytesArray(Arrays.copyOf(out.toByteArray(), out.size() - 23)));
break;
}
}
}