Upgrading the GCS SDK to the most recent version. Adjusting (i.e. improving) the REST mock accordingly. This should significantly boost performance by pulling in https://github.com/googleapis/java-core/issues/86 in some cases.
This commit is contained in:
Armin Braun
GitHub
parent
3cd4b97618
commit
204c366a4e
|
|
@ -25,10 +25,10 @@ esplugin {
|
|||
}
|
||||
|
||||
dependencies {
|
||||
compile 'com.google.cloud:google-cloud-storage:1.77.0'
|
||||
compile 'com.google.cloud:google-cloud-core:1.77.0'
|
||||
compile 'com.google.cloud:google-cloud-storage:1.104.0'
|
||||
compile 'com.google.cloud:google-cloud-core:1.92.5'
|
||||
compile 'com.google.guava:guava:26.0-jre'
|
||||
compile 'com.google.http-client:google-http-client:1.30.1'
|
||||
compile 'com.google.http-client:google-http-client:1.34.2'
|
||||
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
||||
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
||||
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
||||
|
|
@ -37,23 +37,23 @@ dependencies {
|
|||
compile 'com.google.api:api-common:1.8.1'
|
||||
compile 'com.google.api:gax:1.45.0'
|
||||
compile 'org.threeten:threetenbp:1.3.3'
|
||||
compile 'com.google.protobuf:protobuf-java-util:3.7.1'
|
||||
compile 'com.google.protobuf:protobuf-java:3.7.1'
|
||||
compile 'com.google.protobuf:protobuf-java-util:3.11.3'
|
||||
compile 'com.google.protobuf:protobuf-java:3.11.3'
|
||||
compile 'com.google.code.gson:gson:2.7'
|
||||
compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
|
||||
compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
|
||||
compile 'com.google.cloud:google-cloud-core-http:1.77.0'
|
||||
compile 'com.google.auth:google-auth-library-credentials:0.16.1'
|
||||
compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
|
||||
compile 'com.google.auth:google-auth-library-credentials:0.20.0'
|
||||
compile 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
|
||||
compile 'com.google.oauth-client:google-oauth-client:1.28.0'
|
||||
compile 'com.google.api-client:google-api-client:1.28.0'
|
||||
compile 'com.google.api-client:google-api-client:1.30.8'
|
||||
compile 'com.google.http-client:google-http-client-appengine:1.29.2'
|
||||
compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
|
||||
compile 'com.google.api:gax-httpjson:0.62.0'
|
||||
compile 'io.grpc:grpc-context:1.12.0'
|
||||
compile 'io.opencensus:opencensus-api:0.18.0'
|
||||
compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
|
||||
compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
|
||||
compile 'com.google.apis:google-api-services-storage:v1-rev20191011-1.30.3'
|
||||
|
||||
testCompile project(':test:fixtures:gcs-fixture')
|
||||
}
|
||||
|
|
@ -75,6 +75,9 @@ thirdPartyAudit {
|
|||
'com.google.protobuf.UnsafeUtil$1',
|
||||
'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
|
||||
'com.google.protobuf.UnsafeUtil$MemoryAccessor',
|
||||
'com.google.protobuf.MessageSchema',
|
||||
'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
|
||||
'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
|
||||
'com.google.common.cache.Striped64',
|
||||
'com.google.common.cache.Striped64$1',
|
||||
'com.google.common.cache.Striped64$Cell',
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
8fe155d766ed22480939e3a9db428151e0264d9e
|
||||
|
|
@ -0,0 +1 @@
|
|||
b8664bfb9287f269228a4bbc36988174d4f311eb
|
||||
|
|
@ -1 +0,0 @@
|
|||
34dd008901f382507a572f5242d0e5c5ea4ad713
|
||||
|
|
@ -0,0 +1 @@
|
|||
53fc173da380f9b65caad69912223db128dbf7ac
|
||||
|
|
@ -1 +0,0 @@
|
|||
9a15387cc0438ac3f3e625b6050cf39f4e981e13
|
||||
|
|
@ -0,0 +1 @@
|
|||
87a91a373e64ba5c3cdf8cc5cf54b189dd1492f8
|
||||
|
|
@ -1 +0,0 @@
|
|||
3407d434678faef3439a7012efa336e751ddc623
|
||||
|
|
@ -0,0 +1 @@
|
|||
f33d4d6c91a68826816606a2208990eea93fcb2a
|
||||
|
|
@ -1 +0,0 @@
|
|||
7cd83a789fde368a999c1793c6297e7b4e56b2ac
|
||||
|
|
@ -0,0 +1 @@
|
|||
97ca5c35f34ee67dde8f2d4e9b940f0a96393c66
|
||||
|
|
@ -0,0 +1 @@
|
|||
8a083661f3732b21c0addd4fe56fd3f05a9c5a84
|
||||
|
|
@ -1 +0,0 @@
|
|||
e368e1a8bbc0d0a4354f4e5eec076f38f6966050
|
||||
|
|
@ -1 +0,0 @@
|
|||
573aacbda8feb0d43f7056291fbce5496f42a6aa
|
||||
|
|
@ -0,0 +1 @@
|
|||
58b4c500b5ce58271f1a6c5f6767aa8721158ea4
|
||||
|
|
@ -0,0 +1 @@
|
|||
df12be70b968e32442821a2cfdc3cede5a42dec5
|
||||
|
|
@ -1 +0,0 @@
|
|||
0bce1b6dc9e4531169542ab37a1c8641bcaa8afb
|
||||
|
|
@ -0,0 +1 @@
|
|||
fd4ba2dfeb1b010eb20ca27e65fbfb74fbbdcdb9
|
||||
|
|
@ -1 +0,0 @@
|
|||
45dc95896cfad26397675fdabef7b032d6db4bb6
|
||||
|
|
@ -132,6 +132,8 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends ESTestCase {
|
|||
final HttpTransportOptions httpTransportOptions) {
|
||||
StorageOptions options = super.createStorageOptions(clientSettings, httpTransportOptions);
|
||||
return options.toBuilder()
|
||||
.setHost(options.getHost())
|
||||
.setCredentials(options.getCredentials())
|
||||
.setRetrySettings(RetrySettings.newBuilder()
|
||||
.setTotalTimeout(options.getRetrySettings().getTotalTimeout())
|
||||
.setInitialRetryDelay(Duration.ofMillis(10L))
|
||||
|
|
@ -331,7 +333,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends ESTestCase {
|
|||
|
||||
public void testWriteLargeBlob() throws IOException {
|
||||
// See {@link BaseWriteChannel#DEFAULT_CHUNK_SIZE}
|
||||
final int defaultChunkSize = 8 * 256 * 1024;
|
||||
final int defaultChunkSize = 60 * 256 * 1024;
|
||||
final int nbChunks = randomIntBetween(3, 5);
|
||||
final int lastChunkSize = randomIntBetween(1, defaultChunkSize - 1);
|
||||
final int totalChunks = nbChunks + 1;
|
||||
|
|
|
|||
|
|
@ -264,6 +264,8 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRe
|
|||
final HttpTransportOptions httpTransportOptions) {
|
||||
StorageOptions options = super.createStorageOptions(clientSettings, httpTransportOptions);
|
||||
return options.toBuilder()
|
||||
.setHost(options.getHost())
|
||||
.setCredentials(options.getCredentials())
|
||||
.setRetrySettings(RetrySettings.newBuilder()
|
||||
.setTotalTimeout(options.getRetrySettings().getTotalTimeout())
|
||||
.setInitialRetryDelay(Duration.ofMillis(10L))
|
||||
|
|
@ -331,7 +333,7 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRe
|
|||
}
|
||||
|
||||
final String range = exchange.getRequestHeaders().getFirst("Content-Range");
|
||||
return exchange.getRemoteAddress().toString()
|
||||
return exchange.getRemoteAddress().getHostString()
|
||||
+ " " + exchange.getRequestMethod()
|
||||
+ " " + exchange.getRequestURI()
|
||||
+ (range != null ? " " + range : "");
|
||||
|
|
|
|||
|
|
@ -298,9 +298,10 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
|
|||
skippedEmptyLine = markAndContinue && endPos == startPos;
|
||||
startPos = endPos;
|
||||
} else {
|
||||
// removes the trailing end "\r\n--__END_OF_PART__--\r\n" which is 23 bytes long
|
||||
int len = fullRequestBody.length() - startPos - 23;
|
||||
content = Tuple.tuple(name, fullRequestBody.slice(startPos, len));
|
||||
while (isEndOfPart(fullRequestBody, endPos) == false) {
|
||||
endPos = fullRequestBody.indexOf((byte) '\r', endPos + 1);
|
||||
}
|
||||
content = Tuple.tuple(name, fullRequestBody.slice(startPos, endPos - startPos));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -312,6 +313,18 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
|
|||
return Optional.ofNullable(content);
|
||||
}
|
||||
|
||||
private static final byte[] END_OF_PARTS_MARKER = "\r\n--__END_OF_PART__".getBytes(UTF_8);
|
||||
|
||||
private static boolean isEndOfPart(BytesReference fullRequestBody, int endPos) {
|
||||
for (int i = 0; i < END_OF_PARTS_MARKER.length; i++) {
|
||||
final byte b = END_OF_PARTS_MARKER[i];
|
||||
if (fullRequestBody.get(endPos + i) != b) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static final Pattern PATTERN_CONTENT_RANGE = Pattern.compile("bytes ([^/]*)/([0-9\\*]*)");
|
||||
private static final Pattern PATTERN_CONTENT_RANGE_BYTES = Pattern.compile("([0-9]*)-([0-9]*)");
|
||||
|
||||
|
|
|
|||
|
|
@ -30,10 +30,10 @@ dependencies {
|
|||
compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
|
||||
|
||||
// GCS dependencies
|
||||
compile 'com.google.cloud:google-cloud-storage:1.77.0'
|
||||
compile 'com.google.cloud:google-cloud-core:1.77.0'
|
||||
compile 'com.google.cloud:google-cloud-storage:1.104.0'
|
||||
compile 'com.google.cloud:google-cloud-core:1.92.5'
|
||||
compile 'com.google.guava:guava:26.0-jre'
|
||||
compile 'com.google.http-client:google-http-client:1.30.1'
|
||||
compile 'com.google.http-client:google-http-client:1.34.2'
|
||||
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
||||
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
||||
compile "commons-logging:commons-logging:${versions.commonslogging}"
|
||||
|
|
@ -42,23 +42,23 @@ dependencies {
|
|||
compile 'com.google.api:api-common:1.8.1'
|
||||
compile 'com.google.api:gax:1.45.0'
|
||||
compile 'org.threeten:threetenbp:1.3.3'
|
||||
compile 'com.google.protobuf:protobuf-java-util:3.7.1'
|
||||
compile 'com.google.protobuf:protobuf-java:3.7.1'
|
||||
compile 'com.google.protobuf:protobuf-java-util:3.11.3'
|
||||
compile 'com.google.protobuf:protobuf-java:3.11.3'
|
||||
compile 'com.google.code.gson:gson:2.7'
|
||||
compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
|
||||
compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
|
||||
compile 'com.google.cloud:google-cloud-core-http:1.77.0'
|
||||
compile 'com.google.auth:google-auth-library-credentials:0.16.1'
|
||||
compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
|
||||
compile 'com.google.auth:google-auth-library-credentials:0.20.0'
|
||||
compile 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
|
||||
compile 'com.google.oauth-client:google-oauth-client:1.28.0'
|
||||
compile 'com.google.api-client:google-api-client:1.28.0'
|
||||
compile 'com.google.api-client:google-api-client:1.30.8'
|
||||
compile 'com.google.http-client:google-http-client-appengine:1.29.2'
|
||||
compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
|
||||
compile 'com.google.api:gax-httpjson:0.62.0'
|
||||
compile 'io.grpc:grpc-context:1.12.0'
|
||||
compile 'io.opencensus:opencensus-api:0.18.0'
|
||||
compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
|
||||
compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
|
||||
compile 'com.google.apis:google-api-services-storage:v1-rev20191011-1.30.3'
|
||||
|
||||
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
||||
// and whitelist this hack in JarHell
|
||||
|
|
@ -297,6 +297,9 @@ thirdPartyAudit.ignoreViolations(
|
|||
'com.google.protobuf.UnsafeUtil$1',
|
||||
'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
|
||||
'com.google.protobuf.UnsafeUtil$MemoryAccessor',
|
||||
'com.google.protobuf.MessageSchema',
|
||||
'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
|
||||
'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
|
||||
'com.google.common.cache.Striped64',
|
||||
'com.google.common.cache.Striped64$1',
|
||||
'com.google.common.cache.Striped64$Cell',
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
8fe155d766ed22480939e3a9db428151e0264d9e
|
||||
|
|
@ -0,0 +1 @@
|
|||
b8664bfb9287f269228a4bbc36988174d4f311eb
|
||||
|
|
@ -1 +0,0 @@
|
|||
34dd008901f382507a572f5242d0e5c5ea4ad713
|
||||
|
|
@ -0,0 +1 @@
|
|||
53fc173da380f9b65caad69912223db128dbf7ac
|
||||
|
|
@ -1 +0,0 @@
|
|||
9a15387cc0438ac3f3e625b6050cf39f4e981e13
|
||||
|
|
@ -0,0 +1 @@
|
|||
87a91a373e64ba5c3cdf8cc5cf54b189dd1492f8
|
||||
|
|
@ -1 +0,0 @@
|
|||
3407d434678faef3439a7012efa336e751ddc623
|
||||
|
|
@ -0,0 +1 @@
|
|||
f33d4d6c91a68826816606a2208990eea93fcb2a
|
||||
|
|
@ -1 +0,0 @@
|
|||
7cd83a789fde368a999c1793c6297e7b4e56b2ac
|
||||
|
|
@ -0,0 +1 @@
|
|||
97ca5c35f34ee67dde8f2d4e9b940f0a96393c66
|
||||
|
|
@ -0,0 +1 @@
|
|||
8a083661f3732b21c0addd4fe56fd3f05a9c5a84
|
||||
|
|
@ -1 +0,0 @@
|
|||
e368e1a8bbc0d0a4354f4e5eec076f38f6966050
|
||||
|
|
@ -1 +0,0 @@
|
|||
573aacbda8feb0d43f7056291fbce5496f42a6aa
|
||||
|
|
@ -0,0 +1 @@
|
|||
58b4c500b5ce58271f1a6c5f6767aa8721158ea4
|
||||
|
|
@ -0,0 +1 @@
|
|||
df12be70b968e32442821a2cfdc3cede5a42dec5
|
||||
|
|
@ -1 +0,0 @@
|
|||
0bce1b6dc9e4531169542ab37a1c8641bcaa8afb
|
||||
|
|
@ -0,0 +1 @@
|
|||
fd4ba2dfeb1b010eb20ca27e65fbfb74fbbdcdb9
|
||||
|
|
@ -1 +0,0 @@
|
|||
45dc95896cfad26397675fdabef7b032d6db4bb6
|
||||
Loading…
Reference in New Issue