honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade GCS Repository Dependencies (#43142) (#43418) 

* Upgrade to latest GCS SDK and transitive dependencies (I chose the later version here on conflict)
* Remove now unnecessary hack for custom endpoints (the linked bugs were both resolved in the SDK)
This commit is contained in:
Armin Braun 2019-06-20 16:35:54 +02:00 committed by GitHub
parent b4c4018d00
commit 21e74dd7d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 62 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
plugins/repository-gcs/build.gradle
View File

@ -25,39 +25,35 @@ esplugin {
}
dependencies {
compile 'com.google.cloud:google-cloud-storage:1.59.0'
compile 'com.google.cloud:google-cloud-core:1.59.0'
compile 'com.google.guava:guava:20.0'
compile "joda-time:joda-time:${versions.joda}"
compile 'com.google.http-client:google-http-client:1.24.1'
compile 'com.google.cloud:google-cloud-storage:1.77.0'
compile 'com.google.cloud:google-cloud-core:1.77.0'
compile 'com.google.guava:guava:26.0-jre'
compile 'com.google.http-client:google-http-client:1.30.1'
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
compile "commons-logging:commons-logging:${versions.commonslogging}"
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
compile "commons-codec:commons-codec:${versions.commonscodec}"
compile 'com.google.api:api-common:1.7.0'
compile 'com.google.api:gax:1.30.0'
compile 'com.google.api:api-common:1.8.1'
compile 'com.google.api:gax:1.45.0'
compile 'org.threeten:threetenbp:1.3.3'
compile 'com.google.protobuf:protobuf-java-util:3.6.0'
compile 'com.google.protobuf:protobuf-java:3.6.0'
compile 'com.google.protobuf:protobuf-java-util:3.7.1'
compile 'com.google.protobuf:protobuf-java:3.7.1'
compile 'com.google.code.gson:gson:2.7'
compile 'com.google.api.grpc:proto-google-common-protos:1.12.0'
compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
compile 'com.google.cloud:google-cloud-core-http:1.59.0'
compile 'com.google.auth:google-auth-library-credentials:0.10.0'
compile 'com.google.auth:google-auth-library-oauth2-http:0.10.0'
compile 'com.google.oauth-client:google-oauth-client:1.24.1'
compile 'com.google.api-client:google-api-client:1.24.1'
compile 'com.google.http-client:google-http-client-appengine:1.24.1'
compile 'com.google.http-client:google-http-client-jackson:1.24.1'
compile 'org.codehaus.jackson:jackson-core-asl:1.9.11'
compile 'com.google.http-client:google-http-client-jackson2:1.24.1'
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
compile 'com.google.api:gax-httpjson:0.47.0'
compile 'io.opencensus:opencensus-api:0.15.0'
compile 'com.google.cloud:google-cloud-core-http:1.77.0'
compile 'com.google.auth:google-auth-library-credentials:0.16.1'
compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
compile 'com.google.oauth-client:google-oauth-client:1.28.0'
compile 'com.google.api-client:google-api-client:1.28.0'
compile 'com.google.http-client:google-http-client-appengine:1.29.2'
compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
compile 'com.google.api:gax-httpjson:0.62.0'
compile 'io.grpc:grpc-context:1.12.0'
compile 'io.opencensus:opencensus-contrib-http-util:0.15.0'
compile 'com.google.apis:google-api-services-storage:v1-rev135-1.24.1'
compile 'io.opencensus:opencensus-api:0.18.0'
compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
}
dependencyLicenses {
@ -65,7 +61,6 @@ dependencyLicenses {
mapping from: /google-auth-.*/, to: 'google-auth'
mapping from: /google-http-.*/, to: 'google-http'
mapping from: /opencensus.*/, to: 'opencensus'
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /http.*/, to: 'httpclient'
mapping from: /protobuf.*/, to: 'protobuf'
mapping from: /proto-google.*/, to: 'proto-google'
@ -81,6 +76,10 @@ thirdPartyAudit {
'com.google.common.cache.Striped64',
'com.google.common.cache.Striped64$1',
'com.google.common.cache.Striped64$Cell',
'com.google.common.hash.Striped64',
'com.google.common.hash.Striped64$1',
'com.google.common.hash.Striped64$Cell',
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',

1
plugins/repository-gcs/licenses/api-common-1.7.0.jar.sha1
View File

@ -1 +0,0 @@
ea59fb8b2450999345035dec8a6f472543391766

1
plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e89befb19b08ad84b262b2f226ab79aefcaa9d7f

1
plugins/repository-gcs/licenses/gax-1.30.0.jar.sha1
View File

@ -1 +0,0 @@
58fa2feb11b092be0a6ebe705a28736f12374230

1
plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2ade3e3502f9d14e3731347a82ea02372094211f

1
plugins/repository-gcs/licenses/gax-httpjson-0.47.0.jar.sha1
View File

@ -1 +0,0 @@
d096f3142eb3adbf877588d1044895d148d9efcb

1
plugins/repository-gcs/licenses/gax-httpjson-0.62.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
05a1a4736acd1c4f30304be953532be6aecdc2c9

1
plugins/repository-gcs/licenses/google-api-client-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
37de23fb9b8b077de4ecec3192d98e752b0e5d72

1
plugins/repository-gcs/licenses/google-api-client-1.28.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
8fe155d766ed22480939e3a9db428151e0264d9e

1
plugins/repository-gcs/licenses/google-api-services-storage-v1-rev135-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
28d3d391dfc7e7e7951760708ad2f48cecacf38f

1
plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20190426-1.28.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
34dd008901f382507a572f5242d0e5c5ea4ad713

1
plugins/repository-gcs/licenses/google-auth-library-credentials-0.10.0.jar.sha1
View File

@ -1 +0,0 @@
f981288bd84fe6d140ed70d1d8dbe994a64fa3cc

1
plugins/repository-gcs/licenses/google-auth-library-credentials-0.16.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9a15387cc0438ac3f3e625b6050cf39f4e981e13

1
plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.10.0.jar.sha1
View File

@ -1 +0,0 @@
c079a62086121973a23d90f54e2b8c13050fa39d

1
plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.16.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
3407d434678faef3439a7012efa336e751ddc623

1
plugins/repository-gcs/licenses/google-cloud-core-1.59.0.jar.sha1
View File

@ -1 +0,0 @@
f2d0c00917660b244da514f82cba96f7697f2c82

1
plugins/repository-gcs/licenses/google-cloud-core-1.77.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7cd83a789fde368a999c1793c6297e7b4e56b2ac

1
plugins/repository-gcs/licenses/google-cloud-core-http-1.59.0.jar.sha1
View File

@ -1 +0,0 @@
e2a094ec3e8acb15b99f2d4bd42ac9bbc7d9f33e

1
plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e16acbc935a7762ba9b220860ae45c2c67d17d8c

1
plugins/repository-gcs/licenses/google-cloud-storage-1.59.0.jar.sha1
View File

@ -1 +0,0 @@
23dc0edf739ff1fb5a91fbddd7bd1f2cbfe0f827

1
plugins/repository-gcs/licenses/google-cloud-storage-1.77.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e368e1a8bbc0d0a4354f4e5eec076f38f6966050

1
plugins/repository-gcs/licenses/google-http-client-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
396eac8d3fb1332675f82b208f48a469d64f3b4a

1
plugins/repository-gcs/licenses/google-http-client-1.30.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
573aacbda8feb0d43f7056291fbce5496f42a6aa

1
plugins/repository-gcs/licenses/google-http-client-appengine-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
8535031ae10bf6a196e68f25e10c0d6382699cb6

1
plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d93f4d1d8c2496d75221e53173e4c503b7096a4d

1
plugins/repository-gcs/licenses/google-http-client-jackson-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
02c88e77c14effdda76f02a0eac968de74e0bd4e

1
plugins/repository-gcs/licenses/google-http-client-jackson2-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
2ad1dffd8a450055e68d8004fe003033b751d761

1
plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d67891f5a438e1f339387a09628e0ab0af8b612a

1
plugins/repository-gcs/licenses/google-oauth-client-1.24.1.jar.sha1
View File

@ -1 +0,0 @@
7b0e0218b96808868c23a7d0b40566a713931d9f

1
plugins/repository-gcs/licenses/google-oauth-client-1.28.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9a9e5d0c33b663d6475c96ce79b2949545a113af

1
plugins/repository-gcs/licenses/guava-20.0.jar.sha1
View File

@ -1 +0,0 @@
89507701249388e1ed5ddcf8c41f4ce1be7831ef

1
plugins/repository-gcs/licenses/guava-26.0-jre.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6a806eff209f36f635f943e16d97491f00f6bfab

8
plugins/repository-gcs/licenses/jackson-LICENSE
View File

@ -1,8 +0,0 @@
This copy of Jackson JSON processor streaming parser/generator is licensed under the
Apache (Software) License, version 2.0 ("the License").
See the License for details about distribution rights, and the
specific rights regarding derivate works.
You may obtain a copy of the License at:
http://www.apache.org/licenses/LICENSE-2.0

20
plugins/repository-gcs/licenses/jackson-NOTICE
View File

@ -1,20 +0,0 @@
# Jackson JSON processor
Jackson is a high-performance, Free/Open Source JSON processing library.
It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
been in development since 2007.
It is currently developed by a community of developers, as well as supported
commercially by FasterXML.com.
## Licensing
Jackson core and extension components may licensed under different licenses.
To find the details that apply to this artifact see the accompanying LICENSE file.
For more information, including possible other licensing options, contact
FasterXML.com (http://fasterxml.com).
## Credits
A list of contributors may be found from CREDITS file, which is included
in some artifacts (usually source distributions); but is always available
from the source code management (SCM) system project uses.

1
plugins/repository-gcs/licenses/jackson-core-asl-1.9.11.jar.sha1
View File

@ -1 +0,0 @@
e32303ef8bd18a5c9272780d49b81c95e05ddf43

1
plugins/repository-gcs/licenses/opencensus-api-0.15.0.jar.sha1
View File

@ -1 +0,0 @@
9a098392b287d7924660837f4eba0ce252013683

1
plugins/repository-gcs/licenses/opencensus-api-0.18.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b89a8f8dfd1e1e0d68d83c82a855624814b19a6e

1
plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.15.0.jar.sha1
View File

@ -1 +0,0 @@
d88690591669d9b5ba6d91d9eac7736e58ccf3da

1
plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
76a37e4a931d5801a9e25b0c0353e5f37c4d1e8e

1
plugins/repository-gcs/licenses/proto-google-common-protos-1.12.0.jar.sha1
View File

@ -1 +0,0 @@
1140cc74df039deb044ed0e320035e674dc13062

1
plugins/repository-gcs/licenses/proto-google-common-protos-1.16.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2c5f022ea3b8e8df6a619c4cd8faf9af86022daa

1
plugins/repository-gcs/licenses/protobuf-java-3.6.0.jar.sha1
View File

@ -1 +0,0 @@
5333f7e422744d76840c08a106e28e519fbe3acd

1
plugins/repository-gcs/licenses/protobuf-java-3.7.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
0bce1b6dc9e4531169542ab37a1c8641bcaa8afb

1
plugins/repository-gcs/licenses/protobuf-java-util-3.6.0.jar.sha1
View File

@ -1 +0,0 @@
3680d0042d4fe0b95ada844ff24da0698a7f0773

1
plugins/repository-gcs/licenses/protobuf-java-util-3.7.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
45dc95896cfad26397675fdabef7b032d6db4bb6

62
plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.repositories.gcs;
import com.google.api.client.googleapis.GoogleUtils;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.DefaultConnectionFactory;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.http.HttpTransportOptions;
@ -37,10 +36,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.LazyInitializable;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@ -104,10 +100,16 @@ public class GoogleCloudStorageService {
* @return a new client storage instance that can be used to manage objects
* (blobs)
*/
private Storage createClient(final String clientName, final GoogleCloudStorageClientSettings clientSettings) throws IOException {
private static Storage createClient(String clientName, GoogleCloudStorageClientSettings clientSettings) throws IOException {
logger.debug(() -> new ParameterizedMessage("creating GCS client with client_name [{}], endpoint [{}]", clientName,
clientSettings.getHost()));
final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> createHttpTransport(clientSettings.getHost()));
final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> {
final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
// requires java.lang.RuntimePermission "setFactory"
// Pin the TLS trust certificates.
builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
return builder.build();
});
final HttpTransportOptions httpTransportOptions = HttpTransportOptions.newBuilder()
.setConnectTimeout(toTimeout(clientSettings.getConnectTimeout()))
.setReadTimeout(toTimeout(clientSettings.getReadTimeout()))
@ -145,54 +147,6 @@ public class GoogleCloudStorageService {
return storageOptionsBuilder.build().getService();
}
/**
* Pins the TLS trust certificates and, more importantly, overrides connection
* URLs in the case of a custom endpoint setting because some connections don't
* fully honor this setting (bugs in the SDK). The default connection factory
* opens a new connection for each request. This is required for the storage
* instance to be thread-safe.
**/
private static HttpTransport createHttpTransport(final String endpoint) throws Exception {
final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
// requires java.lang.RuntimePermission "setFactory"
builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
if (Strings.hasLength(endpoint)) {
final URL endpointUrl = URI.create(endpoint).toURL();
// it is crucial to open a connection for each URL (see {@code
// DefaultConnectionFactory#openConnection}) instead of reusing connections,
// because the storage instance has to be thread-safe as it is cached.
builder.setConnectionFactory(new DefaultConnectionFactory() {
@Override
public HttpURLConnection openConnection(final URL originalUrl) throws IOException {
// test if the URL is built correctly, ie following the `host` setting
if (originalUrl.getHost().equals(endpointUrl.getHost()) && originalUrl.getPort() == endpointUrl.getPort()
&& originalUrl.getProtocol().equals(endpointUrl.getProtocol())) {
return super.openConnection(originalUrl);
}
// override connection URLs because some don't follow the config. See
// https://github.com/GoogleCloudPlatform/google-cloud-java/issues/3254 and
// https://github.com/GoogleCloudPlatform/google-cloud-java/issues/3255
URI originalUri;
try {
originalUri = originalUrl.toURI();
} catch (final URISyntaxException e) {
throw new RuntimeException(e);
}
String overridePath = "/";
if (originalUri.getRawPath() != null) {
overridePath = originalUri.getRawPath();
}
if (originalUri.getRawQuery() != null) {
overridePath += "?" + originalUri.getRawQuery();
}
return super.openConnection(
new URL(endpointUrl.getProtocol(), endpointUrl.getHost(), endpointUrl.getPort(), overridePath));
}
});
}
return builder.build();
}
/**
* Converts timeout values from the settings to a timeout value for the Google
* Cloud SDK

10
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java
View File

@ -281,6 +281,11 @@ class MockStorage implements Storage {
return null;
}
@Override
public WriteChannel writer(URL signedURL) {
return null;
}
// Everything below this line is not implemented.
@Override
@ -288,6 +293,11 @@ class MockStorage implements Storage {
return null;
}
@Override
public Blob create(BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
return null;
}
@Override
public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
return null;