From 70411cb1f15b5fe03a5d8c09447f781e34febbfe Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 3 Feb 2021 14:29:56 +0000 Subject: [PATCH] HADOOP-17337. S3A NetworkBinding has a runtime dependency on shaded httpclient. (#2599) Contributed by Steve Loughran. Change-Id: I0471322fc88d8bc3896ac439aefb31e6a856936c --- .../impl/ConfigureShadedAWSSocketFactory.java | 47 +++++++++++++++++ .../hadoop/fs/s3a/impl/NetworkBinding.java | 52 ++++++++++--------- 2 files changed, 74 insertions(+), 25 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java new file mode 100644 index 00000000000..8205668e8f3 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import javax.net.ssl.HostnameVerifier; +import java.io.IOException; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory; + +import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; + +/** + * This interacts with the Shaded httpclient library used in the full + * AWS SDK. If the S3A client is used with the unshaded SDK, this + * class will not link. + */ +public class ConfigureShadedAWSSocketFactory implements + NetworkBinding.ConfigureAWSSocketFactory { + + @Override + public void configureSocketFactory(final ClientConfiguration awsConf, + final DelegatingSSLSocketFactory.SSLChannelMode channelMode) + throws IOException { + DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode); + awsConf.getApacheHttpClientConfig().setSslSocketFactory( + new SSLConnectionSocketFactory( + DelegatingSSLSocketFactory.getDefaultFactory(), + (HostnameVerifier) null)); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java index 3eeb01f971b..409ac7bebc4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java @@ -19,14 +19,10 @@ package org.apache.hadoop.fs.s3a.impl; import java.io.IOException; -import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSocketFactory; - import com.amazonaws.ClientConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,20 +39,21 @@ import static org.apache.hadoop.fs.s3a.Constants.SSL_CHANNEL_MODE; /** * Configures network settings when communicating with AWS services. */ -public class NetworkBinding { +public final class NetworkBinding { private static final Logger LOG = LoggerFactory.getLogger(NetworkBinding.class); - private static final String AWS_SOCKET_FACTORY_CLASSNAME = "com.amazonaws" + - ".thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory"; + private static final String BINDING_CLASSNAME = "org.apache.hadoop.fs.s3a.impl.ConfigureShadedAWSSocketFactory"; + + private NetworkBinding() { + } /** * Configures the {@code SSLConnectionSocketFactory} used by the AWS SDK. * A custom Socket Factory can be set using the method * {@code setSslSocketFactory()}. - * If {@code SSLConnectionSocketFactory} cannot be found on the classpath, the value - * of {@link org.apache.hadoop.fs.s3a.Constants#SSL_CHANNEL_MODE} is ignored. - * + * Uses reflection to do this via {@link ConfigureShadedAWSSocketFactory} + * so as to avoid * @param conf the {@link Configuration} used to get the client specified * value of {@code SSL_CHANNEL_MODE} * @param awsConf the {@code ClientConfiguration} to set the @@ -84,28 +81,33 @@ public class NetworkBinding { DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode); try { - // Look for AWS_SOCKET_FACTORY_CLASSNAME on the classpath and instantiate - // an instance using the DelegatingSSLSocketFactory as the - // SSLSocketFactory. - Class sslConnectionSocketFactory = Class.forName( - AWS_SOCKET_FACTORY_CLASSNAME); - Constructor factoryConstructor = - sslConnectionSocketFactory.getDeclaredConstructor( - SSLSocketFactory.class, HostnameVerifier.class); - awsConf.getApacheHttpClientConfig().setSslSocketFactory( - (com.amazonaws.thirdparty.apache.http.conn.ssl. - SSLConnectionSocketFactory) factoryConstructor - .newInstance(DelegatingSSLSocketFactory - .getDefaultFactory(), - (HostnameVerifier) null)); + // use reflection to load in our own binding class. + // this is *probably* overkill, but it is how we can be fully confident + // that no attempt will be made to load/link to the AWS Shaded SDK except + // within this try/catch block + Class clazz = + (Class) Class.forName(BINDING_CLASSNAME); + clazz.getConstructor() + .newInstance() + .configureSocketFactory(awsConf, channelMode); } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException | LinkageError e) { LOG.debug("Unable to create class {}, value of {} will be ignored", - AWS_SOCKET_FACTORY_CLASSNAME, SSL_CHANNEL_MODE, e); + BINDING_CLASSNAME, SSL_CHANNEL_MODE, e); } } + /** + * Interface used to bind to the socket factory, allows the code which + * works with the shaded AWS libraries to exist in their own class. + */ + interface ConfigureAWSSocketFactory { + void configureSocketFactory(ClientConfiguration awsConf, + DelegatingSSLSocketFactory.SSLChannelMode channelMode) + throws IOException; + } + /** * Given an S3 bucket region as returned by a bucket location query, * fix it into a form which can be used by other AWS commands.