From 6accb7809f6a2a556cea05f25c21691f80e63161 Mon Sep 17 00:00:00 2001 From: Mehakmeet Singh Date: Thu, 27 Oct 2022 20:17:57 +0530 Subject: [PATCH] HADOOP-18499. S3A to support HTTPS web proxies (#5083) The option "fs.s3a.proxy.ssl.enabled" controls whether the s3a connects to a proxy over HTTP (default) or HTTPS. Set to "true" to use HTTPS. Contributed by Mehakmeet Singh --- .../org/apache/hadoop/fs/s3a/Constants.java | 2 + .../org/apache/hadoop/fs/s3a/S3AUtils.java | 8 +- .../apache/hadoop/fs/s3a/TestS3AProxy.java | 101 ++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 6d20e3b085a..93cdb40ca49 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -212,6 +212,8 @@ public final class Constants { public static final String PROXY_PASSWORD = "fs.s3a.proxy.password"; public static final String PROXY_DOMAIN = "fs.s3a.proxy.domain"; public static final String PROXY_WORKSTATION = "fs.s3a.proxy.workstation"; + /** Is the proxy secured(proxyProtocol = HTTPS)? */ + public static final String PROXY_SECURED = "fs.s3a.proxy.ssl.enabled"; /** * Number of times the AWS client library should retry errors before diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index bca2c618334..1401c181e7e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -1338,13 +1338,17 @@ public final class S3AUtils { LOG.error(msg); throw new IllegalArgumentException(msg); } + boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false); awsConf.setProxyUsername(proxyUsername); awsConf.setProxyPassword(proxyPassword); awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); + awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP); if (LOG.isDebugEnabled()) { - LOG.debug("Using proxy server {}:{} as user {} with password {} on " + - "domain {} as workstation {}", awsConf.getProxyHost(), + LOG.debug("Using proxy server {}://{}:{} as user {} with password {} " + + "on domain {} as workstation {}", + awsConf.getProxyProtocol(), + awsConf.getProxyHost(), awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()), awsConf.getProxyPassword(), awsConf.getProxyDomain(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java new file mode 100644 index 00000000000..e05ee25adfa --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED; +import static org.apache.hadoop.fs.s3a.S3AUtils.initProxySupport; + +/** + * Tests to verify {@link S3AUtils} translates the proxy configurations + * are set correctly to Client configurations which are later used to construct + * the proxy in AWS SDK. + */ +public class TestS3AProxy extends AbstractHadoopTestBase { + + /** + * Verify Http proxy protocol. + */ + @Test + public void testProxyHttp() throws IOException { + Configuration proxyConfigForHttp = createProxyConfig(false); + verifyProxy(proxyConfigForHttp, false); + } + + /** + * Verify Https proxy protocol. + */ + @Test + public void testProxyHttps() throws IOException { + Configuration proxyConfigForHttps = createProxyConfig(true); + verifyProxy(proxyConfigForHttps, true); + } + + /** + * Verify default proxy protocol. + */ + @Test + public void testProxyDefault() throws IOException { + Configuration proxyConfigDefault = new Configuration(); + proxyConfigDefault.set(PROXY_HOST, "testProxyDefault"); + verifyProxy(proxyConfigDefault, false); + } + + /** + * Assert that the configuration set for a proxy gets translated to Client + * configuration with the correct protocol to be used by AWS SDK. + * @param proxyConfig Configuration used to set the proxy configs. + * @param isExpectedSecured What is the expected protocol for the proxy to + * be? true for https, and false for http. + * @throws IOException + */ + private void verifyProxy(Configuration proxyConfig, + boolean isExpectedSecured) + throws IOException { + ClientConfiguration awsConf = new ClientConfiguration(); + initProxySupport(proxyConfig, "test-bucket", awsConf); + Assertions.assertThat(awsConf.getProxyProtocol()) + .describedAs("Proxy protocol not as expected") + .isEqualTo(isExpectedSecured ? Protocol.HTTPS : Protocol.HTTP); + } + + /** + * Create a configuration file with proxy configs. + * @param isSecured Should the configured proxy be secured or not? + * @return configuration. + */ + private Configuration createProxyConfig(boolean isSecured) { + Configuration conf = new Configuration(); + conf.set(PROXY_HOST, "testProxy"); + conf.set(PROXY_PORT, "1234"); + conf.setBoolean(PROXY_SECURED, isSecured); + return conf; + } +}