From 1cfe7506f7e9aff808af4ec0e57639130a6d0f35 Mon Sep 17 00:00:00 2001 From: Sean Mackrory Date: Fri, 27 Apr 2018 17:17:02 -0600 Subject: [PATCH] HADOOP-15356. Make HTTP timeout configurable in ADLS connector. Contributed by Atul Sikaria and Sean Mackrory. --- .../src/main/resources/core-default.xml | 9 +++ .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 + .../apache/hadoop/fs/adl/AdlFileSystem.java | 13 ++++ .../src/site/markdown/troubleshooting_adl.md | 9 +++ .../fs/adl/live/AdlStorageConfiguration.java | 12 ++- .../fs/adl/live/TestAdlSdkConfiguration.java | 77 +++++++++++++++++++ 6 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index dc13cb92982..98b0ddfbdcb 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2929,6 +2929,15 @@ + + adl.http.timeout + -1 + + Base timeout (in seconds) for HTTP requests from the ADL SDK. Values of + zero or less cause the SDK default to be used instead. + + + diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index e3a4ad6c55a..e124e1101d1 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -105,6 +105,7 @@ public final class AdlConfKeys { static final String ADL_ENABLEUPN_FOR_OWNERGROUP_KEY = "adl.feature.ownerandgroup.enableupn"; static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false; + public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout"; public static void addDeprecatedKeys() { Configuration.addDeprecations(new DeprecationDelta[]{ diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index aa6babea4d2..3e149a69062 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -195,6 +195,14 @@ public class AdlFileSystem extends FileSystem { VersionInfo.getVersion().trim() + "/" + clusterName + "/" + clusterType); + int timeout = conf.getInt(ADL_HTTP_TIMEOUT, -1); + if (timeout > 0) { + // only set timeout if specified in config. Otherwise use SDK default + options.setDefaultTimeout(timeout); + } else { + LOG.info("No valid ADL SDK timeout configured: using SDK default."); + } + adlClient.setOptions(options); boolean trackLatency = conf @@ -320,6 +328,11 @@ public class AdlFileSystem extends FileSystem { return azureTokenProvider; } + @VisibleForTesting + public ADLStoreClient getAdlClient() { + return adlClient; + } + /** * Constructing home directory locally is fine as long as Hadoop * local user name and ADL user name relationship story is not fully baked diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md index 80b2a6f5a36..b362a9cb488 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md @@ -144,3 +144,12 @@ org.apache.hadoop.security.AccessControlException: MKDIRS failed with error 0x83 ``` See "Adding the service principal to your ADL Account". + +## Timeouts + +The timeout used by the ADL SDK can be overridden with the hadoop property +`adl.http.timeout`. Some timeouts in compute frameworks may need to be +addressed by lowering the timeout used by the SDK. A lower timeout at the +storage layer may allow more retries to be attempted and actually increase +the likelihood of success before hitting the framework's timeout, as attempts +that may ultimately fail will fail faster. diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java index a9b4ab8a3b2..1da40adb7e8 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java @@ -79,20 +79,24 @@ public final class AdlStorageConfiguration { if (conf == null) { conf = getConfiguration(); } + return createStorageConnector(conf); + } + public synchronized static FileSystem createStorageConnector( + Configuration fsConfig) throws URISyntaxException, IOException { if (!isContractTestEnabled()) { return null; } - String fileSystem = conf.get(FILE_SYSTEM_KEY); + String fileSystem = fsConfig.get(FILE_SYSTEM_KEY); if (fileSystem == null || fileSystem.trim().length() == 0) { throw new IOException("Default file system not configured."); } - Class clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY, + Class clazz = fsConfig.getClass(FILE_SYSTEM_IMPL_KEY, FILE_SYSTEM_IMPL_DEFAULT); - FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf); - fs.initialize(new URI(fileSystem), conf); + FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, fsConfig); + fs.initialize(new URI(fileSystem), fsConfig); return fs; } } diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java new file mode 100644 index 00000000000..ca762d9315a --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.adl.AdlFileSystem; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; + +import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT; + +/** + * Tests interactions with SDK and ensures configuration is having the desired + * effect. + */ +public class TestAdlSdkConfiguration { + @Test + public void testDefaultTimeout() throws IOException { + AdlFileSystem fs = null; + Configuration conf = null; + int effectiveTimeout; + + conf = AdlStorageConfiguration.getConfiguration(); + conf.setInt(ADL_HTTP_TIMEOUT, -1); + try { + fs = (AdlFileSystem) + (AdlStorageConfiguration.createStorageConnector(conf)); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check test.fs.adl.name property.", e); + } + + // Skip this test if we can't get a real FS + Assume.assumeNotNull(fs); + + effectiveTimeout = fs.getAdlClient().getDefaultTimeout(); + Assert.assertFalse("A negative timeout is not supposed to take effect", + effectiveTimeout < 0); + + conf = AdlStorageConfiguration.getConfiguration(); + conf.setInt(ADL_HTTP_TIMEOUT, 17); + try { + fs = (AdlFileSystem) + (AdlStorageConfiguration.createStorageConnector(conf)); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check test.fs.adl.name property.", e); + } + + effectiveTimeout = fs.getAdlClient().getDefaultTimeout(); + Assert.assertEquals("Timeout is getting set", + effectiveTimeout, 17); + + // The default value may vary by SDK, so that value is not tested here. + } +}