HADOOP-15356. Make HTTP timeout configurable in ADLS connector. Contributed by Atul Sikaria and Sean Mackrory.

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -2929,6 +2929,15 @@
     </description>
   </property>
 
+  <property>
+    <name>adl.http.timeout</name>
+    <value>-1</value>
+    <description>
+      Base timeout (in seconds) for HTTP requests from the ADL SDK. Values of
+      zero or less cause the SDK default to be used instead.
+    </description>
+  </property>
+
   <!-- Azure Data Lake File System Configurations Ends Here-->
 
   <property>

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java

@@ -105,6 +105,7 @@ public final class AdlConfKeys {
   static final String ADL_ENABLEUPN_FOR_OWNERGROUP_KEY =
       "adl.feature.ownerandgroup.enableupn";
   static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+  public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout";
 
   public static void addDeprecatedKeys() {
     Configuration.addDeprecations(new DeprecationDelta[]{

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java

@@ -195,6 +195,14 @@ public void initialize(URI storeUri, Configuration originalConf)
         VersionInfo.getVersion().trim() + "/" + clusterName + "/"
         + clusterType);
 
+    int timeout = conf.getInt(ADL_HTTP_TIMEOUT, -1);
+    if (timeout > 0) {
+      // only set timeout if specified in config. Otherwise use SDK default
+      options.setDefaultTimeout(timeout);
+    } else {
+      LOG.info("No valid ADL SDK timeout configured: using SDK default.");
+    }
+
     adlClient.setOptions(options);
 
     boolean trackLatency = conf
@@ -320,6 +328,11 @@ AzureADTokenProvider getAzureTokenProvider() {
     return azureTokenProvider;
   }
 
+  @VisibleForTesting
+  public ADLStoreClient getAdlClient() {
+    return adlClient;
+  }
+
   /**
    * Constructing home directory locally is fine as long as Hadoop
    * local user name and ADL user name relationship story is not fully baked

hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md

@@ -144,3 +144,12 @@ org.apache.hadoop.security.AccessControlException: MKDIRS failed with error 0x83
 ```
 
 See "Adding the service principal to your ADL Account".
+
+## Timeouts
+
+The timeout used by the ADL SDK can be overridden with the hadoop property
+`adl.http.timeout`.  Some timeouts in compute frameworks may need to be
+addressed by lowering the timeout used by the SDK.  A lower timeout at the
+storage layer may allow more retries to be attempted and actually increase
+the likelihood of success before hitting the framework's timeout, as attempts
+that may ultimately fail will fail faster.

hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java

@@ -79,20 +79,24 @@ public synchronized static FileSystem createStorageConnector()
     if (conf == null) {
       conf = getConfiguration();
     }
+    return createStorageConnector(conf);
+  }
 
+  public synchronized static FileSystem createStorageConnector(
+      Configuration fsConfig) throws URISyntaxException, IOException {
     if (!isContractTestEnabled()) {
       return null;
     }
 
-    String fileSystem = conf.get(FILE_SYSTEM_KEY);
+    String fileSystem = fsConfig.get(FILE_SYSTEM_KEY);
     if (fileSystem == null || fileSystem.trim().length() == 0) {
       throw new IOException("Default file system not configured.");
     }
 
-    Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
+    Class<?> clazz = fsConfig.getClass(FILE_SYSTEM_IMPL_KEY,
         FILE_SYSTEM_IMPL_DEFAULT);
-    FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
-    fs.initialize(new URI(fileSystem), conf);
+    FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, fsConfig);
+    fs.initialize(new URI(fileSystem), fsConfig);
     return fs;
   }
 }

hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT;
+
+/**
+ * Tests interactions with SDK and ensures configuration is having the desired
+ * effect.
+ */
+public class TestAdlSdkConfiguration {
+  @Test
+  public void testDefaultTimeout() throws IOException {
+    AdlFileSystem fs = null;
+    Configuration conf = null;
+    int effectiveTimeout;
+
+    conf = AdlStorageConfiguration.getConfiguration();
+    conf.setInt(ADL_HTTP_TIMEOUT, -1);
+    try {
+      fs = (AdlFileSystem)
+          (AdlStorageConfiguration.createStorageConnector(conf));
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("Can not initialize ADL FileSystem. "
+          + "Please check test.fs.adl.name property.", e);
+    }
+
+    // Skip this test if we can't get a real FS
+    Assume.assumeNotNull(fs);
+
+    effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
+    Assert.assertFalse("A negative timeout is not supposed to take effect",
+        effectiveTimeout < 0);
+
+    conf = AdlStorageConfiguration.getConfiguration();
+    conf.setInt(ADL_HTTP_TIMEOUT, 17);
+    try {
+      fs = (AdlFileSystem)
+          (AdlStorageConfiguration.createStorageConnector(conf));
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("Can not initialize ADL FileSystem. "
+          + "Please check test.fs.adl.name property.", e);
+    }
+
+    effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
+    Assert.assertEquals("Timeout is getting set",
+        effectiveTimeout, 17);
+
+    // The default value may vary by SDK, so that value is not tested here.
+  }
+}