HADOOP-16438. ADLS Gen1 OpenSSL config control.
Contributed by Sneha Vijayarajan. Change-Id: Ib79ea6b4a90ad068033e175f3f59c5185868872d
This commit is contained in:
parent
60af8793b4
commit
147f98629c
|
@ -3373,6 +3373,20 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>adl.ssl.channel.mode</name>
|
||||
<value></value>
|
||||
<description>
|
||||
Valid inputs are OpenSSL, Default_JSE and Default (case insensitive).
|
||||
If config is missing or is invalid, SSL Channel mode will be set to Default.
|
||||
|
||||
When OpenSSL, SSL socket connections are created in OpenSSL mode.
|
||||
When Default_JSE, SSL socket connections are created in the default JSE mode.
|
||||
When Default, SSL socket connections are attempted with OpenSSL
|
||||
and will fallback to Default_JSE mode if OpenSSL is not available at runtime.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<!-- Azure Data Lake File System Configurations Ends Here-->
|
||||
|
||||
<property>
|
||||
|
|
|
@ -33,7 +33,7 @@
|
|||
<minimalJsonVersion>0.9.1</minimalJsonVersion>
|
||||
<file.encoding>UTF-8</file.encoding>
|
||||
<downloadSources>true</downloadSources>
|
||||
<azure.data.lake.store.sdk.version>2.3.3</azure.data.lake.store.sdk.version>
|
||||
<azure.data.lake.store.sdk.version>2.3.6</azure.data.lake.store.sdk.version>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
|
|
|
@ -106,6 +106,7 @@ public final class AdlConfKeys {
|
|||
"adl.feature.ownerandgroup.enableupn";
|
||||
static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
|
||||
public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout";
|
||||
public static final String ADL_SSL_CHANNEL_MODE = "adl.ssl.channel.mode";
|
||||
|
||||
public static void addDeprecatedKeys() {
|
||||
Configuration.addDeprecations(new DeprecationDelta[]{
|
||||
|
|
|
@ -203,6 +203,10 @@ public class AdlFileSystem extends FileSystem {
|
|||
LOG.info("No valid ADL SDK timeout configured: using SDK default.");
|
||||
}
|
||||
|
||||
String sslChannelMode = conf.get(ADL_SSL_CHANNEL_MODE,
|
||||
"Default");
|
||||
options.setSSLChannelMode(sslChannelMode);
|
||||
|
||||
adlClient.setOptions(options);
|
||||
|
||||
boolean trackLatency = conf
|
||||
|
|
|
@ -153,3 +153,13 @@ addressed by lowering the timeout used by the SDK. A lower timeout at the
|
|||
storage layer may allow more retries to be attempted and actually increase
|
||||
the likelihood of success before hitting the framework's timeout, as attempts
|
||||
that may ultimately fail will fail faster.
|
||||
|
||||
## SSL Socket Channel Mode
|
||||
|
||||
ADL SDK will by default attempt to create secure socket connections over
|
||||
OpenSSL as they provide significant performance improvements over Https. If
|
||||
there are runtime issues, SDK will default connections over Default_JSE. This
|
||||
can be overridden with the hadoop property `adl.ssl.channel.mode`. Possible
|
||||
values for this config are OpenSSL, Default_JSE and Default (default).
|
||||
Setting the config to OpenSSL or Default_JSE will try the connection to
|
||||
only that mode.
|
||||
|
|
|
@ -19,6 +19,8 @@
|
|||
|
||||
package org.apache.hadoop.fs.adl.live;
|
||||
|
||||
import com.microsoft.azure.datalake.store.SSLSocketFactoryEx.SSLChannelMode;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.adl.AdlFileSystem;
|
||||
import org.junit.Assert;
|
||||
|
@ -29,6 +31,7 @@ import java.io.IOException;
|
|||
import java.net.URISyntaxException;
|
||||
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_SSL_CHANNEL_MODE;
|
||||
|
||||
/**
|
||||
* Tests interactions with SDK and ensures configuration is having the desired
|
||||
|
@ -53,7 +56,6 @@ public class TestAdlSdkConfiguration {
|
|||
|
||||
// Skip this test if we can't get a real FS
|
||||
Assume.assumeNotNull(fs);
|
||||
|
||||
effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
|
||||
Assert.assertFalse("A negative timeout is not supposed to take effect",
|
||||
effectiveTimeout < 0);
|
||||
|
@ -74,4 +76,32 @@ public class TestAdlSdkConfiguration {
|
|||
|
||||
// The default value may vary by SDK, so that value is not tested here.
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSSLChannelModeConfig()
|
||||
throws IOException, URISyntaxException {
|
||||
testSSLChannelMode(SSLChannelMode.OpenSSL, "OpenSSL");
|
||||
testSSLChannelMode(SSLChannelMode.Default_JSE, "Default_JSE");
|
||||
testSSLChannelMode(SSLChannelMode.Default, "Default");
|
||||
// If config set is invalid, SSL channel mode will be Default.
|
||||
testSSLChannelMode(SSLChannelMode.Default, "Invalid");
|
||||
// Config value is case insensitive.
|
||||
testSSLChannelMode(SSLChannelMode.OpenSSL, "openssl");
|
||||
}
|
||||
|
||||
public void testSSLChannelMode(SSLChannelMode expectedMode,
|
||||
String sslChannelModeConfigValue) throws IOException, URISyntaxException {
|
||||
|
||||
AdlFileSystem fs = null;
|
||||
Configuration conf = null;
|
||||
|
||||
conf = AdlStorageConfiguration.getConfiguration();
|
||||
conf.set(ADL_SSL_CHANNEL_MODE, sslChannelModeConfigValue);
|
||||
fs = (AdlFileSystem) (AdlStorageConfiguration.createStorageConnector(conf));
|
||||
|
||||
SSLChannelMode sslChannelMode = fs.getAdlClient().getSSLChannelMode();
|
||||
Assert.assertEquals(
|
||||
"Unexpected SSL Channel Mode for adl.ssl.channel.mode config value : "
|
||||
+ sslChannelModeConfigValue, expectedMode, sslChannelMode);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue