Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16044. ABFS: Better exception handling of DNS errors followup 

Contributed by Da Zhou.
This commit is contained in:
Da Zhou 2019-01-14 19:32:56 +00:00 committed by Steve Loughran
parent 6d0bffe17e
commit 30863c5ae3
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 10 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
View File

@ -34,8 +34,6 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidAbfsRestOperati
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator.HttpException; import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator.HttpException;
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.UNKNOWN;
/** /**
* The AbfsRestOperation for Rest AbfsClient. * The AbfsRestOperation for Rest AbfsClient.
*/ */
@ -169,6 +167,10 @@ public class AbfsRestOperation {
httpOperation.processResponse(buffer, bufferOffset, bufferLength); httpOperation.processResponse(buffer, bufferOffset, bufferLength);
} catch (IOException ex) { } catch (IOException ex) {
if (ex instanceof UnknownHostException) {
LOG.warn(String.format("Unknown host name: %s. Retrying to resolve the host name...", httpOperation.getUrl().getHost()));
}
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
if (httpOperation != null) { if (httpOperation != null) {
LOG.debug("HttpRequestFailure: " + httpOperation.toString(), ex); LOG.debug("HttpRequestFailure: " + httpOperation.toString(), ex);
@ -177,14 +179,6 @@ public class AbfsRestOperation {
} }
} }
if (ex instanceof UnknownHostException) {
throw new AbfsRestOperationException(
UNKNOWN.getStatusCode(),
UNKNOWN.getErrorCode(),
String.format("Can not reach endpoint: %s, please check the account setting in configuration file", ex.getMessage()),
ex);
}
if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
throw new InvalidAbfsRestOperationException(ex); throw new InvalidAbfsRestOperationException(ex);
} }

8
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.fs.azurebfs;
import java.util.UUID; import java.util.UUID;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -55,8 +56,11 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
} }
} }
@Ignore("Enable this to verify the log warning message format for HostNotFoundException")
@Test @Test
public void verifyUnknownHost() throws Exception { public void testUnknownHost() throws Exception {
// When hitting hostName not found exception, the retry will take about 14 mins until failed.
// This test is to verify that the "Unknown host name: %s. Retrying to resolve the host name..." is logged as warning during the retry.
AbfsConfiguration conf = this.getConfiguration(); AbfsConfiguration conf = this.getConfiguration();
String accountName = this.getAccountName(); String accountName = this.getAccountName();
String fakeAccountName = "fake" + UUID.randomUUID() + accountName.substring(accountName.indexOf(".")); String fakeAccountName = "fake" + UUID.randomUUID() + accountName.substring(accountName.indexOf("."));
@ -66,7 +70,7 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
conf.set(FS_AZURE_ACCOUNT_KEY + "." + fakeAccountName, this.getAccountKey()); conf.set(FS_AZURE_ACCOUNT_KEY + "." + fakeAccountName, this.getAccountKey());
intercept(AbfsRestOperationException.class, intercept(AbfsRestOperationException.class,
"Can not reach endpoint: " + fakeAccountName, "UnknownHostException: " + fakeAccountName,
() -> FileSystem.get(conf.getRawConfiguration())); () -> FileSystem.get(conf.getRawConfiguration()));
} }
} }