HADOOP-18606. ABFS: Add reason in x-ms-client-request-id on a retried API call. (#5299)
Contributed by Pranav Saxena
This commit is contained in:
Pranav Saxena
GitHub
parent
dd9ef9e0e7
commit
2b156c2b32
|
|
@ -111,6 +111,17 @@ public final class AbfsHttpConstants {
|
|||
public static final char CHAR_EQUALS = '=';
|
||||
public static final char CHAR_STAR = '*';
|
||||
public static final char CHAR_PLUS = '+';
|
||||
/**
|
||||
* Value that differentiates categories of the http_status.
|
||||
* <pre>
|
||||
* 100 - 199 : Informational responses
|
||||
* 200 - 299 : Successful responses
|
||||
* 300 - 399 : Redirection messages
|
||||
* 400 - 499 : Client error responses
|
||||
* 500 - 599 : Server error responses
|
||||
* </pre>
|
||||
*/
|
||||
public static final Integer HTTP_STATUS_CATEGORY_QUOTIENT = 100;
|
||||
|
||||
private AbfsHttpConstants() {}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,6 +66,10 @@ public enum AzureServiceErrorCode {
|
|||
return this.errorCode;
|
||||
}
|
||||
|
||||
public String getErrorMessage() {
|
||||
return this.errorMessage;
|
||||
}
|
||||
|
||||
public static List<AzureServiceErrorCode> getAzureServiceCode(int httpStatusCode) {
|
||||
List<AzureServiceErrorCode> errorCodes = new ArrayList<>();
|
||||
if (httpStatusCode == UNKNOWN.httpStatusCode) {
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import java.util.List;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.classification.VisibleForTesting;
|
||||
import org.apache.hadoop.fs.azurebfs.AbfsStatistic;
|
||||
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
|
||||
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
|
||||
|
|
@ -73,6 +74,12 @@ public class AbfsRestOperation {
|
|||
private AbfsHttpOperation result;
|
||||
private AbfsCounters abfsCounters;
|
||||
|
||||
/**
|
||||
* This variable contains the reason of last API call within the same
|
||||
* AbfsRestOperation object.
|
||||
*/
|
||||
private String failureReason;
|
||||
|
||||
/**
|
||||
* Checks if there is non-null HTTP response.
|
||||
* @return true if there is a non-null HTTP response from the ABFS call.
|
||||
|
|
@ -208,7 +215,7 @@ public class AbfsRestOperation {
|
|||
private void completeExecute(TracingContext tracingContext)
|
||||
throws AzureBlobFileSystemException {
|
||||
// see if we have latency reports from the previous requests
|
||||
String latencyHeader = this.client.getAbfsPerfTracker().getClientLatency();
|
||||
String latencyHeader = getClientLatency();
|
||||
if (latencyHeader != null && !latencyHeader.isEmpty()) {
|
||||
AbfsHttpHeader httpHeader =
|
||||
new AbfsHttpHeader(HttpHeaderConfigurations.X_MS_ABFS_CLIENT_LATENCY, latencyHeader);
|
||||
|
|
@ -237,6 +244,11 @@ public class AbfsRestOperation {
|
|||
LOG.trace("{} REST operation complete", operationType);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
String getClientLatency() {
|
||||
return client.getAbfsPerfTracker().getClientLatency();
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a single HTTP operation to complete the REST operation. If it
|
||||
* fails, there may be a retry. The retryCount is incremented with each
|
||||
|
|
@ -248,9 +260,9 @@ public class AbfsRestOperation {
|
|||
|
||||
try {
|
||||
// initialize the HTTP request and open the connection
|
||||
httpOperation = new AbfsHttpOperation(url, method, requestHeaders);
|
||||
httpOperation = createHttpOperation();
|
||||
incrementCounter(AbfsStatistic.CONNECTIONS_MADE, 1);
|
||||
tracingContext.constructHeader(httpOperation);
|
||||
tracingContext.constructHeader(httpOperation, failureReason);
|
||||
|
||||
switch(client.getAuthType()) {
|
||||
case Custom:
|
||||
|
|
@ -303,6 +315,7 @@ public class AbfsRestOperation {
|
|||
} catch (UnknownHostException ex) {
|
||||
String hostname = null;
|
||||
hostname = httpOperation.getHost();
|
||||
failureReason = RetryReason.getAbbreviation(ex, null, null);
|
||||
LOG.warn("Unknown host name: {}. Retrying to resolve the host name...",
|
||||
hostname);
|
||||
if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
|
||||
|
|
@ -314,6 +327,8 @@ public class AbfsRestOperation {
|
|||
LOG.debug("HttpRequestFailure: {}, {}", httpOperation, ex);
|
||||
}
|
||||
|
||||
failureReason = RetryReason.getAbbreviation(ex, -1, "");
|
||||
|
||||
if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
|
||||
throw new InvalidAbfsRestOperationException(ex);
|
||||
}
|
||||
|
|
@ -326,6 +341,8 @@ public class AbfsRestOperation {
|
|||
LOG.debug("HttpRequest: {}: {}", operationType, httpOperation);
|
||||
|
||||
if (client.getRetryPolicy().shouldRetry(retryCount, httpOperation.getStatusCode())) {
|
||||
int status = httpOperation.getStatusCode();
|
||||
failureReason = RetryReason.getAbbreviation(null, status, httpOperation.getStorageErrorMessage());
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -334,6 +351,15 @@ public class AbfsRestOperation {
|
|||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates new object of {@link AbfsHttpOperation} with the url, method, and
|
||||
* requestHeaders fields of the AbfsRestOperation object.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
AbfsHttpOperation createHttpOperation() throws IOException {
|
||||
return new AbfsHttpOperation(url, method, requestHeaders);
|
||||
}
|
||||
|
||||
/**
|
||||
* Incrementing Abfs counters with a long value.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ClientErrorRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ConnectionResetRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ConnectionTimeoutRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ReadTimeoutRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.RetryReasonCategory;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ServerErrorRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownHostRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownIOExceptionRetryReason;
|
||||
import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownSocketExceptionRetryReason;
|
||||
|
||||
|
||||
/**
|
||||
* This utility class exposes methods to convert a server response-error to a
|
||||
* category of error.
|
||||
*/
|
||||
final class RetryReason {
|
||||
|
||||
/**
|
||||
* Linked-list of the implementations of RetryReasonCategory. The objects in the
|
||||
* list are arranged by the rank of their significance.
|
||||
* <ul>
|
||||
* <li>ServerError (statusCode==5XX), ClientError (statusCode==4XX) are
|
||||
* independent of other retryReason categories.</li>
|
||||
* <li>Since {@link java.net.SocketException} is subclass of
|
||||
* {@link java.io.IOException},
|
||||
* hence, {@link UnknownIOExceptionRetryReason} is placed before
|
||||
* {@link UnknownSocketExceptionRetryReason}</li>
|
||||
* <li>Since, connectionTimeout, readTimeout, and connectionReset are
|
||||
* {@link java.net.SocketTimeoutException} exceptions with different messages,
|
||||
* hence, {@link ConnectionTimeoutRetryReason}, {@link ReadTimeoutRetryReason},
|
||||
* {@link ConnectionResetRetryReason} are above {@link UnknownIOExceptionRetryReason}.
|
||||
* There is no order between the three reasons as they are differentiated
|
||||
* by exception-message.</li>
|
||||
* <li>Since, {@link java.net.UnknownHostException} is subclass of
|
||||
* {@link java.io.IOException}, {@link UnknownHostRetryReason} is placed
|
||||
* over {@link UnknownIOExceptionRetryReason}</li>
|
||||
* </ul>
|
||||
*/
|
||||
private static List<RetryReasonCategory> rankedReasonCategories
|
||||
= new LinkedList<RetryReasonCategory>() {{
|
||||
add(new ServerErrorRetryReason());
|
||||
add(new ClientErrorRetryReason());
|
||||
add(new UnknownIOExceptionRetryReason());
|
||||
add(new UnknownSocketExceptionRetryReason());
|
||||
add(new ConnectionTimeoutRetryReason());
|
||||
add(new ReadTimeoutRetryReason());
|
||||
add(new UnknownHostRetryReason());
|
||||
add(new ConnectionResetRetryReason());
|
||||
}};
|
||||
|
||||
private RetryReason() {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to get correct abbreviation for a given set of exception, statusCode,
|
||||
* storageStatusCode.
|
||||
*
|
||||
* @param ex exception caught during server communication.
|
||||
* @param statusCode statusCode in the server response.
|
||||
* @param storageErrorMessage storageErrorMessage in the server response.
|
||||
*
|
||||
* @return abbreviation for the the given set of exception, statusCode, storageStatusCode.
|
||||
*/
|
||||
static String getAbbreviation(Exception ex,
|
||||
Integer statusCode,
|
||||
String storageErrorMessage) {
|
||||
String result = null;
|
||||
for (RetryReasonCategory retryReasonCategory : rankedReasonCategories) {
|
||||
final String abbreviation
|
||||
= retryReasonCategory.captureAndGetAbbreviation(ex,
|
||||
statusCode, storageErrorMessage);
|
||||
if (abbreviation != null) {
|
||||
result = abbreviation;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services;
|
||||
|
||||
public final class RetryReasonConstants {
|
||||
|
||||
private RetryReasonConstants() {
|
||||
|
||||
}
|
||||
public static final String CONNECTION_TIMEOUT_JDK_MESSAGE = "connect timed out";
|
||||
public static final String READ_TIMEOUT_JDK_MESSAGE = "Read timed out";
|
||||
public static final String CONNECTION_RESET_MESSAGE = "Connection reset";
|
||||
public static final String OPERATION_BREACH_MESSAGE = "Operations per second is over the account limit.";
|
||||
public static final String CONNECTION_RESET_ABBREVIATION = "CR";
|
||||
public static final String CONNECTION_TIMEOUT_ABBREVIATION = "CT";
|
||||
public static final String READ_TIMEOUT_ABBREVIATION = "RT";
|
||||
public static final String INGRESS_LIMIT_BREACH_ABBREVIATION = "ING";
|
||||
public static final String EGRESS_LIMIT_BREACH_ABBREVIATION = "EGR";
|
||||
public static final String OPERATION_LIMIT_BREACH_ABBREVIATION = "OPR";
|
||||
public static final String UNKNOWN_HOST_EXCEPTION_ABBREVIATION = "UH";
|
||||
public static final String IO_EXCEPTION_ABBREVIATION = "IOE";
|
||||
public static final String SOCKET_EXCEPTION_ABBREVIATION = "SE";
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_STATUS_CATEGORY_QUOTIENT;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for 4XX status-code.
|
||||
*/
|
||||
public class ClientErrorRetryReason extends RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
if (statusCode == null || statusCode / HTTP_STATUS_CATEGORY_QUOTIENT != 4) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return statusCode + "";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for connection-reset exception.
|
||||
*/
|
||||
public class ConnectionResetRetryReason extends
|
||||
RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return checkExceptionMessage(ex, CONNECTION_RESET_MESSAGE);
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return CONNECTION_RESET_ABBREVIATION;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for connection-timeout.
|
||||
*/
|
||||
public class ConnectionTimeoutRetryReason extends
|
||||
RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return CONNECTION_TIMEOUT_ABBREVIATION;
|
||||
}
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return checkExceptionMessage(ex, CONNECTION_TIMEOUT_JDK_MESSAGE);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for read-timeout.
|
||||
*/
|
||||
public class ReadTimeoutRetryReason extends RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return checkExceptionMessage(ex, READ_TIMEOUT_JDK_MESSAGE);
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return READ_TIMEOUT_ABBREVIATION;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Provides methods to define if given exception can be categorised to certain category.
|
||||
* Each category has a different implementation of the abstract class.
|
||||
*/
|
||||
public abstract class RetryReasonCategory {
|
||||
|
||||
/**
|
||||
* Returns if given server response error can be categorised by the implementation.
|
||||
*
|
||||
* @param ex exception captured in the server response.
|
||||
* @param statusCode statusCode on the server response
|
||||
* @param serverErrorMessage serverErrorMessage on the server response.
|
||||
*
|
||||
* @return <ol><li>true if server response error can be categorised by the implementation</li>
|
||||
* <li>false if response error can not be categorised by the implementation</li></ol>
|
||||
*/
|
||||
abstract Boolean canCapture(Exception ex,
|
||||
Integer statusCode,
|
||||
String serverErrorMessage);
|
||||
|
||||
/**
|
||||
* Returns the abbreviation corresponding to the server response error.
|
||||
*
|
||||
* @param statusCode statusCode on the server response
|
||||
* @param serverErrorMessage serverErrorMessage on the server response.
|
||||
*
|
||||
* @return abbreviation on the basis of the statusCode and the serverErrorMessage
|
||||
*/
|
||||
abstract String getAbbreviation(Integer statusCode, String serverErrorMessage);
|
||||
|
||||
/**
|
||||
* Converts the server-error response to an abbreviation if the response can be
|
||||
* categorised by the implementation.
|
||||
*
|
||||
* @param ex exception received while making API request
|
||||
* @param statusCode statusCode received in the server-response
|
||||
* @param serverErrorMessage error-message received in the server-response
|
||||
*
|
||||
* @return abbreviation if the server-response can be categorised by the implementation.
|
||||
* null if the server-response can not be categorised by the implementation.
|
||||
*/
|
||||
public String captureAndGetAbbreviation(Exception ex,
|
||||
Integer statusCode,
|
||||
String serverErrorMessage) {
|
||||
if (canCapture(ex, statusCode, serverErrorMessage)) {
|
||||
return getAbbreviation(statusCode, serverErrorMessage);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a required search-string is in the exception's message.
|
||||
*/
|
||||
Boolean checkExceptionMessage(final Exception exceptionCaptured,
|
||||
final String search) {
|
||||
if (search == null) {
|
||||
return false;
|
||||
}
|
||||
if (exceptionCaptured != null
|
||||
&& exceptionCaptured.getMessage() != null
|
||||
&& exceptionCaptured.getMessage()
|
||||
.toLowerCase(Locale.US)
|
||||
.contains(search.toLowerCase(Locale.US))) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
|
||||
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_STATUS_CATEGORY_QUOTIENT;
|
||||
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
|
||||
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for 5XX status-code.
|
||||
*/
|
||||
public class ServerErrorRetryReason extends RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
if (statusCode == null || statusCode / HTTP_STATUS_CATEGORY_QUOTIENT != 5) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
if (statusCode == HTTP_UNAVAILABLE && serverErrorMessage != null) {
|
||||
String splitedServerErrorMessage = serverErrorMessage.split(System.lineSeparator(),
|
||||
2)[0];
|
||||
if (INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage().equalsIgnoreCase(
|
||||
splitedServerErrorMessage)) {
|
||||
return INGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
}
|
||||
if (EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage().equalsIgnoreCase(
|
||||
splitedServerErrorMessage)) {
|
||||
return EGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
}
|
||||
if (OPERATION_BREACH_MESSAGE.equalsIgnoreCase(
|
||||
splitedServerErrorMessage)) {
|
||||
return OPERATION_LIMIT_BREACH_ABBREVIATION;
|
||||
}
|
||||
return HTTP_UNAVAILABLE + "";
|
||||
}
|
||||
return statusCode + "";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for {@link UnknownHostException}.
|
||||
*/
|
||||
public class UnknownHostRetryReason extends RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
if (ex instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
|
||||
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for {@link IOException}.
|
||||
*/
|
||||
public class UnknownIOExceptionRetryReason extends
|
||||
RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
if (ex instanceof IOException) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return IO_EXCEPTION_ABBREVIATION;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import java.net.SocketException;
|
||||
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
|
||||
|
||||
/**
|
||||
* Category that can capture server-response errors for {@link SocketException}.
|
||||
*/
|
||||
public class UnknownSocketExceptionRetryReason extends
|
||||
RetryReasonCategory {
|
||||
|
||||
@Override
|
||||
Boolean canCapture(final Exception ex,
|
||||
final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
if (ex instanceof SocketException) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
String getAbbreviation(final Integer statusCode,
|
||||
final String serverErrorMessage) {
|
||||
return SOCKET_EXCEPTION_ABBREVIATION;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A retryReasonCategory defines methods applicable on server-response errors.
|
||||
*/
|
||||
@Private
|
||||
@Evolving
|
||||
package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
|
|
@ -152,8 +152,10 @@ public class TracingContext {
|
|||
* X_MS_CLIENT_REQUEST_ID header of the http operation
|
||||
* @param httpOperation AbfsHttpOperation instance to set header into
|
||||
* connection
|
||||
* @param previousFailure List of failures seen before this API trigger on
|
||||
* same operation from AbfsClient.
|
||||
*/
|
||||
public void constructHeader(AbfsHttpOperation httpOperation) {
|
||||
public void constructHeader(AbfsHttpOperation httpOperation, String previousFailure) {
|
||||
clientRequestId = UUID.randomUUID().toString();
|
||||
switch (format) {
|
||||
case ALL_ID_FORMAT: // Optional IDs (e.g. streamId) may be empty
|
||||
|
|
@ -161,6 +163,7 @@ public class TracingContext {
|
|||
clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":"
|
||||
+ primaryRequestId + ":" + streamID + ":" + opType + ":"
|
||||
+ retryCount;
|
||||
header = addFailureReasons(header, previousFailure);
|
||||
break;
|
||||
case TWO_ID_FORMAT:
|
||||
header = clientCorrelationID + ":" + clientRequestId;
|
||||
|
|
@ -174,6 +177,14 @@ public class TracingContext {
|
|||
httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header);
|
||||
}
|
||||
|
||||
private String addFailureReasons(final String header,
|
||||
final String previousFailure) {
|
||||
if (previousFailure == null) {
|
||||
return header;
|
||||
}
|
||||
return String.format("%s_%s", header, previousFailure);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return header representing the request associated with the tracingContext
|
||||
* @return Header string set into X_MS_CLIENT_REQUEST_ID
|
||||
|
|
|
|||
|
|
@ -0,0 +1,302 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.SocketException;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.stubbing.Stubber;
|
||||
|
||||
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
|
||||
|
||||
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
|
||||
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
|
||||
import static java.net.HttpURLConnection.HTTP_OK;
|
||||
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
|
||||
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
|
||||
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.AuthType.OAuth;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.nullable;
|
||||
|
||||
public class TestAbfsRestOperationMockFailures {
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForConnectTimeoutRetry() throws Exception {
|
||||
Exception[] exceptions = new Exception[1];
|
||||
String[] abbreviations = new String[1];
|
||||
exceptions[0] = new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE);
|
||||
abbreviations[0] = CONNECTION_TIMEOUT_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForConnectAndReadTimeoutRetry()
|
||||
throws Exception {
|
||||
Exception[] exceptions = new Exception[2];
|
||||
String[] abbreviations = new String[2];
|
||||
exceptions[0] = new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE);
|
||||
abbreviations[0] = CONNECTION_TIMEOUT_ABBREVIATION;
|
||||
exceptions[1] = new SocketTimeoutException(READ_TIMEOUT_JDK_MESSAGE);
|
||||
abbreviations[1] = READ_TIMEOUT_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForReadTimeoutRetry() throws Exception {
|
||||
Exception[] exceptions = new Exception[1];
|
||||
String[] abbreviations = new String[1];
|
||||
exceptions[0] = new SocketTimeoutException(READ_TIMEOUT_JDK_MESSAGE);
|
||||
abbreviations[0] = READ_TIMEOUT_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForUnknownHostRetry() throws Exception {
|
||||
Exception[] exceptions = new Exception[1];
|
||||
String[] abbreviations = new String[1];
|
||||
exceptions[0] = new UnknownHostException();
|
||||
abbreviations[0] = UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForConnectionResetRetry() throws Exception {
|
||||
Exception[] exceptions = new Exception[1];
|
||||
String[] abbreviations = new String[1];
|
||||
exceptions[0] = new SocketTimeoutException(CONNECTION_RESET_MESSAGE + " by peer");
|
||||
abbreviations[0] = CONNECTION_RESET_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForUnknownSocketExRetry() throws Exception {
|
||||
Exception[] exceptions = new Exception[1];
|
||||
String[] abbreviations = new String[1];
|
||||
exceptions[0] = new SocketException("unknown");
|
||||
abbreviations[0] = SOCKET_EXCEPTION_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdForIOERetry() throws Exception {
|
||||
Exception[] exceptions = new Exception[1];
|
||||
String[] abbreviations = new String[1];
|
||||
exceptions[0] = new InterruptedIOException();
|
||||
abbreviations[0] = IO_EXCEPTION_ABBREVIATION;
|
||||
testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdFor400Retry() throws Exception {
|
||||
testClientRequestIdForStatusRetry(HTTP_BAD_REQUEST, "", "400");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdFor500Retry() throws Exception {
|
||||
testClientRequestIdForStatusRetry(HTTP_INTERNAL_ERROR, "", "500");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdFor503INGRetry() throws Exception {
|
||||
testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE,
|
||||
INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage(),
|
||||
INGRESS_LIMIT_BREACH_ABBREVIATION);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdFor503egrRetry() throws Exception {
|
||||
testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE,
|
||||
EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage(),
|
||||
EGRESS_LIMIT_BREACH_ABBREVIATION);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdFor503OPRRetry() throws Exception {
|
||||
testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE,
|
||||
OPERATION_BREACH_MESSAGE, OPERATION_LIMIT_BREACH_ABBREVIATION);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientRequestIdFor503OtherRetry() throws Exception {
|
||||
testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE, "Other.", "503");
|
||||
}
|
||||
|
||||
private void testClientRequestIdForStatusRetry(int status,
|
||||
String serverErrorMessage,
|
||||
String keyExpected) throws Exception {
|
||||
|
||||
AbfsClient abfsClient = Mockito.mock(AbfsClient.class);
|
||||
ExponentialRetryPolicy retryPolicy = Mockito.mock(
|
||||
ExponentialRetryPolicy.class);
|
||||
addMockBehaviourToAbfsClient(abfsClient, retryPolicy);
|
||||
|
||||
|
||||
AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation(
|
||||
AbfsRestOperationType.ReadFile,
|
||||
abfsClient,
|
||||
"PUT",
|
||||
null,
|
||||
new ArrayList<>()
|
||||
));
|
||||
|
||||
AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class);
|
||||
addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation);
|
||||
|
||||
Mockito.doNothing()
|
||||
.doNothing()
|
||||
.when(httpOperation)
|
||||
.processResponse(nullable(byte[].class), nullable(int.class),
|
||||
nullable(int.class));
|
||||
|
||||
int[] statusCount = new int[1];
|
||||
statusCount[0] = 0;
|
||||
Mockito.doAnswer(answer -> {
|
||||
if (statusCount[0] <= 5) {
|
||||
statusCount[0]++;
|
||||
return status;
|
||||
}
|
||||
return HTTP_OK;
|
||||
}).when(httpOperation).getStatusCode();
|
||||
|
||||
Mockito.doReturn(serverErrorMessage)
|
||||
.when(httpOperation)
|
||||
.getStorageErrorMessage();
|
||||
|
||||
TracingContext tracingContext = Mockito.mock(TracingContext.class);
|
||||
Mockito.doNothing().when(tracingContext).setRetryCount(nullable(int.class));
|
||||
|
||||
int[] count = new int[1];
|
||||
count[0] = 0;
|
||||
Mockito.doAnswer(invocationOnMock -> {
|
||||
if (count[0] == 1) {
|
||||
Assertions.assertThat((String) invocationOnMock.getArgument(1))
|
||||
.isEqualTo(keyExpected);
|
||||
}
|
||||
count[0]++;
|
||||
return null;
|
||||
}).when(tracingContext).constructHeader(any(), any());
|
||||
|
||||
abfsRestOperation.execute(tracingContext);
|
||||
Assertions.assertThat(count[0]).isEqualTo(2);
|
||||
|
||||
}
|
||||
|
||||
private void testClientRequestIdForTimeoutRetry(Exception[] exceptions,
|
||||
String[] abbreviationsExpected,
|
||||
int len) throws Exception {
|
||||
AbfsClient abfsClient = Mockito.mock(AbfsClient.class);
|
||||
ExponentialRetryPolicy retryPolicy = Mockito.mock(
|
||||
ExponentialRetryPolicy.class);
|
||||
addMockBehaviourToAbfsClient(abfsClient, retryPolicy);
|
||||
|
||||
|
||||
AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation(
|
||||
AbfsRestOperationType.ReadFile,
|
||||
abfsClient,
|
||||
"PUT",
|
||||
null,
|
||||
new ArrayList<>()
|
||||
));
|
||||
|
||||
AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class);
|
||||
addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation);
|
||||
|
||||
Stubber stubber = Mockito.doThrow(exceptions[0]);
|
||||
for (int iteration = 1; iteration < len; iteration++) {
|
||||
stubber.doThrow(exceptions[iteration]);
|
||||
}
|
||||
stubber
|
||||
.doNothing()
|
||||
.when(httpOperation)
|
||||
.processResponse(nullable(byte[].class), nullable(int.class),
|
||||
nullable(int.class));
|
||||
|
||||
Mockito.doReturn(HTTP_OK).when(httpOperation).getStatusCode();
|
||||
|
||||
TracingContext tracingContext = Mockito.mock(TracingContext.class);
|
||||
Mockito.doNothing().when(tracingContext).setRetryCount(nullable(int.class));
|
||||
|
||||
int[] count = new int[1];
|
||||
count[0] = 0;
|
||||
Mockito.doAnswer(invocationOnMock -> {
|
||||
if (count[0] > 0 && count[0] <= len) {
|
||||
Assertions.assertThat((String) invocationOnMock.getArgument(1))
|
||||
.isEqualTo(abbreviationsExpected[count[0] - 1]);
|
||||
}
|
||||
count[0]++;
|
||||
return null;
|
||||
}).when(tracingContext).constructHeader(any(), any());
|
||||
|
||||
abfsRestOperation.execute(tracingContext);
|
||||
Assertions.assertThat(count[0]).isEqualTo(len + 1);
|
||||
}
|
||||
|
||||
private void addMockBehaviourToRestOpAndHttpOp(final AbfsRestOperation abfsRestOperation,
|
||||
final AbfsHttpOperation httpOperation) throws IOException {
|
||||
HttpURLConnection httpURLConnection = Mockito.mock(HttpURLConnection.class);
|
||||
Mockito.doNothing()
|
||||
.when(httpURLConnection)
|
||||
.setRequestProperty(nullable(String.class), nullable(String.class));
|
||||
Mockito.doReturn(httpURLConnection).when(httpOperation).getConnection();
|
||||
Mockito.doReturn("").when(abfsRestOperation).getClientLatency();
|
||||
Mockito.doReturn(httpOperation).when(abfsRestOperation).createHttpOperation();
|
||||
}
|
||||
|
||||
private void addMockBehaviourToAbfsClient(final AbfsClient abfsClient,
|
||||
final ExponentialRetryPolicy retryPolicy) throws IOException {
|
||||
Mockito.doReturn(OAuth).when(abfsClient).getAuthType();
|
||||
Mockito.doReturn("").when(abfsClient).getAccessToken();
|
||||
AbfsThrottlingIntercept intercept = Mockito.mock(
|
||||
AbfsThrottlingIntercept.class);
|
||||
Mockito.doReturn(intercept).when(abfsClient).getIntercept();
|
||||
Mockito.doNothing()
|
||||
.when(intercept)
|
||||
.sendingRequest(any(), nullable(AbfsCounters.class));
|
||||
Mockito.doNothing().when(intercept).updateMetrics(any(), any());
|
||||
|
||||
Mockito.doReturn(retryPolicy).when(abfsClient).getRetryPolicy();
|
||||
Mockito.doReturn(true)
|
||||
.when(retryPolicy)
|
||||
.shouldRetry(nullable(Integer.class), nullable(Integer.class));
|
||||
Mockito.doReturn(false).when(retryPolicy).shouldRetry(1, HTTP_OK);
|
||||
Mockito.doReturn(false).when(retryPolicy).shouldRetry(2, HTTP_OK);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.services;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.SocketException;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.junit.Test;
|
||||
|
||||
import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
|
||||
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
|
||||
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
|
||||
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
|
||||
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
|
||||
import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
|
||||
|
||||
public class TestRetryReason {
|
||||
|
||||
@Test
|
||||
public void test4xxStatusRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_FORBIDDEN, null))
|
||||
.describedAs("Abbreviation for 4xx should be equal to 4xx")
|
||||
.isEqualTo(HTTP_FORBIDDEN + "");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConnectionResetRetryReason() {
|
||||
SocketException connReset = new SocketException(CONNECTION_RESET_MESSAGE.toUpperCase());
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(connReset, null, null)).isEqualTo(CONNECTION_RESET_ABBREVIATION);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConnectionTimeoutRetryReason() {
|
||||
SocketTimeoutException connectionTimeoutException = new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE);
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(connectionTimeoutException, null, null)).isEqualTo(
|
||||
CONNECTION_TIMEOUT_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadTimeoutRetryReason() {
|
||||
SocketTimeoutException connectionTimeoutException = new SocketTimeoutException(READ_TIMEOUT_JDK_MESSAGE);
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(connectionTimeoutException, null, null)).isEqualTo(
|
||||
READ_TIMEOUT_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEgressLimitRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage())).isEqualTo(
|
||||
EGRESS_LIMIT_BREACH_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIngressLimitRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage())).isEqualTo(
|
||||
INGRESS_LIMIT_BREACH_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOperationLimitRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, OPERATION_BREACH_MESSAGE)).isEqualTo(
|
||||
OPERATION_LIMIT_BREACH_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test503UnknownRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, null)).isEqualTo(
|
||||
"503"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test500RetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_INTERNAL_ERROR, null)).isEqualTo(
|
||||
"500"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUnknownHostRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(new UnknownHostException(), null, null)).isEqualTo(
|
||||
UNKNOWN_HOST_EXCEPTION_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUnknownIOExceptionRetryReason() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(new IOException(), null, null)).isEqualTo(
|
||||
IO_EXCEPTION_ABBREVIATION
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUnknownSocketException() {
|
||||
Assertions.assertThat(RetryReason.getAbbreviation(new SocketException(), null, null)).isEqualTo(
|
||||
SOCKET_EXCEPTION_ABBREVIATION
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue