Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26809: Report client backoff time for server overloaded (#4729) 

Co-authored-by: Briana Augenreich <baugenreich@hubspot.com>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
Bri Augenreich 2022-08-31 21:23:40 -04:00 committed by GitHub
parent 10d85f3161
commit 308cd729d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 25 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBatchRpcRetryingCaller.java
View File

@ -487,6 +487,11 @@ class AsyncBatchRpcRetryingCaller<T> {
} else {
delayNs = getPauseTime(pauseNsToUse, tries - 1);
}
if (isServerOverloaded) {
Optional<MetricsConnection> metrics = conn.getConnectionMetrics();
metrics.ifPresent(m -> m.incrementServerOverloadedBackoffTime(delayNs, TimeUnit.NANOSECONDS));
}
retryTimer.newTimeout(t -> groupAndSend(actions, tries + 1), delayNs, TimeUnit.NANOSECONDS);
}

4
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRpcRetryingCaller.java
View File

@ -139,6 +139,10 @@ public abstract class AsyncRpcRetryingCaller<T> {
delayNs = getPauseTime(pauseNsToUse, tries - 1);
}
tries++;
if (HBaseServerException.isServerOverloaded(error)) {
Optional<MetricsConnection> metrics = conn.getConnectionMetrics();
metrics.ifPresent(m -> m.incrementServerOverloadedBackoffTime(delayNs, TimeUnit.NANOSECONDS));
}
retryTimer.newTimeout(t -> doCall(), delayNs, TimeUnit.NANOSECONDS);
}

8
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.java
View File

@ -113,6 +113,8 @@ class AsyncScanSingleRegionRpcRetryingCaller {
private final Runnable completeWhenNoMoreResultsInRegion;
private final AsyncConnectionImpl conn;
private final CompletableFuture<Boolean> future;
private final HBaseRpcController controller;
@ -318,6 +320,7 @@ class AsyncScanSingleRegionRpcRetryingCaller {
long pauseNsForServerOverloaded, int maxAttempts, long scanTimeoutNs, long rpcTimeoutNs,
int startLogErrorsCnt) {
this.retryTimer = retryTimer;
this.conn = conn;
this.scan = scan;
this.scanMetrics = scanMetrics;
this.scannerId = scannerId;
@ -441,6 +444,11 @@ class AsyncScanSingleRegionRpcRetryingCaller {
return;
}
tries++;
if (HBaseServerException.isServerOverloaded(error)) {
Optional<MetricsConnection> metrics = conn.getConnectionMetrics();
metrics.ifPresent(m -> m.incrementServerOverloadedBackoffTime(delayNs, TimeUnit.NANOSECONDS));
}
retryTimer.newTimeout(t -> call(), delayNs, TimeUnit.NANOSECONDS);
}

8
hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
View File

@ -315,6 +315,7 @@ public class MetricsConnection implements StatisticTrackable {
protected final Histogram numActionsPerServerHist;
protected final Counter nsLookups;
protected final Counter nsLookupsFailed;
protected final Timer overloadedBackoffTimer;
// dynamic metrics
@ -376,6 +377,8 @@ public class MetricsConnection implements StatisticTrackable {
registry.histogram(name(MetricsConnection.class, "numActionsPerServer", scope));
this.nsLookups = registry.counter(name(this.getClass(), NS_LOOKUPS, scope));
this.nsLookupsFailed = registry.counter(name(this.getClass(), NS_LOOKUPS_FAILED, scope));
this.overloadedBackoffTimer =
registry.timer(name(this.getClass(), "overloadedBackoffDurationMs", scope));
this.reporter = JmxReporter.forRegistry(this.registry).build();
this.reporter.start();
@ -449,6 +452,11 @@ public class MetricsConnection implements StatisticTrackable {
this.runnerStats.updateDelayInterval(interval);
}
/** Update the overloaded backoff time **/
public void incrementServerOverloadedBackoffTime(long time, TimeUnit timeUnit) {
overloadedBackoffTimer.update(time, timeUnit);
}
/**
* Get a metric for {@code key} from {@code map}, or create it with {@code factory}.
*/