From 6f7b965808f71f44e2617c50d366a6375fdfbbfa Mon Sep 17 00:00:00 2001 From: Symious Date: Fri, 5 Nov 2021 11:03:07 +0800 Subject: [PATCH] HDFS-16296. RouterRpcFairnessPolicyController add rejected permits for each nameservice (#3613) --- .../metrics/FederationRPCMBean.java | 6 +++++ .../metrics/FederationRPCMetrics.java | 5 +++++ .../federation/router/RouterRpcClient.java | 22 +++++++++++++++++++ .../federation/MiniRouterDFSCluster.java | 10 +++++++++ .../fairness/TestRouterHandlersFairness.java | 14 ++++++++++++ 5 files changed, 57 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java index a4469a3025a..7bf253c148f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java @@ -120,4 +120,10 @@ public interface FederationRPCMBean { * @return Number of operations rejected due to lack of permits. */ long getProxyOpPermitRejected(); + + /** + * Get the number of operations rejected due to lack of permits of each namespace. + * @return Number of operations rejected due to lack of permits of each namespace. + */ + String getProxyOpPermitRejectedPerNs(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java index 1e6aa8050d9..1709f0b9ee8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java @@ -286,4 +286,9 @@ public void incrProxyOpPermitRejected() { public long getProxyOpPermitRejected() { return proxyOpPermitRejected.value(); } + + @Override + public String getProxyOpPermitRejectedPerNs() { + return rpcServer.getRPCClient().getRejectedPermitsPerNsJSON(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index ea65245e719..b09c963969d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -47,6 +47,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -54,6 +55,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -134,6 +136,7 @@ public class RouterRpcClient { /** Fairness manager to control handlers assigned per NS. */ private RouterRpcFairnessPolicyController routerRpcFairnessPolicyController; + private Map rejectedPermitsPerNs = new ConcurrentHashMap<>(); /** * Create a router RPC client to manage remote procedure calls to NNs. @@ -319,6 +322,15 @@ public String getAsyncCallerPoolJson() { return JSON.toString(info); } + /** + * JSON representation of the rejected permits for each nameservice. + * + * @return String representation of the rejected permits for each nameservice. + */ + public String getRejectedPermitsPerNsJSON() { + return JSON.toString(rejectedPermitsPerNs); + } + /** * Get ClientProtocol proxy client for a NameNode. Each combination of user + * NN must use a unique proxy client. Previously created clients are cached @@ -1544,6 +1556,7 @@ private void acquirePermit( if (rpcMonitor != null) { rpcMonitor.getRPCMetrics().incrProxyOpPermitRejected(); } + incrRejectedPermitForNs(nsId); LOG.debug("Permit denied for ugi: {} for method: {}", ugi, m.getMethodName()); String msg = @@ -1576,4 +1589,13 @@ private void releasePermit( return (AbstractRouterRpcFairnessPolicyController )routerRpcFairnessPolicyController; } + + private void incrRejectedPermitForNs(String ns) { + rejectedPermitsPerNs.computeIfAbsent(ns, k -> new LongAdder()).increment(); + } + + public Long getRejectedPermitForNs(String ns) { + return rejectedPermitsPerNs.containsKey(ns) ? + rejectedPermitsPerNs.get(ns).longValue() : 0L; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index 8a7a03e018b..ac6ecd4398c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -86,6 +86,8 @@ import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport; import org.apache.hadoop.hdfs.server.federation.router.Router; import org.apache.hadoop.hdfs.server.federation.router.RouterClient; +import org.apache.hadoop.hdfs.server.federation.router.RouterRpcClient; +import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; @@ -267,6 +269,14 @@ public DFSClient getClient() throws IOException, URISyntaxException { public Configuration getConf() { return conf; } + + public RouterRpcServer getRouterRpcServer() { + return router.getRpcServer(); + } + + public RouterRpcClient getRouterRpcClient() { + return getRouterRpcServer().getRPCClient(); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java index c3fc324255c..233f7e00786 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java @@ -146,6 +146,7 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness) Configuration conf = new HdfsConfiguration(); final int numOps = 10; final AtomicInteger overloadException = new AtomicInteger(); + int originalRejectedPermits = getTotalRejectedPermits(routerContext); for (int i = 0; i < numOps; i++) { DFSClient routerClient = null; @@ -177,6 +178,9 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness) } overloadException.get(); } + int latestRejectedPermits = getTotalRejectedPermits(routerContext); + assertEquals(latestRejectedPermits - originalRejectedPermits, + overloadException.get()); if (fairness) { assertTrue(overloadException.get() > 0); @@ -208,4 +212,14 @@ private void invokeConcurrent(ClientProtocol routerProto, String clientName) routerProto.renewLease(clientName); } + private int getTotalRejectedPermits(RouterContext routerContext) { + int totalRejectedPermits = 0; + for (String ns : cluster.getNameservices()) { + totalRejectedPermits += routerContext.getRouterRpcClient() + .getRejectedPermitForNs(ns); + } + totalRejectedPermits += routerContext.getRouterRpcClient() + .getRejectedPermitForNs(RouterRpcFairnessConstants.CONCURRENT_NS); + return totalRejectedPermits; + } }