Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16273. RBF: RouterRpcFairnessPolicyController add availableHandleOnPerNs metrics (#3553) 

Reviewed-by: Inigo Goiri <inigoiri@apache.org>
Reviewed-by: Hui Fei <ferhui@apache.org>
This commit is contained in:
Xiangyi Zhu 2021-11-08 08:30:53 +08:00 committed by GitHub
parent 45f164a854
commit e5cee76785
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 62 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java
View File

@ -23,6 +23,8 @@ import java.util.Map;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -75,4 +77,19 @@ public class AbstractRouterRpcFairnessPolicyController
protected int getAvailablePermits(String nsId) {
return this.permits.get(nsId).availablePermits();
}
@Override
public String getAvailableHandlerOnPerNs() {
JSONObject json = new JSONObject();
for (Map.Entry<String, Semaphore> entry : permits.entrySet()) {
try {
String nsId = entry.getKey();
int availableHandler = entry.getValue().availablePermits();
json.put(nsId, availableHandler);
} catch (JSONException e) {
LOG.warn("Cannot put {} into JSONObject", entry.getKey(), e);
}
}
return json.toString();
}
}

5
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java
View File

@ -46,4 +46,9 @@ public class NoRouterRpcFairnessPolicyController implements
public void shutdown() {
// Nothing for now.
}
@Override
public String getAvailableHandlerOnPerNs(){
return "N/A";
}
}

5
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java
View File

@ -62,4 +62,9 @@ public interface RouterRpcFairnessPolicyController {
* Shutdown steps to stop accepting new permission requests and clean-up.
*/
void shutdown();
/**
* Returns the JSON string of the available handler for each Ns.
*/
String getAvailableHandlerOnPerNs();
}

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
View File

@ -109,6 +109,12 @@ public interface FederationRPCMBean {
*/
String getRpcClientConnections();
/**
* JSON representation of the available handler per Ns.
* @return JSON string representation.
*/
String getAvailableHandlerOnPerNs();
/**
* Get the JSON representation of the async caller thread pool.
* @return JSON string representation of the async caller thread pool.

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
View File

@ -233,6 +233,12 @@ public class FederationRPCMetrics implements FederationRPCMBean {
return rpcServer.getRPCClient().getJSON();
}
@Override
public String getAvailableHandlerOnPerNs() {
return rpcServer.getRPCClient().
getRouterRpcFairnessPolicyController().getAvailableHandlerOnPerNs();
}
@Override
public String getAsyncCallerPool() {
return rpcServer.getRPCClient().getAsyncCallerPoolJson();

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
View File

@ -64,7 +64,6 @@ import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.federation.fairness.AbstractRouterRpcFairnessPolicyController;
import org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessPolicyController;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
@ -1584,10 +1583,9 @@ public class RouterRpcClient {
}
@VisibleForTesting
public AbstractRouterRpcFairnessPolicyController
public RouterRpcFairnessPolicyController
getRouterRpcFairnessPolicyController() {
return (AbstractRouterRpcFairnessPolicyController
)routerRpcFairnessPolicyController;
return routerRpcFairnessPolicyController;
}
private void incrRejectedPermitForNs(String ns) {

21
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java
View File

@ -30,6 +30,7 @@ import static org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnes
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@ -101,6 +102,26 @@ public class TestRouterRpcFairnessPolicyController {
verifyInstantiationError(conf, 1, 3);
}
@Test
public void testGetAvailableHandlerOnPerNs() {
RouterRpcFairnessPolicyController routerRpcFairnessPolicyController
= getFairnessPolicyController(30);
assertEquals("{\"concurrent\":10,\"ns2\":10,\"ns1\":10}",
routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs());
routerRpcFairnessPolicyController.acquirePermit("ns1");
assertEquals("{\"concurrent\":10,\"ns2\":10,\"ns1\":9}",
routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs());
}
@Test
public void testGetAvailableHandlerOnPerNsForNoFairness() {
Configuration conf = new Configuration();
RouterRpcFairnessPolicyController routerRpcFairnessPolicyController =
FederationUtil.newFairnessPolicyController(conf);
assertEquals("N/A",
routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs());
}
@Test
public void testAllocationErrorForLowPreconfiguredHandlers() {
Configuration conf = createConf(1);