HDFS-15970. Print network topology on the web (#2896)
This commit is contained in:
litao
GitHub
parent
e66d7d91be
commit
85a3532849
|
|
@ -166,7 +166,7 @@ void start() throws IOException {
|
|||
|
||||
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
|
||||
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
|
||||
setupServlets(httpServer, conf);
|
||||
setupServlets(httpServer);
|
||||
httpServer.start();
|
||||
|
||||
int connIdx = 0;
|
||||
|
|
@ -243,7 +243,7 @@ void setAliasMap(InMemoryAliasMap aliasMap) {
|
|||
httpServer.setAttribute(ALIASMAP_ATTRIBUTE_KEY, aliasMap);
|
||||
}
|
||||
|
||||
private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
|
||||
private static void setupServlets(HttpServer2 httpServer) {
|
||||
httpServer.addInternalServlet("startupProgress",
|
||||
StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class);
|
||||
httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class,
|
||||
|
|
@ -253,6 +253,8 @@ private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
|
|||
httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
|
||||
IsNameNodeActiveServlet.PATH_SPEC,
|
||||
IsNameNodeActiveServlet.class);
|
||||
httpServer.addInternalServlet("topology",
|
||||
NetworkTopologyServlet.PATH_SPEC, NetworkTopologyServlet.class);
|
||||
}
|
||||
|
||||
static FSImage getFsImageFromContext(ServletContext context) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,187 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs.server.namenode;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonFactory;
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.net.Node;
|
||||
import org.apache.hadoop.net.NodeBase;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
||||
import javax.servlet.ServletContext;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.ws.rs.core.HttpHeaders;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* A servlet to print out the network topology.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class NetworkTopologyServlet extends DfsServlet {
|
||||
|
||||
public static final String PATH_SPEC = "/topology";
|
||||
|
||||
protected static final String FORMAT_JSON = "json";
|
||||
protected static final String FORMAT_TEXT = "text";
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response)
|
||||
throws IOException {
|
||||
final ServletContext context = getServletContext();
|
||||
|
||||
String format = parseAcceptHeader(request);
|
||||
if (FORMAT_TEXT.equals(format)) {
|
||||
response.setContentType("text/plain; charset=UTF-8");
|
||||
} else if (FORMAT_JSON.equals(format)) {
|
||||
response.setContentType("application/json; charset=UTF-8");
|
||||
}
|
||||
|
||||
NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
|
||||
BlockManager bm = nn.getNamesystem().getBlockManager();
|
||||
List<Node> leaves = bm.getDatanodeManager().getNetworkTopology()
|
||||
.getLeaves(NodeBase.ROOT);
|
||||
|
||||
try (PrintStream out = new PrintStream(
|
||||
response.getOutputStream(), false, "UTF-8")) {
|
||||
printTopology(out, leaves, format);
|
||||
} catch (Throwable t) {
|
||||
String errMsg = "Print network topology failed. "
|
||||
+ StringUtils.stringifyException(t);
|
||||
response.sendError(HttpServletResponse.SC_GONE, errMsg);
|
||||
throw new IOException(errMsg);
|
||||
} finally {
|
||||
response.getOutputStream().close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Display each rack and the nodes assigned to that rack, as determined
|
||||
* by the NameNode, in a hierarchical manner. The nodes and racks are
|
||||
* sorted alphabetically.
|
||||
*
|
||||
* @param stream print stream
|
||||
* @param leaves leaves nodes under base scope
|
||||
* @param format the response format
|
||||
*/
|
||||
public void printTopology(PrintStream stream, List<Node> leaves,
|
||||
String format) throws BadFormatException, IOException {
|
||||
if (leaves.isEmpty()) {
|
||||
stream.print("No DataNodes");
|
||||
return;
|
||||
}
|
||||
|
||||
// Build a map of rack -> nodes
|
||||
Map<String, TreeSet<String>> tree = new HashMap<>();
|
||||
for(Node dni : leaves) {
|
||||
String location = dni.getNetworkLocation();
|
||||
String name = dni.getName();
|
||||
|
||||
tree.putIfAbsent(location, new TreeSet<>());
|
||||
tree.get(location).add(name);
|
||||
}
|
||||
|
||||
// Sort the racks (and nodes) alphabetically, display in order
|
||||
ArrayList<String> racks = new ArrayList<>(tree.keySet());
|
||||
Collections.sort(racks);
|
||||
|
||||
if (FORMAT_JSON.equals(format)) {
|
||||
printJsonFormat(stream, tree, racks);
|
||||
} else if (FORMAT_TEXT.equals(format)) {
|
||||
printTextFormat(stream, tree, racks);
|
||||
} else {
|
||||
throw new BadFormatException("Bad format: " + format);
|
||||
}
|
||||
}
|
||||
|
||||
private void printJsonFormat(PrintStream stream, Map<String,
|
||||
TreeSet<String>> tree, ArrayList<String> racks) throws IOException {
|
||||
JsonFactory dumpFactory = new JsonFactory();
|
||||
JsonGenerator dumpGenerator = dumpFactory.createGenerator(stream);
|
||||
dumpGenerator.writeStartArray();
|
||||
|
||||
for(String r : racks) {
|
||||
dumpGenerator.writeStartObject();
|
||||
dumpGenerator.writeFieldName(r);
|
||||
TreeSet<String> nodes = tree.get(r);
|
||||
dumpGenerator.writeStartArray();
|
||||
|
||||
for(String n : nodes) {
|
||||
dumpGenerator.writeStartObject();
|
||||
dumpGenerator.writeStringField("ip", n);
|
||||
String hostname = NetUtils.getHostNameOfIP(n);
|
||||
if(hostname != null) {
|
||||
dumpGenerator.writeStringField("hostname", hostname);
|
||||
}
|
||||
dumpGenerator.writeEndObject();
|
||||
}
|
||||
dumpGenerator.writeEndArray();
|
||||
dumpGenerator.writeEndObject();
|
||||
}
|
||||
dumpGenerator.writeEndArray();
|
||||
dumpGenerator.flush();
|
||||
|
||||
if (!dumpGenerator.isClosed()) {
|
||||
dumpGenerator.close();
|
||||
}
|
||||
}
|
||||
|
||||
private void printTextFormat(PrintStream stream, Map<String,
|
||||
TreeSet<String>> tree, ArrayList<String> racks) {
|
||||
for(String r : racks) {
|
||||
stream.println("Rack: " + r);
|
||||
TreeSet<String> nodes = tree.get(r);
|
||||
|
||||
for(String n : nodes) {
|
||||
stream.print(" " + n);
|
||||
String hostname = NetUtils.getHostNameOfIP(n);
|
||||
if(hostname != null) {
|
||||
stream.print(" (" + hostname + ")");
|
||||
}
|
||||
stream.println();
|
||||
}
|
||||
stream.println();
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static String parseAcceptHeader(HttpServletRequest request) {
|
||||
String format = request.getHeader(HttpHeaders.ACCEPT);
|
||||
return format != null && format.contains(FORMAT_JSON) ?
|
||||
FORMAT_JSON : FORMAT_TEXT;
|
||||
}
|
||||
|
||||
public static class BadFormatException extends Exception {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public BadFormatException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -52,6 +52,7 @@
|
|||
<li><a href="jmx">Metrics</a></li>
|
||||
<li><a href="conf">Configuration</a></li>
|
||||
<li><a href="stacks">Process Thread Dump</a></li>
|
||||
<li><a href="topology">Network Topology</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@
|
|||
<li><a href="jmx">Metrics</a></li>
|
||||
<li><a href="conf">Configuration</a></li>
|
||||
<li><a href="stacks">Process Thread Dump</a></li>
|
||||
<li><a href="topology">Network Topology</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,202 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs.server.namenode;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
import org.apache.hadoop.net.StaticMapping;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class TestNetworkTopologyServlet {
|
||||
|
||||
@Test
|
||||
public void testPrintTopologyTextFormat() throws IOException {
|
||||
StaticMapping.resetMap();
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
int dataNodesNum = 0;
|
||||
final ArrayList<String> rackList = new ArrayList<String>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < 2; j++) {
|
||||
rackList.add("/rack" + i);
|
||||
dataNodesNum++;
|
||||
}
|
||||
}
|
||||
|
||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||
.numDataNodes(dataNodesNum)
|
||||
.racks(rackList.toArray(new String[rackList.size()]))
|
||||
.build();
|
||||
cluster.waitActive();
|
||||
|
||||
// get http uri
|
||||
String httpUri = cluster.getHttpUri(0);
|
||||
|
||||
// send http request
|
||||
URL url = new URL(httpUri + "/topology");
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setReadTimeout(20000);
|
||||
conn.setConnectTimeout(20000);
|
||||
conn.connect();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
|
||||
StringBuilder sb =
|
||||
new StringBuilder("-- Network Topology -- \n");
|
||||
sb.append(out);
|
||||
sb.append("\n-- Network Topology -- ");
|
||||
String topology = sb.toString();
|
||||
|
||||
// assert rack info
|
||||
assertTrue(topology.contains("/rack0"));
|
||||
assertTrue(topology.contains("/rack1"));
|
||||
assertTrue(topology.contains("/rack2"));
|
||||
assertTrue(topology.contains("/rack3"));
|
||||
assertTrue(topology.contains("/rack4"));
|
||||
|
||||
// assert node number
|
||||
assertEquals(topology.split("127.0.0.1").length - 1,
|
||||
dataNodesNum);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrintTopologyJsonFormat() throws IOException {
|
||||
StaticMapping.resetMap();
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
int dataNodesNum = 0;
|
||||
final ArrayList<String> rackList = new ArrayList<String>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int j = 0; j < 2; j++) {
|
||||
rackList.add("/rack" + i);
|
||||
dataNodesNum++;
|
||||
}
|
||||
}
|
||||
|
||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||
.numDataNodes(dataNodesNum)
|
||||
.racks(rackList.toArray(new String[rackList.size()]))
|
||||
.build();
|
||||
cluster.waitActive();
|
||||
|
||||
// get http uri
|
||||
String httpUri = cluster.getHttpUri(0);
|
||||
|
||||
// send http request
|
||||
URL url = new URL(httpUri + "/topology");
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setReadTimeout(20000);
|
||||
conn.setConnectTimeout(20000);
|
||||
conn.setRequestProperty("Accept", "application/json");
|
||||
conn.connect();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
|
||||
String topology = out.toString();
|
||||
|
||||
// parse json
|
||||
JsonNode racks = new ObjectMapper().readTree(topology);
|
||||
|
||||
// assert rack number
|
||||
assertEquals(racks.size(), 5);
|
||||
|
||||
// assert node number
|
||||
Iterator<JsonNode> elements = racks.elements();
|
||||
int dataNodesCount = 0;
|
||||
while(elements.hasNext()){
|
||||
JsonNode rack = elements.next();
|
||||
Iterator<Map.Entry<String, JsonNode>> fields = rack.fields();
|
||||
while (fields.hasNext()) {
|
||||
dataNodesCount += fields.next().getValue().size();
|
||||
}
|
||||
}
|
||||
assertEquals(dataNodesCount, dataNodesNum);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrintTopologyNoDatanodesTextFormat() throws IOException {
|
||||
StaticMapping.resetMap();
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||
.numDataNodes(0)
|
||||
.build();
|
||||
cluster.waitActive();
|
||||
|
||||
// get http uri
|
||||
String httpUri = cluster.getHttpUri(0);
|
||||
|
||||
// send http request
|
||||
URL url = new URL(httpUri + "/topology");
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setReadTimeout(20000);
|
||||
conn.setConnectTimeout(20000);
|
||||
conn.connect();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
|
||||
StringBuilder sb =
|
||||
new StringBuilder("-- Network Topology -- \n");
|
||||
sb.append(out);
|
||||
sb.append("\n-- Network Topology -- ");
|
||||
String topology = sb.toString();
|
||||
|
||||
// assert node number
|
||||
assertTrue(topology.contains("No DataNodes"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
|
||||
StaticMapping.resetMap();
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||
.numDataNodes(0)
|
||||
.build();
|
||||
cluster.waitActive();
|
||||
|
||||
// get http uri
|
||||
String httpUri = cluster.getHttpUri(0);
|
||||
|
||||
// send http request
|
||||
URL url = new URL(httpUri + "/topology");
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setReadTimeout(20000);
|
||||
conn.setConnectTimeout(20000);
|
||||
conn.setRequestProperty("Accept", "application/json");
|
||||
conn.connect();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
|
||||
StringBuilder sb =
|
||||
new StringBuilder("-- Network Topology -- \n");
|
||||
sb.append(out);
|
||||
sb.append("\n-- Network Topology -- ");
|
||||
String topology = sb.toString();
|
||||
|
||||
// assert node number
|
||||
assertTrue(topology.contains("No DataNodes"));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue