HDFS-4374. Display NameNode startup progress in UI. Contributed by Chris Nauroth.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1502331 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth
parent
19346d8ca6
commit
2409882a47
|
|
@ -322,6 +322,8 @@ Release 2.1.0-beta - 2013-07-02
|
|||
|
||||
HDFS-4373. Add HTTP API for querying NameNode startup progress. (cnauroth)
|
||||
|
||||
HDFS-4374. Display NameNode startup progress in UI. (cnauroth)
|
||||
|
||||
IMPROVEMENTS
|
||||
|
||||
HDFS-4461. DirectoryScanner: volume path prefix takes up memory for every
|
||||
|
|
|
|||
|
|
@ -52,6 +52,12 @@ import org.apache.hadoop.hdfs.server.common.JspHelper;
|
|||
import org.apache.hadoop.hdfs.server.common.Storage;
|
||||
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressView;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
|
||||
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
|
@ -76,7 +82,7 @@ class NamenodeJspHelper {
|
|||
}
|
||||
|
||||
static String getSafeModeText(FSNamesystem fsn) {
|
||||
if (!fsn.isInSafeMode())
|
||||
if (fsn == null || !fsn.isInSafeMode())
|
||||
return "";
|
||||
return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
|
||||
}
|
||||
|
|
@ -94,6 +100,10 @@ class NamenodeJspHelper {
|
|||
}
|
||||
|
||||
static String getInodeLimitText(FSNamesystem fsn) {
|
||||
if (fsn == null) {
|
||||
return "";
|
||||
}
|
||||
|
||||
long inodes = fsn.dir.totalInodes();
|
||||
long blocks = fsn.getBlocksTotal();
|
||||
long maxobjects = fsn.getMaxObjects();
|
||||
|
|
@ -133,15 +143,21 @@ class NamenodeJspHelper {
|
|||
|
||||
/** Return a table containing version information. */
|
||||
static String getVersionTable(FSNamesystem fsn) {
|
||||
return "<div class='dfstable'><table>"
|
||||
+ "\n <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime()
|
||||
+ "</td></tr>\n" + "\n <tr><td class='col1'>Version:</td><td>"
|
||||
+ VersionInfo.getVersion() + ", " + VersionInfo.getRevision()
|
||||
+ "</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate()
|
||||
+ " by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch()
|
||||
+ "</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId()
|
||||
+ "</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId()
|
||||
+ "</td></tr>\n</table></div>";
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("<div class='dfstable'><table>");
|
||||
if (fsn != null) {
|
||||
sb.append("\n <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime());
|
||||
}
|
||||
sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Version:</td><td>");
|
||||
sb.append(VersionInfo.getVersion() + ", " + VersionInfo.getRevision());
|
||||
sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate());
|
||||
sb.append(" by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch());
|
||||
if (fsn != null) {
|
||||
sb.append("</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId());
|
||||
sb.append("</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId());
|
||||
}
|
||||
sb.append("</td></tr>\n</table></div>");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -149,6 +165,10 @@ class NamenodeJspHelper {
|
|||
* @return a warning if files are corrupt, otherwise return an empty string.
|
||||
*/
|
||||
static String getCorruptFilesWarning(FSNamesystem fsn) {
|
||||
if (fsn == null) {
|
||||
return "";
|
||||
}
|
||||
|
||||
long missingBlocks = fsn.getMissingBlocksCount();
|
||||
if (missingBlocks > 0) {
|
||||
StringBuilder result = new StringBuilder();
|
||||
|
|
@ -196,6 +216,9 @@ class NamenodeJspHelper {
|
|||
void generateConfReport(JspWriter out, NameNode nn,
|
||||
HttpServletRequest request) throws IOException {
|
||||
FSNamesystem fsn = nn.getNamesystem();
|
||||
if (fsn == null) {
|
||||
return;
|
||||
}
|
||||
FSImage fsImage = fsn.getFSImage();
|
||||
List<Storage.StorageDirectory> removedStorageDirs
|
||||
= fsImage.getStorage().getRemovedStorageDirs();
|
||||
|
|
@ -233,6 +256,9 @@ class NamenodeJspHelper {
|
|||
*/
|
||||
void generateJournalReport(JspWriter out, NameNode nn,
|
||||
HttpServletRequest request) throws IOException {
|
||||
if (nn.getNamesystem() == null) {
|
||||
return;
|
||||
}
|
||||
FSEditLog log = nn.getFSImage().getEditLog();
|
||||
Preconditions.checkArgument(log != null, "no edit log set in %s", nn);
|
||||
|
||||
|
|
@ -297,6 +323,9 @@ class NamenodeJspHelper {
|
|||
void generateHealthReport(JspWriter out, NameNode nn,
|
||||
HttpServletRequest request) throws IOException {
|
||||
FSNamesystem fsn = nn.getNamesystem();
|
||||
if (fsn == null) {
|
||||
return;
|
||||
}
|
||||
final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
|
||||
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
|
||||
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
|
||||
|
|
@ -426,6 +455,142 @@ class NamenodeJspHelper {
|
|||
out.print("There are no datanodes in the cluster.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the Startup Progress report.
|
||||
*
|
||||
* @param out JspWriter to receive output
|
||||
* @param prog StartupProgress tracking NameNode startup progress
|
||||
* @throws IOException thrown if there is an I/O error
|
||||
*/
|
||||
void generateStartupProgress(JspWriter out, StartupProgress prog)
|
||||
throws IOException {
|
||||
StartupProgressView view = prog.createView();
|
||||
FormattedWriter fout = new FormattedWriter(out);
|
||||
fout.println("<div id=\"startupprogress\">");
|
||||
fout.println("<div><span>Elapsed Time:</span> %s</div>",
|
||||
StringUtils.formatTime(view.getElapsedTime()));
|
||||
fout.println("<div><span>Percent Complete:</span> %s</div>",
|
||||
StringUtils.formatPercent(view.getPercentComplete(), 2));
|
||||
fout.println("<table>");
|
||||
fout.println("<tr>");
|
||||
fout.println("<th>Phase</th>");
|
||||
fout.println("<th>Completion</th>");
|
||||
fout.println("<th>Elapsed Time</th>");
|
||||
fout.println("</tr>");
|
||||
for (Phase phase: view.getPhases()) {
|
||||
final String timeClass;
|
||||
Status status = view.getStatus(phase);
|
||||
if (status == Status.PENDING) {
|
||||
timeClass = "later";
|
||||
} else if (status == Status.RUNNING) {
|
||||
timeClass = "current";
|
||||
} else {
|
||||
timeClass = "prior";
|
||||
}
|
||||
|
||||
fout.println("<tr class=\"phase %s\">", timeClass);
|
||||
printPhase(fout, view, phase);
|
||||
fout.println("</tr>");
|
||||
|
||||
for (Step step: view.getSteps(phase)) {
|
||||
fout.println("<tr class=\"step %s\">", timeClass);
|
||||
printStep(fout, view, phase, step);
|
||||
fout.println("</tr>");
|
||||
}
|
||||
}
|
||||
fout.println("</table>");
|
||||
fout.println("</div>");
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints one line of content for a phase in the Startup Progress report.
|
||||
*
|
||||
* @param fout FormattedWriter to receive output
|
||||
* @param view StartupProgressView containing information to print
|
||||
* @param phase Phase to print
|
||||
* @throws IOException thrown if there is an I/O error
|
||||
*/
|
||||
private void printPhase(FormattedWriter fout, StartupProgressView view,
|
||||
Phase phase) throws IOException {
|
||||
StringBuilder phaseLine = new StringBuilder();
|
||||
phaseLine.append(phase.getDescription());
|
||||
String file = view.getFile(phase);
|
||||
if (file != null) {
|
||||
phaseLine.append(" ").append(file);
|
||||
}
|
||||
long size = view.getSize(phase);
|
||||
if (size != Long.MIN_VALUE) {
|
||||
phaseLine.append(" (").append(StringUtils.byteDesc(size)).append(")");
|
||||
}
|
||||
fout.println("<td class=\"startupdesc\">%s</td>", phaseLine.toString());
|
||||
fout.println("<td>%s</td>", StringUtils.formatPercent(
|
||||
view.getPercentComplete(phase), 2));
|
||||
fout.println("<td>%s</td>", view.getStatus(phase) == Status.PENDING ? "" :
|
||||
StringUtils.formatTime(view.getElapsedTime(phase)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints one line of content for a step in the Startup Progress report.
|
||||
*
|
||||
* @param fout FormattedWriter to receive output
|
||||
* @param view StartupProgressView containing information to print
|
||||
* @param phase Phase to print
|
||||
* @param step Step to print
|
||||
* @throws IOException thrown if there is an I/O error
|
||||
*/
|
||||
private void printStep(FormattedWriter fout, StartupProgressView view,
|
||||
Phase phase, Step step) throws IOException {
|
||||
StringBuilder stepLine = new StringBuilder();
|
||||
String file = step.getFile();
|
||||
if (file != null) {
|
||||
stepLine.append(file);
|
||||
}
|
||||
long size = step.getSize();
|
||||
if (size != Long.MIN_VALUE) {
|
||||
stepLine.append(" (").append(StringUtils.byteDesc(size)).append(")");
|
||||
}
|
||||
StepType type = step.getType();
|
||||
if (type != null) {
|
||||
stepLine.append(" ").append(type.getDescription());
|
||||
}
|
||||
|
||||
fout.println("<td class=\"startupdesc\">%s (%d/%d)</td>",
|
||||
stepLine.toString(), view.getCount(phase, step),
|
||||
view.getTotal(phase, step));
|
||||
fout.println("<td>%s</td>", StringUtils.formatPercent(
|
||||
view.getPercentComplete(phase), 2));
|
||||
fout.println("<td>%s</td>", view.getStatus(phase) == Status.PENDING ? "" :
|
||||
StringUtils.formatTime(view.getElapsedTime(phase)));
|
||||
}
|
||||
|
||||
/**
|
||||
* JspWriter wrapper that helps simplify printing formatted lines.
|
||||
*/
|
||||
private static class FormattedWriter {
|
||||
private final JspWriter out;
|
||||
|
||||
/**
|
||||
* Creates a new FormattedWriter that delegates to the given JspWriter.
|
||||
*
|
||||
* @param out JspWriter to wrap
|
||||
*/
|
||||
FormattedWriter(JspWriter out) {
|
||||
this.out = out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints one formatted line, followed by line terminator, using the
|
||||
* English locale.
|
||||
*
|
||||
* @param format String format
|
||||
* @param args Object... any number of arguments to match format
|
||||
* @throws IOException thrown if there is an I/O error
|
||||
*/
|
||||
void println(String format, Object... args) throws IOException {
|
||||
out.println(StringUtils.format(format, args));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static String getDelegationToken(final NamenodeProtocols nn,
|
||||
|
|
|
|||
|
|
@ -34,17 +34,26 @@
|
|||
boolean isActive = (nnHAState == HAServiceState.ACTIVE);
|
||||
String namenodeRole = nn.getRole().toString();
|
||||
String namenodeState = nnHAState.toString();
|
||||
String namenodeLabel = nn.getNameNodeAddressHostPortString();
|
||||
String namenodeLabel = nn.getRpcServer() != null ?
|
||||
nn.getNameNodeAddressHostPortString() : null;
|
||||
%>
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
|
||||
<% if (namenodeLabel != null) { %>
|
||||
<title>Hadoop <%=namenodeRole%> <%=namenodeLabel%></title>
|
||||
<% } else { %>
|
||||
<title>Hadoop <%=namenodeRole%></title>
|
||||
<% } %>
|
||||
</head>
|
||||
<body>
|
||||
<% if (namenodeLabel != null) { %>
|
||||
<h1><%=namenodeRole%> '<%=namenodeLabel%>' (<%=namenodeState%>)</h1>
|
||||
<% } else { %>
|
||||
<h1><%=namenodeRole%> (<%=namenodeState%>)</h1>
|
||||
<% } %>
|
||||
<%= NamenodeJspHelper.getVersionTable(fsn) %>
|
||||
<br />
|
||||
<% if (isActive) { %>
|
||||
|
|
@ -64,6 +73,8 @@
|
|||
<hr/>
|
||||
<% healthjsp.generateConfReport(out, nn, request); %>
|
||||
<hr>
|
||||
<h3>Startup Progress</h3>
|
||||
<% healthjsp.generateStartupProgress(out, nn.getStartupProgress()); %>
|
||||
<%
|
||||
out.println(ServletUtil.htmlFooter());
|
||||
%>
|
||||
|
|
|
|||
|
|
@ -155,3 +155,36 @@ span.failed {
|
|||
div.security {
|
||||
width:100%;
|
||||
}
|
||||
|
||||
#startupprogress table, #startupprogress th, #startupprogress td {
|
||||
border-collapse: collapse;
|
||||
border-left: 1px solid black;
|
||||
border-right: 1px solid black;
|
||||
padding: 5px;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
#startupprogress table {
|
||||
border: 1px solid black;
|
||||
}
|
||||
|
||||
.phase {
|
||||
border-top: 1px solid black;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.current {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.later {
|
||||
color: gray;
|
||||
}
|
||||
|
||||
.step .startupdesc {
|
||||
text-indent: 20px;
|
||||
}
|
||||
|
||||
#startupprogress span {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,22 +18,28 @@
|
|||
package org.apache.hadoop.hdfs.server.namenode;
|
||||
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.jsp.JspWriter;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
|
||||
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
public class TestNameNodeJspHelper {
|
||||
|
||||
|
|
@ -79,4 +85,42 @@ public class TestNameNodeJspHelper {
|
|||
Assert.assertTrue("security mode doesn't match. Should be OFF",
|
||||
securityOnOff.contains("OFF"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGenerateStartupProgress() throws Exception {
|
||||
cluster.waitClusterUp();
|
||||
NamenodeJspHelper.HealthJsp jsp = new NamenodeJspHelper.HealthJsp();
|
||||
StartupProgress prog = NameNode.getStartupProgress();
|
||||
JspWriter out = mock(JspWriter.class);
|
||||
jsp.generateStartupProgress(out, prog);
|
||||
ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
|
||||
verify(out, atLeastOnce()).println(captor.capture());
|
||||
List<String> contents = captor.getAllValues();
|
||||
|
||||
// Verify 100% overall completion and all phases mentioned in output.
|
||||
Assert.assertTrue(containsMatch(contents, "Elapsed Time\\:"));
|
||||
Assert.assertTrue(containsMatch(contents, "Percent Complete\\:.*?100\\.00%"));
|
||||
Assert.assertTrue(containsMatch(contents, LOADING_FSIMAGE.getDescription()));
|
||||
Assert.assertTrue(containsMatch(contents, LOADING_EDITS.getDescription()));
|
||||
Assert.assertTrue(containsMatch(contents,
|
||||
SAVING_CHECKPOINT.getDescription()));
|
||||
Assert.assertTrue(containsMatch(contents, SAFEMODE.getDescription()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the list contains any string that partially matches the regex.
|
||||
*
|
||||
* @param list List<String> containing strings to check
|
||||
* @param regex String regex to check
|
||||
* @return boolean true if some string in list partially matches regex
|
||||
*/
|
||||
private static boolean containsMatch(List<String> list, String regex) {
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
for (String str: list) {
|
||||
if (pattern.matcher(str).find()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue