Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-4374. Display NameNode startup progress in UI. Contributed by Chris Nauroth. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1502331 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-07-11 19:20:38 +00:00
parent 19346d8ca6
commit 2409882a47
5 changed files with 267 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -321,6 +321,8 @@ Release 2.1.0-beta - 2013-07-02
HDFS-4372. Track NameNode startup progress. (cnauroth)
HDFS-4373. Add HTTP API for querying NameNode startup progress. (cnauroth)
HDFS-4374. Display NameNode startup progress in UI. (cnauroth)
IMPROVEMENTS

185
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
View File

@ -52,6 +52,12 @@ import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressView;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
@ -76,7 +82,7 @@ class NamenodeJspHelper {
}
static String getSafeModeText(FSNamesystem fsn) {
if (!fsn.isInSafeMode())
if (fsn == null || !fsn.isInSafeMode())
return "";
return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
}
@ -94,6 +100,10 @@ class NamenodeJspHelper {
}
static String getInodeLimitText(FSNamesystem fsn) {
if (fsn == null) {
return "";
}
long inodes = fsn.dir.totalInodes();
long blocks = fsn.getBlocksTotal();
long maxobjects = fsn.getMaxObjects();
@ -133,15 +143,21 @@ class NamenodeJspHelper {
/** Return a table containing version information. */
static String getVersionTable(FSNamesystem fsn) {
return "<div class='dfstable'><table>"
+ "\n <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime()
+ "</td></tr>\n" + "\n <tr><td class='col1'>Version:</td><td>"
+ VersionInfo.getVersion() + ", " + VersionInfo.getRevision()
+ "</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate()
+ " by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch()
+ "</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId()
+ "</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId()
+ "</td></tr>\n</table></div>";
StringBuilder sb = new StringBuilder();
sb.append("<div class='dfstable'><table>");
if (fsn != null) {
sb.append("\n <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime());
}
sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Version:</td><td>");
sb.append(VersionInfo.getVersion() + ", " + VersionInfo.getRevision());
sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate());
sb.append(" by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch());
if (fsn != null) {
sb.append("</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId());
sb.append("</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId());
}
sb.append("</td></tr>\n</table></div>");
return sb.toString();
}
/**
@ -149,6 +165,10 @@ class NamenodeJspHelper {
* @return a warning if files are corrupt, otherwise return an empty string.
*/
static String getCorruptFilesWarning(FSNamesystem fsn) {
if (fsn == null) {
return "";
}
long missingBlocks = fsn.getMissingBlocksCount();
if (missingBlocks > 0) {
StringBuilder result = new StringBuilder();
@ -196,6 +216,9 @@ class NamenodeJspHelper {
void generateConfReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
FSNamesystem fsn = nn.getNamesystem();
if (fsn == null) {
return;
}
FSImage fsImage = fsn.getFSImage();
List<Storage.StorageDirectory> removedStorageDirs
= fsImage.getStorage().getRemovedStorageDirs();
@ -233,6 +256,9 @@ class NamenodeJspHelper {
*/
void generateJournalReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
if (nn.getNamesystem() == null) {
return;
}
FSEditLog log = nn.getFSImage().getEditLog();
Preconditions.checkArgument(log != null, "no edit log set in %s", nn);
@ -297,6 +323,9 @@ class NamenodeJspHelper {
void generateHealthReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
FSNamesystem fsn = nn.getNamesystem();
if (fsn == null) {
return;
}
final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
@ -426,6 +455,142 @@ class NamenodeJspHelper {
out.print("There are no datanodes in the cluster.");
}
}
/**
* Generates the Startup Progress report.
*
* @param out JspWriter to receive output
* @param prog StartupProgress tracking NameNode startup progress
* @throws IOException thrown if there is an I/O error
*/
void generateStartupProgress(JspWriter out, StartupProgress prog)
throws IOException {
StartupProgressView view = prog.createView();
FormattedWriter fout = new FormattedWriter(out);
fout.println("<div id=\"startupprogress\">");
fout.println("<div><span>Elapsed Time:</span> %s</div>",
StringUtils.formatTime(view.getElapsedTime()));
fout.println("<div><span>Percent Complete:</span> %s</div>",
StringUtils.formatPercent(view.getPercentComplete(), 2));
fout.println("<table>");
fout.println("<tr>");
fout.println("<th>Phase</th>");
fout.println("<th>Completion</th>");
fout.println("<th>Elapsed Time</th>");
fout.println("</tr>");
for (Phase phase: view.getPhases()) {
final String timeClass;
Status status = view.getStatus(phase);
if (status == Status.PENDING) {
timeClass = "later";
} else if (status == Status.RUNNING) {
timeClass = "current";
} else {
timeClass = "prior";
}
fout.println("<tr class=\"phase %s\">", timeClass);
printPhase(fout, view, phase);
fout.println("</tr>");
for (Step step: view.getSteps(phase)) {
fout.println("<tr class=\"step %s\">", timeClass);
printStep(fout, view, phase, step);
fout.println("</tr>");
}
}
fout.println("</table>");
fout.println("</div>");
}
/**
* Prints one line of content for a phase in the Startup Progress report.
*
* @param fout FormattedWriter to receive output
* @param view StartupProgressView containing information to print
* @param phase Phase to print
* @throws IOException thrown if there is an I/O error
*/
private void printPhase(FormattedWriter fout, StartupProgressView view,
Phase phase) throws IOException {
StringBuilder phaseLine = new StringBuilder();
phaseLine.append(phase.getDescription());
String file = view.getFile(phase);
if (file != null) {
phaseLine.append(" ").append(file);
}
long size = view.getSize(phase);
if (size != Long.MIN_VALUE) {
phaseLine.append(" (").append(StringUtils.byteDesc(size)).append(")");
}
fout.println("<td class=\"startupdesc\">%s</td>", phaseLine.toString());
fout.println("<td>%s</td>", StringUtils.formatPercent(
view.getPercentComplete(phase), 2));
fout.println("<td>%s</td>", view.getStatus(phase) == Status.PENDING ? "" :
StringUtils.formatTime(view.getElapsedTime(phase)));
}
/**
* Prints one line of content for a step in the Startup Progress report.
*
* @param fout FormattedWriter to receive output
* @param view StartupProgressView containing information to print
* @param phase Phase to print
* @param step Step to print
* @throws IOException thrown if there is an I/O error
*/
private void printStep(FormattedWriter fout, StartupProgressView view,
Phase phase, Step step) throws IOException {
StringBuilder stepLine = new StringBuilder();
String file = step.getFile();
if (file != null) {
stepLine.append(file);
}
long size = step.getSize();
if (size != Long.MIN_VALUE) {
stepLine.append(" (").append(StringUtils.byteDesc(size)).append(")");
}
StepType type = step.getType();
if (type != null) {
stepLine.append(" ").append(type.getDescription());
}
fout.println("<td class=\"startupdesc\">%s (%d/%d)</td>",
stepLine.toString(), view.getCount(phase, step),
view.getTotal(phase, step));
fout.println("<td>%s</td>", StringUtils.formatPercent(
view.getPercentComplete(phase), 2));
fout.println("<td>%s</td>", view.getStatus(phase) == Status.PENDING ? "" :
StringUtils.formatTime(view.getElapsedTime(phase)));
}
/**
* JspWriter wrapper that helps simplify printing formatted lines.
*/
private static class FormattedWriter {
private final JspWriter out;
/**
* Creates a new FormattedWriter that delegates to the given JspWriter.
*
* @param out JspWriter to wrap
*/
FormattedWriter(JspWriter out) {
this.out = out;
}
/**
* Prints one formatted line, followed by line terminator, using the
* English locale.
*
* @param format String format
* @param args Object... any number of arguments to match format
* @throws IOException thrown if there is an I/O error
*/
void println(String format, Object... args) throws IOException {
out.println(StringUtils.format(format, args));
}
}
}
static String getDelegationToken(final NamenodeProtocols nn,

13
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
View File

@ -34,17 +34,26 @@
boolean isActive = (nnHAState == HAServiceState.ACTIVE);
String namenodeRole = nn.getRole().toString();
String namenodeState = nnHAState.toString();
String namenodeLabel = nn.getNameNodeAddressHostPortString();
String namenodeLabel = nn.getRpcServer() != null ?
nn.getNameNodeAddressHostPortString() : null;
%>
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
<% if (namenodeLabel != null) { %>
<title>Hadoop <%=namenodeRole%>&nbsp;<%=namenodeLabel%></title>
<% } else { %>
<title>Hadoop <%=namenodeRole%></title>
<% } %>
</head>
<body>
<% if (namenodeLabel != null) { %>
<h1><%=namenodeRole%> '<%=namenodeLabel%>' (<%=namenodeState%>)</h1>
<% } else { %>
<h1><%=namenodeRole%> (<%=namenodeState%>)</h1>
<% } %>
<%= NamenodeJspHelper.getVersionTable(fsn) %>
<br />
<% if (isActive) { %>
@ -64,6 +73,8 @@
<hr/>
<% healthjsp.generateConfReport(out, nn, request); %>
<hr>
<h3>Startup Progress</h3>
<% healthjsp.generateStartupProgress(out, nn.getStartupProgress()); %>
<%
out.println(ServletUtil.htmlFooter());
%>

33
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css
View File

@ -155,3 +155,36 @@ span.failed {
div.security {
width:100%;
}
#startupprogress table, #startupprogress th, #startupprogress td {
border-collapse: collapse;
border-left: 1px solid black;
border-right: 1px solid black;
padding: 5px;
text-align: left;
}
#startupprogress table {
border: 1px solid black;
}
.phase {
border-top: 1px solid black;
font-weight: bold;
}
.current {
font-style: italic;
}
.later {
color: gray;
}
.step .startupdesc {
text-indent: 20px;
}
#startupprogress span {
font-weight: bold;
}

46
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java
View File

@ -18,22 +18,28 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.mockito.Mockito.mock;
import static org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase.*;
import static org.mockito.Mockito.*;
import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.jsp.JspWriter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
public class TestNameNodeJspHelper {
@ -79,4 +85,42 @@ public class TestNameNodeJspHelper {
Assert.assertTrue("security mode doesn't match. Should be OFF",
securityOnOff.contains("OFF"));
}
@Test
public void testGenerateStartupProgress() throws Exception {
cluster.waitClusterUp();
NamenodeJspHelper.HealthJsp jsp = new NamenodeJspHelper.HealthJsp();
StartupProgress prog = NameNode.getStartupProgress();
JspWriter out = mock(JspWriter.class);
jsp.generateStartupProgress(out, prog);
ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
verify(out, atLeastOnce()).println(captor.capture());
List<String> contents = captor.getAllValues();
// Verify 100% overall completion and all phases mentioned in output.
Assert.assertTrue(containsMatch(contents, "Elapsed Time\\:"));
Assert.assertTrue(containsMatch(contents, "Percent Complete\\:.*?100\\.00%"));
Assert.assertTrue(containsMatch(contents, LOADING_FSIMAGE.getDescription()));
Assert.assertTrue(containsMatch(contents, LOADING_EDITS.getDescription()));
Assert.assertTrue(containsMatch(contents,
SAVING_CHECKPOINT.getDescription()));
Assert.assertTrue(containsMatch(contents, SAFEMODE.getDescription()));
}
/**
* Checks if the list contains any string that partially matches the regex.
*
* @param list List<String> containing strings to check
* @param regex String regex to check
* @return boolean true if some string in list partially matches regex
*/
private static boolean containsMatch(List<String> list, String regex) {
Pattern pattern = Pattern.compile(regex);
for (String str: list) {
if (pattern.matcher(str).find()) {
return true;
}
}
return false;
}
}