diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 47baaa19924..b3f81b0b8a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -321,6 +321,8 @@ Release 2.1.0-beta - 2013-07-02 HDFS-4372. Track NameNode startup progress. (cnauroth) HDFS-4373. Add HTTP API for querying NameNode startup progress. (cnauroth) + + HDFS-4374. Display NameNode startup progress in UI. (cnauroth) IMPROVEMENTS diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index 544599e9c7f..67152a01422 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -52,6 +52,12 @@ import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressView; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.Text; @@ -76,7 +82,7 @@ class NamenodeJspHelper { } static String getSafeModeText(FSNamesystem fsn) { - if (!fsn.isInSafeMode()) + if (fsn == null || !fsn.isInSafeMode()) return ""; return "Safe mode is ON. " + fsn.getSafeModeTip() + "
"; } @@ -94,6 +100,10 @@ class NamenodeJspHelper { } static String getInodeLimitText(FSNamesystem fsn) { + if (fsn == null) { + return ""; + } + long inodes = fsn.dir.totalInodes(); long blocks = fsn.getBlocksTotal(); long maxobjects = fsn.getMaxObjects(); @@ -133,15 +143,21 @@ class NamenodeJspHelper { /** Return a table containing version information. */ static String getVersionTable(FSNamesystem fsn) { - return "
" - + "\n \n" + "\n \n" + "\n \n \n \n
Started:" + fsn.getStartTime() - + "
Version:" - + VersionInfo.getVersion() + ", " + VersionInfo.getRevision() - + "
Compiled:" + VersionInfo.getDate() - + " by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch() - + "
Cluster ID:" + fsn.getClusterId() - + "
Block Pool ID:" + fsn.getBlockPoolId() - + "
"; + StringBuilder sb = new StringBuilder(); + sb.append("
"); + if (fsn != null) { + sb.append("\n \n" + "\n \n" + "\n \n \n \n
Started:" + fsn.getStartTime()); + } + sb.append("
Version:"); + sb.append(VersionInfo.getVersion() + ", " + VersionInfo.getRevision()); + sb.append("
Compiled:" + VersionInfo.getDate()); + sb.append(" by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch()); + if (fsn != null) { + sb.append("
Cluster ID:" + fsn.getClusterId()); + sb.append("
Block Pool ID:" + fsn.getBlockPoolId()); + } + sb.append("
"); + return sb.toString(); } /** @@ -149,6 +165,10 @@ class NamenodeJspHelper { * @return a warning if files are corrupt, otherwise return an empty string. */ static String getCorruptFilesWarning(FSNamesystem fsn) { + if (fsn == null) { + return ""; + } + long missingBlocks = fsn.getMissingBlocksCount(); if (missingBlocks > 0) { StringBuilder result = new StringBuilder(); @@ -196,6 +216,9 @@ class NamenodeJspHelper { void generateConfReport(JspWriter out, NameNode nn, HttpServletRequest request) throws IOException { FSNamesystem fsn = nn.getNamesystem(); + if (fsn == null) { + return; + } FSImage fsImage = fsn.getFSImage(); List removedStorageDirs = fsImage.getStorage().getRemovedStorageDirs(); @@ -233,6 +256,9 @@ class NamenodeJspHelper { */ void generateJournalReport(JspWriter out, NameNode nn, HttpServletRequest request) throws IOException { + if (nn.getNamesystem() == null) { + return; + } FSEditLog log = nn.getFSImage().getEditLog(); Preconditions.checkArgument(log != null, "no edit log set in %s", nn); @@ -297,6 +323,9 @@ class NamenodeJspHelper { void generateHealthReport(JspWriter out, NameNode nn, HttpServletRequest request) throws IOException { FSNamesystem fsn = nn.getNamesystem(); + if (fsn == null) { + return; + } final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager(); final List live = new ArrayList(); final List dead = new ArrayList(); @@ -426,6 +455,142 @@ class NamenodeJspHelper { out.print("There are no datanodes in the cluster."); } } + + /** + * Generates the Startup Progress report. + * + * @param out JspWriter to receive output + * @param prog StartupProgress tracking NameNode startup progress + * @throws IOException thrown if there is an I/O error + */ + void generateStartupProgress(JspWriter out, StartupProgress prog) + throws IOException { + StartupProgressView view = prog.createView(); + FormattedWriter fout = new FormattedWriter(out); + fout.println("
"); + fout.println("
Elapsed Time: %s
", + StringUtils.formatTime(view.getElapsedTime())); + fout.println("
Percent Complete: %s
", + StringUtils.formatPercent(view.getPercentComplete(), 2)); + fout.println(""); + fout.println(""); + fout.println(""); + fout.println(""); + fout.println(""); + fout.println(""); + for (Phase phase: view.getPhases()) { + final String timeClass; + Status status = view.getStatus(phase); + if (status == Status.PENDING) { + timeClass = "later"; + } else if (status == Status.RUNNING) { + timeClass = "current"; + } else { + timeClass = "prior"; + } + + fout.println("", timeClass); + printPhase(fout, view, phase); + fout.println(""); + + for (Step step: view.getSteps(phase)) { + fout.println("", timeClass); + printStep(fout, view, phase, step); + fout.println(""); + } + } + fout.println("
PhaseCompletionElapsed Time
"); + fout.println("
"); + } + + /** + * Prints one line of content for a phase in the Startup Progress report. + * + * @param fout FormattedWriter to receive output + * @param view StartupProgressView containing information to print + * @param phase Phase to print + * @throws IOException thrown if there is an I/O error + */ + private void printPhase(FormattedWriter fout, StartupProgressView view, + Phase phase) throws IOException { + StringBuilder phaseLine = new StringBuilder(); + phaseLine.append(phase.getDescription()); + String file = view.getFile(phase); + if (file != null) { + phaseLine.append(" ").append(file); + } + long size = view.getSize(phase); + if (size != Long.MIN_VALUE) { + phaseLine.append(" (").append(StringUtils.byteDesc(size)).append(")"); + } + fout.println("%s", phaseLine.toString()); + fout.println("%s", StringUtils.formatPercent( + view.getPercentComplete(phase), 2)); + fout.println("%s", view.getStatus(phase) == Status.PENDING ? "" : + StringUtils.formatTime(view.getElapsedTime(phase))); + } + + /** + * Prints one line of content for a step in the Startup Progress report. + * + * @param fout FormattedWriter to receive output + * @param view StartupProgressView containing information to print + * @param phase Phase to print + * @param step Step to print + * @throws IOException thrown if there is an I/O error + */ + private void printStep(FormattedWriter fout, StartupProgressView view, + Phase phase, Step step) throws IOException { + StringBuilder stepLine = new StringBuilder(); + String file = step.getFile(); + if (file != null) { + stepLine.append(file); + } + long size = step.getSize(); + if (size != Long.MIN_VALUE) { + stepLine.append(" (").append(StringUtils.byteDesc(size)).append(")"); + } + StepType type = step.getType(); + if (type != null) { + stepLine.append(" ").append(type.getDescription()); + } + + fout.println("%s (%d/%d)", + stepLine.toString(), view.getCount(phase, step), + view.getTotal(phase, step)); + fout.println("%s", StringUtils.formatPercent( + view.getPercentComplete(phase), 2)); + fout.println("%s", view.getStatus(phase) == Status.PENDING ? "" : + StringUtils.formatTime(view.getElapsedTime(phase))); + } + + /** + * JspWriter wrapper that helps simplify printing formatted lines. + */ + private static class FormattedWriter { + private final JspWriter out; + + /** + * Creates a new FormattedWriter that delegates to the given JspWriter. + * + * @param out JspWriter to wrap + */ + FormattedWriter(JspWriter out) { + this.out = out; + } + + /** + * Prints one formatted line, followed by line terminator, using the + * English locale. + * + * @param format String format + * @param args Object... any number of arguments to match format + * @throws IOException thrown if there is an I/O error + */ + void println(String format, Object... args) throws IOException { + out.println(StringUtils.format(format, args)); + } + } } static String getDelegationToken(final NamenodeProtocols nn, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp index a89e0367d5c..1201b95ff75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp @@ -34,17 +34,26 @@ boolean isActive = (nnHAState == HAServiceState.ACTIVE); String namenodeRole = nn.getRole().toString(); String namenodeState = nnHAState.toString(); - String namenodeLabel = nn.getNameNodeAddressHostPortString(); + String namenodeLabel = nn.getRpcServer() != null ? + nn.getNameNodeAddressHostPortString() : null; %> +<% if (namenodeLabel != null) { %> Hadoop <%=namenodeRole%> <%=namenodeLabel%> +<% } else { %> +Hadoop <%=namenodeRole%> +<% } %> +<% if (namenodeLabel != null) { %>

<%=namenodeRole%> '<%=namenodeLabel%>' (<%=namenodeState%>)

+<% } else { %> +

<%=namenodeRole%> (<%=namenodeState%>)

+<% } %> <%= NamenodeJspHelper.getVersionTable(fsn) %>
<% if (isActive) { %> @@ -64,6 +73,8 @@
<% healthjsp.generateConfReport(out, nn, request); %>
+

Startup Progress

+<% healthjsp.generateStartupProgress(out, nn.getStartupProgress()); %> <% out.println(ServletUtil.htmlFooter()); %> diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css index 11c76ebdd36..05b1f546426 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css @@ -155,3 +155,36 @@ span.failed { div.security { width:100%; } + +#startupprogress table, #startupprogress th, #startupprogress td { + border-collapse: collapse; + border-left: 1px solid black; + border-right: 1px solid black; + padding: 5px; + text-align: left; +} + +#startupprogress table { + border: 1px solid black; +} + +.phase { + border-top: 1px solid black; + font-weight: bold; +} + +.current { + font-style: italic; +} + +.later { + color: gray; +} + +.step .startupdesc { + text-indent: 20px; +} + +#startupprogress span { + font-weight: bold; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java index 6f2aada5347..8119d662a6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java @@ -18,22 +18,28 @@ package org.apache.hadoop.hdfs.server.namenode; -import static org.mockito.Mockito.mock; +import static org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase.*; +import static org.mockito.Mockito.*; import java.io.IOException; +import java.util.List; +import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; +import javax.servlet.jsp.JspWriter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; public class TestNameNodeJspHelper { @@ -79,4 +85,42 @@ public class TestNameNodeJspHelper { Assert.assertTrue("security mode doesn't match. Should be OFF", securityOnOff.contains("OFF")); } + + @Test + public void testGenerateStartupProgress() throws Exception { + cluster.waitClusterUp(); + NamenodeJspHelper.HealthJsp jsp = new NamenodeJspHelper.HealthJsp(); + StartupProgress prog = NameNode.getStartupProgress(); + JspWriter out = mock(JspWriter.class); + jsp.generateStartupProgress(out, prog); + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + verify(out, atLeastOnce()).println(captor.capture()); + List contents = captor.getAllValues(); + + // Verify 100% overall completion and all phases mentioned in output. + Assert.assertTrue(containsMatch(contents, "Elapsed Time\\:")); + Assert.assertTrue(containsMatch(contents, "Percent Complete\\:.*?100\\.00%")); + Assert.assertTrue(containsMatch(contents, LOADING_FSIMAGE.getDescription())); + Assert.assertTrue(containsMatch(contents, LOADING_EDITS.getDescription())); + Assert.assertTrue(containsMatch(contents, + SAVING_CHECKPOINT.getDescription())); + Assert.assertTrue(containsMatch(contents, SAFEMODE.getDescription())); + } + + /** + * Checks if the list contains any string that partially matches the regex. + * + * @param list List containing strings to check + * @param regex String regex to check + * @return boolean true if some string in list partially matches regex + */ + private static boolean containsMatch(List list, String regex) { + Pattern pattern = Pattern.compile(regex); + for (String str: list) { + if (pattern.matcher(str).find()) { + return true; + } + } + return false; + } }