HDFS-5561. Merge change r1545791 from trunk.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1545793 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
5aa5bad303
commit
e65e4c8afa
|
@ -137,6 +137,9 @@ Release 2.3.0 - UNRELEASED
|
||||||
|
|
||||||
HDFS-5525. Inline dust templates for new Web UI. (Haohui Mai via jing9)
|
HDFS-5525. Inline dust templates for new Web UI. (Haohui Mai via jing9)
|
||||||
|
|
||||||
|
HDFS-5561. FSNameSystem#getNameJournalStatus() in JMX should return plain
|
||||||
|
text instead of HTML. (Haohui Mai via jing9)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
|
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
|
||||||
|
|
|
@ -150,5 +150,5 @@ interface AsyncLogger {
|
||||||
* Append an HTML-formatted report for this logger's status to the provided
|
* Append an HTML-formatted report for this logger's status to the provided
|
||||||
* StringBuilder. This is displayed on the NN web UI.
|
* StringBuilder. This is displayed on the NN web UI.
|
||||||
*/
|
*/
|
||||||
public void appendHtmlReport(StringBuilder sb);
|
public void appendReport(StringBuilder sb);
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,7 +31,6 @@ import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRe
|
||||||
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
|
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
|
||||||
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
|
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
|
||||||
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
|
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
|
||||||
import org.apache.jasper.compiler.JspUtil;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
|
@ -177,17 +176,16 @@ class AsyncLoggerSet {
|
||||||
* state of the underlying loggers.
|
* state of the underlying loggers.
|
||||||
* @param sb the StringBuilder to append to
|
* @param sb the StringBuilder to append to
|
||||||
*/
|
*/
|
||||||
void appendHtmlReport(StringBuilder sb) {
|
void appendReport(StringBuilder sb) {
|
||||||
sb.append("<table class=\"storage\">");
|
for (int i = 0, len = loggers.size(); i < len; ++i) {
|
||||||
sb.append("<thead><tr><td>JN</td><td>Status</td></tr></thead>\n");
|
AsyncLogger l = loggers.get(i);
|
||||||
for (AsyncLogger l : loggers) {
|
if (i != 0) {
|
||||||
sb.append("<tr>");
|
sb.append(", ");
|
||||||
sb.append("<td>" + JspUtil.escapeXml(l.toString()) + "</td>");
|
}
|
||||||
sb.append("<td>");
|
sb.append(l).append(" (");
|
||||||
l.appendHtmlReport(sb);
|
l.appendReport(sb);
|
||||||
sb.append("</td></tr>\n");
|
sb.append(")");
|
||||||
}
|
}
|
||||||
sb.append("</table>");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -569,7 +569,7 @@ public class IPCLoggerChannel implements AsyncLogger {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized void appendHtmlReport(StringBuilder sb) {
|
public synchronized void appendReport(StringBuilder sb) {
|
||||||
sb.append("Written txid ").append(highestAckedTxId);
|
sb.append("Written txid ").append(highestAckedTxId);
|
||||||
long behind = getLagTxns();
|
long behind = getLagTxns();
|
||||||
if (behind > 0) {
|
if (behind > 0) {
|
||||||
|
|
|
@ -114,10 +114,10 @@ class QuorumOutputStream extends EditLogOutputStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String generateHtmlReport() {
|
public String generateReport() {
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
sb.append("Writing segment beginning at txid " + segmentTxId + "<br/>\n");
|
sb.append("Writing segment beginning at txid " + segmentTxId + ". \n");
|
||||||
loggers.appendHtmlReport(sb);
|
loggers.appendReport(sb);
|
||||||
return sb.toString();
|
return sb.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,6 @@ import static org.apache.hadoop.util.Time.now;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.jasper.compiler.JspUtil;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A generic abstract class to support journaling of edits logs into
|
* A generic abstract class to support journaling of edits logs into
|
||||||
|
@ -141,10 +140,10 @@ public abstract class EditLogOutputStream implements Closeable {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return a short HTML snippet suitable for describing the current
|
* @return a short text snippet suitable for describing the current
|
||||||
* status of the stream
|
* status of the stream
|
||||||
*/
|
*/
|
||||||
public String generateHtmlReport() {
|
public String generateReport() {
|
||||||
return JspUtil.escapeXml(this.toString());
|
return toString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6603,7 +6603,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
||||||
} else if (openForWrite) {
|
} else if (openForWrite) {
|
||||||
EditLogOutputStream elos = jas.getCurrentStream();
|
EditLogOutputStream elos = jas.getCurrentStream();
|
||||||
if (elos != null) {
|
if (elos != null) {
|
||||||
jasMap.put("stream", elos.generateHtmlReport());
|
jasMap.put("stream", elos.generateReport());
|
||||||
} else {
|
} else {
|
||||||
jasMap.put("stream", "not currently writing");
|
jasMap.put("stream", "not currently writing");
|
||||||
}
|
}
|
||||||
|
|
|
@ -341,7 +341,7 @@ class NamenodeJspHelper {
|
||||||
} else if (openForWrite) {
|
} else if (openForWrite) {
|
||||||
EditLogOutputStream elos = jas.getCurrentStream();
|
EditLogOutputStream elos = jas.getCurrentStream();
|
||||||
if (elos != null) {
|
if (elos != null) {
|
||||||
out.println(elos.generateHtmlReport());
|
out.println(elos.generateReport());
|
||||||
} else {
|
} else {
|
||||||
out.println("not currently writing");
|
out.println("not currently writing");
|
||||||
}
|
}
|
||||||
|
|
|
@ -91,7 +91,6 @@ public class TestIPCLoggerChannel {
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testQueueLimiting() throws Exception {
|
public void testQueueLimiting() throws Exception {
|
||||||
|
|
||||||
// Block the underlying fake proxy from actually completing any calls.
|
// Block the underlying fake proxy from actually completing any calls.
|
||||||
DelayAnswer delayer = new DelayAnswer(LOG);
|
DelayAnswer delayer = new DelayAnswer(LOG);
|
||||||
Mockito.doAnswer(delayer).when(mockProxy).journal(
|
Mockito.doAnswer(delayer).when(mockProxy).journal(
|
||||||
|
|
|
@ -25,6 +25,8 @@ import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import junit.framework.Assert;
|
||||||
|
|
||||||
import org.apache.commons.logging.impl.Log4JLogger;
|
import org.apache.commons.logging.impl.Log4JLogger;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hdfs.qjournal.client.AsyncLogger;
|
import org.apache.hadoop.hdfs.qjournal.client.AsyncLogger;
|
||||||
|
@ -124,7 +126,7 @@ public class TestQuorumJournalManagerUnit {
|
||||||
.when(spyLoggers.get(2)).startLogSegment(Mockito.anyLong());
|
.when(spyLoggers.get(2)).startLogSegment(Mockito.anyLong());
|
||||||
qjm.startLogSegment(1);
|
qjm.startLogSegment(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testQuorumOfLoggersFail() throws Exception {
|
public void testQuorumOfLoggersFail() throws Exception {
|
||||||
futureReturns(null).when(spyLoggers.get(0)).startLogSegment(Mockito.anyLong());
|
futureReturns(null).when(spyLoggers.get(0)).startLogSegment(Mockito.anyLong());
|
||||||
|
@ -140,6 +142,16 @@ public class TestQuorumJournalManagerUnit {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testQuorumOutputStreamReport() throws Exception {
|
||||||
|
futureReturns(null).when(spyLoggers.get(0)).startLogSegment(Mockito.anyLong());
|
||||||
|
futureReturns(null).when(spyLoggers.get(1)).startLogSegment(Mockito.anyLong());
|
||||||
|
futureReturns(null).when(spyLoggers.get(2)).startLogSegment(Mockito.anyLong());
|
||||||
|
QuorumOutputStream os = (QuorumOutputStream) qjm.startLogSegment(1);
|
||||||
|
String report = os.generateReport();
|
||||||
|
Assert.assertFalse("Report should be plain text", report.contains("<"));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testWriteEdits() throws Exception {
|
public void testWriteEdits() throws Exception {
|
||||||
EditLogOutputStream stm = createLogSegment();
|
EditLogOutputStream stm = createLogSegment();
|
||||||
|
|
|
@ -26,24 +26,27 @@ import java.io.IOException;
|
||||||
|
|
||||||
import org.apache.hadoop.test.PathUtils;
|
import org.apache.hadoop.test.PathUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.io.IOUtils;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test the EditLogFileOutputStream
|
* Test the EditLogFileOutputStream
|
||||||
*/
|
*/
|
||||||
public class TestEditLogFileOutputStream {
|
public class TestEditLogFileOutputStream {
|
||||||
private final static File TEST_DIR = PathUtils.getTestDir(TestEditLogFileOutputStream.class);
|
private final static File TEST_DIR = PathUtils
|
||||||
private static final File TEST_EDITS =
|
.getTestDir(TestEditLogFileOutputStream.class);
|
||||||
new File(TEST_DIR, "testEditLogFileOutput.log");
|
private static final File TEST_EDITS = new File(TEST_DIR,
|
||||||
final static int MIN_PREALLOCATION_LENGTH =
|
"testEditLogFileOutput.log");
|
||||||
EditLogFileOutputStream.MIN_PREALLOCATION_LENGTH;
|
final static int MIN_PREALLOCATION_LENGTH = EditLogFileOutputStream.MIN_PREALLOCATION_LENGTH;
|
||||||
|
|
||||||
private Configuration conf;
|
private Configuration conf;
|
||||||
|
|
||||||
static {
|
@BeforeClass
|
||||||
|
public static void disableFsync() {
|
||||||
// No need to fsync for the purposes of tests. This makes
|
// No need to fsync for the purposes of tests. This makes
|
||||||
// the tests run much faster.
|
// the tests run much faster.
|
||||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||||
|
@ -52,7 +55,8 @@ public class TestEditLogFileOutputStream {
|
||||||
@Before
|
@Before
|
||||||
@After
|
@After
|
||||||
public void deleteEditsFile() {
|
public void deleteEditsFile() {
|
||||||
if (TEST_EDITS.exists()) TEST_EDITS.delete();
|
if (TEST_EDITS.exists())
|
||||||
|
TEST_EDITS.delete();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
|
@ -66,18 +70,18 @@ public class TestEditLogFileOutputStream {
|
||||||
elos.flushAndSync(true);
|
elos.flushAndSync(true);
|
||||||
assertEquals(expectedLength, elos.getFile().length());
|
assertEquals(expectedLength, elos.getFile().length());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests writing to the EditLogFileOutputStream. Due to preallocation, the
|
* Tests writing to the EditLogFileOutputStream. Due to preallocation, the
|
||||||
* length of the edit log will usually be longer than its valid contents.
|
* length of the edit log will usually be longer than its valid contents.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRawWrites() throws IOException {
|
public void testRawWrites() throws IOException {
|
||||||
EditLogFileOutputStream elos = new EditLogFileOutputStream(conf, TEST_EDITS,
|
EditLogFileOutputStream elos = new EditLogFileOutputStream(conf,
|
||||||
0);
|
TEST_EDITS, 0);
|
||||||
try {
|
try {
|
||||||
byte[] small = new byte[] {1,2,3,4,5,8,7};
|
byte[] small = new byte[] { 1, 2, 3, 4, 5, 8, 7 };
|
||||||
elos.create();
|
elos.create();
|
||||||
// The first (small) write we make extends the file by 1 MB due to
|
// The first (small) write we make extends the file by 1 MB due to
|
||||||
// preallocation.
|
// preallocation.
|
||||||
|
@ -102,7 +106,8 @@ public class TestEditLogFileOutputStream {
|
||||||
}
|
}
|
||||||
flushAndCheckLength(elos, 4 * MIN_PREALLOCATION_LENGTH);
|
flushAndCheckLength(elos, 4 * MIN_PREALLOCATION_LENGTH);
|
||||||
} finally {
|
} finally {
|
||||||
if (elos != null) elos.close();
|
if (elos != null)
|
||||||
|
elos.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,8 +118,8 @@ public class TestEditLogFileOutputStream {
|
||||||
@Test
|
@Test
|
||||||
public void testEditLogFileOutputStreamCloseAbort() throws IOException {
|
public void testEditLogFileOutputStreamCloseAbort() throws IOException {
|
||||||
// abort after a close should just ignore
|
// abort after a close should just ignore
|
||||||
EditLogFileOutputStream editLogStream =
|
EditLogFileOutputStream editLogStream = new EditLogFileOutputStream(conf,
|
||||||
new EditLogFileOutputStream(conf, TEST_EDITS, 0);
|
TEST_EDITS, 0);
|
||||||
editLogStream.close();
|
editLogStream.close();
|
||||||
editLogStream.abort();
|
editLogStream.abort();
|
||||||
}
|
}
|
||||||
|
@ -126,8 +131,8 @@ public class TestEditLogFileOutputStream {
|
||||||
@Test
|
@Test
|
||||||
public void testEditLogFileOutputStreamCloseClose() throws IOException {
|
public void testEditLogFileOutputStreamCloseClose() throws IOException {
|
||||||
// close after a close should result in an IOE
|
// close after a close should result in an IOE
|
||||||
EditLogFileOutputStream editLogStream =
|
EditLogFileOutputStream editLogStream = new EditLogFileOutputStream(conf,
|
||||||
new EditLogFileOutputStream(conf, TEST_EDITS, 0);
|
TEST_EDITS, 0);
|
||||||
editLogStream.close();
|
editLogStream.close();
|
||||||
try {
|
try {
|
||||||
editLogStream.close();
|
editLogStream.close();
|
||||||
|
@ -136,7 +141,7 @@ public class TestEditLogFileOutputStream {
|
||||||
assertTrue(msg, msg.contains("Trying to use aborted output stream"));
|
assertTrue(msg, msg.contains("Trying to use aborted output stream"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests EditLogFileOutputStream doesn't throw NullPointerException on being
|
* Tests EditLogFileOutputStream doesn't throw NullPointerException on being
|
||||||
* abort/abort sequence. See HDFS-2011.
|
* abort/abort sequence. See HDFS-2011.
|
||||||
|
@ -144,9 +149,13 @@ public class TestEditLogFileOutputStream {
|
||||||
@Test
|
@Test
|
||||||
public void testEditLogFileOutputStreamAbortAbort() throws IOException {
|
public void testEditLogFileOutputStreamAbortAbort() throws IOException {
|
||||||
// abort after a close should just ignore
|
// abort after a close should just ignore
|
||||||
EditLogFileOutputStream editLogStream =
|
EditLogFileOutputStream editLogStream = null;
|
||||||
new EditLogFileOutputStream(conf, TEST_EDITS, 0);
|
try {
|
||||||
editLogStream.abort();
|
editLogStream = new EditLogFileOutputStream(conf, TEST_EDITS, 0);
|
||||||
editLogStream.abort();
|
editLogStream.abort();
|
||||||
|
editLogStream.abort();
|
||||||
|
} finally {
|
||||||
|
IOUtils.cleanup(null, editLogStream);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,7 +46,7 @@ public class TestNameNodeMXBean {
|
||||||
*/
|
*/
|
||||||
private static final double DELTA = 0.000001;
|
private static final double DELTA = 0.000001;
|
||||||
|
|
||||||
@SuppressWarnings({ "unchecked", "deprecation" })
|
@SuppressWarnings({ "unchecked" })
|
||||||
@Test
|
@Test
|
||||||
public void testNameNodeMXBeanInfo() throws Exception {
|
public void testNameNodeMXBeanInfo() throws Exception {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
|
@ -152,7 +152,7 @@ public class TestNameNodeMXBean {
|
||||||
assertEquals(0, statusMap.get("failed").size());
|
assertEquals(0, statusMap.get("failed").size());
|
||||||
|
|
||||||
// This will cause the first dir to fail.
|
// This will cause the first dir to fail.
|
||||||
File failedNameDir = new File(nameDirUris.toArray(new URI[0])[0]);
|
File failedNameDir = new File(nameDirUris.iterator().next());
|
||||||
assertEquals(0, FileUtil.chmod(
|
assertEquals(0, FileUtil.chmod(
|
||||||
new File(failedNameDir, "current").getAbsolutePath(), "000"));
|
new File(failedNameDir, "current").getAbsolutePath(), "000"));
|
||||||
cluster.getNameNodeRpc().rollEditLog();
|
cluster.getNameNodeRpc().rollEditLog();
|
||||||
|
|
Loading…
Reference in New Issue