svn merge -c 1326762 from trunk for HDFS-3279. Move the FSEditLog constructor with @VisibleForTesting to TestEditLog.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1326818 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
fdffec2f25
commit
630e8b0669
|
@ -251,6 +251,9 @@ Release 2.0.0 - UNRELEASED
|
||||||
|
|
||||||
HDFS-2708. Stats for the # of blocks per DN. (atm)
|
HDFS-2708. Stats for the # of blocks per DN. (atm)
|
||||||
|
|
||||||
|
HDFS-3279. Move the FSEditLog constructor with @VisibleForTesting to
|
||||||
|
TestEditLog. (Arpit Gupta via szetszwo)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-2477. Optimize computing the diff between a block report and the
|
HDFS-2477. Optimize computing the diff between a block report and the
|
||||||
|
|
|
@ -25,7 +25,6 @@ import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.lang.reflect.Constructor;
|
import java.lang.reflect.Constructor;
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
@ -157,20 +156,6 @@ public class FSEditLog {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Construct FSEditLog with default configuration, taking editDirs from NNStorage
|
|
||||||
*
|
|
||||||
* @param storage Storage object used by namenode
|
|
||||||
*/
|
|
||||||
@VisibleForTesting
|
|
||||||
FSEditLog(NNStorage storage) throws IOException {
|
|
||||||
Configuration conf = new Configuration();
|
|
||||||
// Make sure the edits dirs are set in the provided configuration object.
|
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
|
|
||||||
StringUtils.join(storage.getEditsDirectories(), ","));
|
|
||||||
init(conf, storage, FSNamesystem.getNamespaceEditsDirs(conf));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor for FSEditLog. Underlying journals are constructed, but
|
* Constructor for FSEditLog. Underlying journals are constructed, but
|
||||||
* no streams are opened until open() is called.
|
* no streams are opened until open() is called.
|
||||||
|
|
|
@ -143,6 +143,20 @@ public class TestEditLog extends TestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct FSEditLog with default configuration, taking editDirs from NNStorage
|
||||||
|
*
|
||||||
|
* @param storage Storage object used by namenode
|
||||||
|
*/
|
||||||
|
private static FSEditLog getFSEditLog(NNStorage storage) throws IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
// Make sure the edits dirs are set in the provided configuration object.
|
||||||
|
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
|
||||||
|
StringUtils.join(",", storage.getEditsDirectories()));
|
||||||
|
FSEditLog log = new FSEditLog(conf, storage, FSNamesystem.getNamespaceEditsDirs(conf));
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test case for an empty edit log from a prior version of Hadoop.
|
* Test case for an empty edit log from a prior version of Hadoop.
|
||||||
*/
|
*/
|
||||||
|
@ -864,7 +878,7 @@ public class TestEditLog extends TestCase {
|
||||||
storage = mockStorageWithEdits(
|
storage = mockStorageWithEdits(
|
||||||
"[1,100]|[101,200]|[201,]",
|
"[1,100]|[101,200]|[201,]",
|
||||||
"[1,100]|[101,200]|[201,]");
|
"[1,100]|[101,200]|[201,]");
|
||||||
log = new FSEditLog(storage);
|
log = getFSEditLog(storage);
|
||||||
log.initJournalsForWrite();
|
log.initJournalsForWrite();
|
||||||
assertEquals("[[1,100], [101,200]]",
|
assertEquals("[[1,100], [101,200]]",
|
||||||
log.getEditLogManifest(1).toString());
|
log.getEditLogManifest(1).toString());
|
||||||
|
@ -876,7 +890,7 @@ public class TestEditLog extends TestCase {
|
||||||
storage = mockStorageWithEdits(
|
storage = mockStorageWithEdits(
|
||||||
"[1,100]|[101,200]",
|
"[1,100]|[101,200]",
|
||||||
"[1,100]|[201,300]|[301,400]"); // nothing starting at 101
|
"[1,100]|[201,300]|[301,400]"); // nothing starting at 101
|
||||||
log = new FSEditLog(storage);
|
log = getFSEditLog(storage);
|
||||||
log.initJournalsForWrite();
|
log.initJournalsForWrite();
|
||||||
assertEquals("[[1,100], [101,200], [201,300], [301,400]]",
|
assertEquals("[[1,100], [101,200], [201,300], [301,400]]",
|
||||||
log.getEditLogManifest(1).toString());
|
log.getEditLogManifest(1).toString());
|
||||||
|
@ -886,7 +900,7 @@ public class TestEditLog extends TestCase {
|
||||||
storage = mockStorageWithEdits(
|
storage = mockStorageWithEdits(
|
||||||
"[1,100]|[301,400]", // gap from 101 to 300
|
"[1,100]|[301,400]", // gap from 101 to 300
|
||||||
"[301,400]|[401,500]");
|
"[301,400]|[401,500]");
|
||||||
log = new FSEditLog(storage);
|
log = getFSEditLog(storage);
|
||||||
log.initJournalsForWrite();
|
log.initJournalsForWrite();
|
||||||
assertEquals("[[301,400], [401,500]]",
|
assertEquals("[[301,400], [401,500]]",
|
||||||
log.getEditLogManifest(1).toString());
|
log.getEditLogManifest(1).toString());
|
||||||
|
@ -896,7 +910,7 @@ public class TestEditLog extends TestCase {
|
||||||
storage = mockStorageWithEdits(
|
storage = mockStorageWithEdits(
|
||||||
"[1,100]|[101,150]", // short log at 101
|
"[1,100]|[101,150]", // short log at 101
|
||||||
"[1,50]|[101,200]"); // short log at 1
|
"[1,50]|[101,200]"); // short log at 1
|
||||||
log = new FSEditLog(storage);
|
log = getFSEditLog(storage);
|
||||||
log.initJournalsForWrite();
|
log.initJournalsForWrite();
|
||||||
assertEquals("[[1,100], [101,200]]",
|
assertEquals("[[1,100], [101,200]]",
|
||||||
log.getEditLogManifest(1).toString());
|
log.getEditLogManifest(1).toString());
|
||||||
|
@ -909,7 +923,7 @@ public class TestEditLog extends TestCase {
|
||||||
storage = mockStorageWithEdits(
|
storage = mockStorageWithEdits(
|
||||||
"[1,100]|[101,]",
|
"[1,100]|[101,]",
|
||||||
"[1,100]|[101,200]");
|
"[1,100]|[101,200]");
|
||||||
log = new FSEditLog(storage);
|
log = getFSEditLog(storage);
|
||||||
log.initJournalsForWrite();
|
log.initJournalsForWrite();
|
||||||
assertEquals("[[1,100], [101,200]]",
|
assertEquals("[[1,100], [101,200]]",
|
||||||
log.getEditLogManifest(1).toString());
|
log.getEditLogManifest(1).toString());
|
||||||
|
@ -999,7 +1013,7 @@ public class TestEditLog extends TestCase {
|
||||||
Collections.<URI>emptyList(),
|
Collections.<URI>emptyList(),
|
||||||
editUris);
|
editUris);
|
||||||
storage.format(new NamespaceInfo());
|
storage.format(new NamespaceInfo());
|
||||||
FSEditLog editlog = new FSEditLog(storage);
|
FSEditLog editlog = getFSEditLog(storage);
|
||||||
// open the edit log and add two transactions
|
// open the edit log and add two transactions
|
||||||
// logGenerationStamp is used, simply because it doesn't
|
// logGenerationStamp is used, simply because it doesn't
|
||||||
// require complex arguments.
|
// require complex arguments.
|
||||||
|
@ -1081,7 +1095,7 @@ public class TestEditLog extends TestCase {
|
||||||
new AbortSpec(9, 0),
|
new AbortSpec(9, 0),
|
||||||
new AbortSpec(10, 1));
|
new AbortSpec(10, 1));
|
||||||
long totaltxnread = 0;
|
long totaltxnread = 0;
|
||||||
FSEditLog editlog = new FSEditLog(storage);
|
FSEditLog editlog = getFSEditLog(storage);
|
||||||
editlog.initJournalsForWrite();
|
editlog.initJournalsForWrite();
|
||||||
long startTxId = 1;
|
long startTxId = 1;
|
||||||
Iterable<EditLogInputStream> editStreams = editlog.selectInputStreams(startTxId,
|
Iterable<EditLogInputStream> editStreams = editlog.selectInputStreams(startTxId,
|
||||||
|
@ -1131,7 +1145,7 @@ public class TestEditLog extends TestCase {
|
||||||
assertEquals(1, files.length);
|
assertEquals(1, files.length);
|
||||||
assertTrue(files[0].delete());
|
assertTrue(files[0].delete());
|
||||||
|
|
||||||
FSEditLog editlog = new FSEditLog(storage);
|
FSEditLog editlog = getFSEditLog(storage);
|
||||||
editlog.initJournalsForWrite();
|
editlog.initJournalsForWrite();
|
||||||
long startTxId = 1;
|
long startTxId = 1;
|
||||||
try {
|
try {
|
||||||
|
|
Loading…
Reference in New Issue