Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-3853. Port MiniDFSCluster enableManagedDfsDirsRedundancy. Contributed by Colin Patrick McCabe 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1377545 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-08-27 00:11:39 +00:00
parent 6cac9aa7f9
commit 804c45867a
4 changed files with 48 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -233,6 +233,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3844. Add @Override and remove {@inheritdoc} and unnecessary HDFS-3844. Add @Override and remove {@inheritdoc} and unnecessary
imports. (Jing Zhao via suresh) imports. (Jing Zhao via suresh)
HDFS-3853. Port MiniDFSCluster enableManagedDfsDirsRedundancy
option to branch-2. (Colin Patrick McCabe via eli)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log HDFS-2982. Startup performance suffers when there are many edit log

41
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -134,6 +134,7 @@ public class MiniDFSCluster {
private boolean format = true; private boolean format = true;
private boolean manageNameDfsDirs = true; private boolean manageNameDfsDirs = true;
private boolean manageNameDfsSharedDirs = true; private boolean manageNameDfsSharedDirs = true;
private boolean enableManagedDfsDirsRedundancy = true;
private boolean manageDataDfsDirs = true; private boolean manageDataDfsDirs = true;
private StartupOption option = null; private StartupOption option = null;
private String[] racks = null; private String[] racks = null;
@ -198,6 +199,14 @@ public class MiniDFSCluster {
return this; return this;
} }
/**
* Default: true
*/
public Builder enableManagedDfsDirsRedundancy(boolean val) {
this.enableManagedDfsDirsRedundancy = val;
return this;
}
/** /**
* Default: true * Default: true
*/ */
@ -316,6 +325,7 @@ public class MiniDFSCluster {
builder.format, builder.format,
builder.manageNameDfsDirs, builder.manageNameDfsDirs,
builder.manageNameDfsSharedDirs, builder.manageNameDfsSharedDirs,
builder.enableManagedDfsDirsRedundancy,
builder.manageDataDfsDirs, builder.manageDataDfsDirs,
builder.option, builder.option,
builder.racks, builder.racks,
@ -558,7 +568,8 @@ public class MiniDFSCluster {
long[] simulatedCapacities) throws IOException { long[] simulatedCapacities) throws IOException {
this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster
initMiniDFSCluster(conf, numDataNodes, format, initMiniDFSCluster(conf, numDataNodes, format,
manageNameDfsDirs, true, manageDataDfsDirs, operation, racks, hosts, manageNameDfsDirs, true, true, manageDataDfsDirs,
operation, racks, hosts,
simulatedCapacities, null, true, false, simulatedCapacities, null, true, false,
MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false); MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false);
} }
@ -566,8 +577,8 @@ public class MiniDFSCluster {
private void initMiniDFSCluster( private void initMiniDFSCluster(
Configuration conf, Configuration conf,
int numDataNodes, boolean format, boolean manageNameDfsDirs, int numDataNodes, boolean format, boolean manageNameDfsDirs,
boolean manageNameDfsSharedDirs, boolean manageDataDfsDirs, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy,
StartupOption operation, String[] racks, boolean manageDataDfsDirs, StartupOption operation, String[] racks,
String[] hosts, long[] simulatedCapacities, String clusterId, String[] hosts, long[] simulatedCapacities, String clusterId,
boolean waitSafeMode, boolean setupHostsFile, boolean waitSafeMode, boolean setupHostsFile,
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
@ -613,6 +624,7 @@ public class MiniDFSCluster {
federation = nnTopology.isFederated(); federation = nnTopology.isFederated();
createNameNodesAndSetConf( createNameNodesAndSetConf(
nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs, nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
enableManagedDfsDirsRedundancy,
format, operation, clusterId, conf); format, operation, clusterId, conf);
if (format) { if (format) {
@ -635,7 +647,8 @@ public class MiniDFSCluster {
private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology,
boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs,
boolean format, StartupOption operation, String clusterId, boolean enableManagedDfsDirsRedundancy, boolean format,
StartupOption operation, String clusterId,
Configuration conf) throws IOException { Configuration conf) throws IOException {
Preconditions.checkArgument(nnTopology.countNameNodes() > 0, Preconditions.checkArgument(nnTopology.countNameNodes() > 0,
"empty NN topology: no namenodes specified!"); "empty NN topology: no namenodes specified!");
@ -692,7 +705,7 @@ public class MiniDFSCluster {
int nnCounterForFormat = nnCounter; int nnCounterForFormat = nnCounter;
for (NNConf nn : nameservice.getNNs()) { for (NNConf nn : nameservice.getNNs()) {
initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs,
nnCounterForFormat); enableManagedDfsDirsRedundancy, nnCounterForFormat);
Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf); Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
if (format) { if (format) {
for (URI nameDirUri : namespaceDirs) { for (URI nameDirUri : namespaceDirs) {
@ -723,7 +736,8 @@ public class MiniDFSCluster {
// Start all Namenodes // Start all Namenodes
for (NNConf nn : nameservice.getNNs()) { for (NNConf nn : nameservice.getNNs()) {
initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs, nnCounter); initNameNodeConf(conf, nsId, nn.getNnId(), manageNameDfsDirs,
enableManagedDfsDirsRedundancy, nnCounter);
createNameNode(nnCounter++, conf, numDataNodes, false, operation, createNameNode(nnCounter++, conf, numDataNodes, false, operation,
clusterId, nsId, nn.getNnId()); clusterId, nsId, nn.getNnId());
} }
@ -748,8 +762,8 @@ public class MiniDFSCluster {
private void initNameNodeConf(Configuration conf, private void initNameNodeConf(Configuration conf,
String nameserviceId, String nnId, String nameserviceId, String nnId,
boolean manageNameDfsDirs, int nnIndex) boolean manageNameDfsDirs, boolean enableManagedDfsDirsRedundancy,
throws IOException { int nnIndex) throws IOException {
if (nameserviceId != null) { if (nameserviceId != null) {
conf.set(DFS_NAMESERVICE_ID, nameserviceId); conf.set(DFS_NAMESERVICE_ID, nameserviceId);
} }
@ -758,12 +772,21 @@ public class MiniDFSCluster {
} }
if (manageNameDfsDirs) { if (manageNameDfsDirs) {
if (enableManagedDfsDirsRedundancy) {
conf.set(DFS_NAMENODE_NAME_DIR_KEY, conf.set(DFS_NAMENODE_NAME_DIR_KEY,
fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1)))+","+ fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1)))+","+
fileAsURI(new File(base_dir, "name" + (2*nnIndex + 2)))); fileAsURI(new File(base_dir, "name" + (2*nnIndex + 2))));
conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY, conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY,
fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1)))+","+ fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1)))+","+
fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 2)))); fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 2))));
} else {
conf.set(DFS_NAMENODE_NAME_DIR_KEY,
fileAsURI(new File(base_dir, "name" + (2*nnIndex + 1))).
toString());
conf.set(DFS_NAMENODE_CHECKPOINT_DIR_KEY,
fileAsURI(new File(base_dir, "namesecondary" + (2*nnIndex + 1))).
toString());
}
} }
} }
@ -2188,7 +2211,7 @@ public class MiniDFSCluster {
String nnId = null; String nnId = null;
initNameNodeAddress(conf, nameserviceId, initNameNodeAddress(conf, nameserviceId,
new NNConf(nnId).setIpcPort(namenodePort)); new NNConf(nnId).setIpcPort(namenodePort));
initNameNodeConf(conf, nameserviceId, nnId, true, nnIndex); initNameNodeConf(conf, nameserviceId, nnId, true, true, nnIndex);
createNameNode(nnIndex, conf, numDataNodes, true, null, null, createNameNode(nnIndex, conf, numDataNodes, true, null, null,
nameserviceId, nnId); nameserviceId, nnId);

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
View File

@ -70,7 +70,7 @@ public class TestFSEditLogLoader {
MiniDFSCluster cluster = null; MiniDFSCluster cluster = null;
FileSystem fileSys = null; FileSystem fileSys = null;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES)
.build(); .enableManagedDfsDirsRedundancy(false).build();
cluster.waitActive(); cluster.waitActive();
fileSys = cluster.getFileSystem(); fileSys = cluster.getFileSystem();
final FSNamesystem namesystem = cluster.getNamesystem(); final FSNamesystem namesystem = cluster.getNamesystem();
@ -100,7 +100,7 @@ public class TestFSEditLogLoader {
bld.append("Recent opcode offsets: (\\d+\\s*){4}$"); bld.append("Recent opcode offsets: (\\d+\\s*){4}$");
try { try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES)
.format(false).build(); .enableManagedDfsDirsRedundancy(false).format(false).build();
fail("should not be able to start"); fail("should not be able to start");
} catch (IOException e) { } catch (IOException e) {
assertTrue("error message contains opcodes message", assertTrue("error message contains opcodes message",

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
View File

@ -347,7 +347,7 @@ public class TestNameNodeRecovery {
StorageDirectory sd = null; StorageDirectory sd = null;
try { try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.build(); .enableManagedDfsDirsRedundancy(false).build();
cluster.waitActive(); cluster.waitActive();
if (!finalize) { if (!finalize) {
// Normally, the in-progress edit log would be finalized by // Normally, the in-progress edit log would be finalized by
@ -383,7 +383,7 @@ public class TestNameNodeRecovery {
try { try {
LOG.debug("trying to start normally (this should fail)..."); LOG.debug("trying to start normally (this should fail)...");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(false).build(); .enableManagedDfsDirsRedundancy(false).format(false).build();
cluster.waitActive(); cluster.waitActive();
cluster.shutdown(); cluster.shutdown();
if (needRecovery) { if (needRecovery) {
@ -408,7 +408,8 @@ public class TestNameNodeRecovery {
try { try {
LOG.debug("running recovery..."); LOG.debug("running recovery...");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(false).startupOption(recoverStartOpt).build(); .enableManagedDfsDirsRedundancy(false).format(false)
.startupOption(recoverStartOpt).build();
} catch (IOException e) { } catch (IOException e) {
fail("caught IOException while trying to recover. " + fail("caught IOException while trying to recover. " +
"message was " + e.getMessage() + "message was " + e.getMessage() +
@ -424,7 +425,7 @@ public class TestNameNodeRecovery {
try { try {
LOG.debug("starting cluster normally after recovery..."); LOG.debug("starting cluster normally after recovery...");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(false).build(); .enableManagedDfsDirsRedundancy(false).format(false).build();
LOG.debug("successfully recovered the " + corruptor.getName() + LOG.debug("successfully recovered the " + corruptor.getName() +
" corrupted edit log"); " corrupted edit log");
cluster.waitActive(); cluster.waitActive();