HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
This commit is contained in:
Yiqun Lin
parent
1a407bc990
commit
0725953efe
|
|
@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
|
||||
import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
|
||||
|
|
@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
|||
"dfs.use.dfs.network.topology";
|
||||
public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
|
||||
|
||||
public static final String DFS_NET_TOPOLOGY_IMPL_KEY =
|
||||
"dfs.net.topology.impl";
|
||||
|
||||
public static final Class<DFSNetworkTopology> DFS_NET_TOPOLOGY_IMPL_DEFAULT =
|
||||
DFSNetworkTopology.class;
|
||||
|
||||
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
|
||||
@Deprecated
|
||||
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
|
||||
|
|
|
|||
|
|
@ -22,11 +22,13 @@ import com.google.common.base.Preconditions;
|
|||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.StorageType;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
|
||||
import org.apache.hadoop.net.NetworkTopology;
|
||||
import org.apache.hadoop.net.Node;
|
||||
import org.apache.hadoop.net.NodeBase;
|
||||
import org.apache.hadoop.util.ReflectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
|
@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology {
|
|||
private static final Random RANDOM = new Random();
|
||||
|
||||
public static DFSNetworkTopology getInstance(Configuration conf) {
|
||||
DFSNetworkTopology nt = new DFSNetworkTopology();
|
||||
return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY);
|
||||
|
||||
DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass(
|
||||
DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY,
|
||||
DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT,
|
||||
DFSNetworkTopology.class), conf);
|
||||
return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -4859,6 +4859,20 @@
|
|||
<value>true</value>
|
||||
<description>
|
||||
Enables DFSNetworkTopology to choose nodes for placing replicas.
|
||||
When enabled, NetworkTopology will be instantiated as class defined in
|
||||
property dfs.net.topology.impl, otherwise NetworkTopology will be
|
||||
instantiated as class defined in property net.topology.impl.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.net.topology.impl</name>
|
||||
<value>org.apache.hadoop.hdfs.net.DFSNetworkTopology</value>
|
||||
<description>
|
||||
The implementation class of NetworkTopology used in HDFS. By default,
|
||||
the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and
|
||||
used in block placement.
|
||||
This property only works when dfs.use.dfs.network.topology is true.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
|||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.StorageType;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
|
||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||
|
|
@ -61,11 +63,13 @@ import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
|
|||
import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
|
||||
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
|
||||
import org.apache.hadoop.net.DNSToSwitchMapping;
|
||||
import org.apache.hadoop.net.NetworkTopology;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.internal.util.reflection.Whitebox;
|
||||
|
||||
import static org.hamcrest.core.Is.is;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
|
|
@ -622,4 +626,52 @@ public class TestDatanodeManager {
|
|||
// Approximately load tasks if the ratio between queue length is large.
|
||||
verifyPendingRecoveryTasks(400, 1, 20, 20, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNetworkTopologyInstantiation() throws Exception {
|
||||
// case 1, dfs.use.dfs.network.topology=true, use the default
|
||||
// DFSNetworkTopology impl.
|
||||
Configuration conf1 = new HdfsConfiguration();
|
||||
FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
|
||||
DatanodeManager dm1 = mockDatanodeManager(fsn, conf1);
|
||||
assertEquals(DFSNetworkTopology.class, dm1.getNetworkTopology().getClass());
|
||||
|
||||
// case 2, dfs.use.dfs.network.topology=false, use the default
|
||||
// NetworkTopology impl.
|
||||
Configuration conf2 = new HdfsConfiguration();
|
||||
conf2.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
|
||||
DatanodeManager dm2 = mockDatanodeManager(fsn, conf2);
|
||||
assertEquals(NetworkTopology.class, dm2.getNetworkTopology()
|
||||
.getClass());
|
||||
|
||||
// case 3, dfs.use.dfs.network.topology=false, and specify the
|
||||
// net.topology.impl property.
|
||||
Configuration conf3 = new HdfsConfiguration();
|
||||
conf3.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
|
||||
MockDfsNetworkTopology.class, NetworkTopology.class);
|
||||
conf3.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
|
||||
DatanodeManager dm3 = mockDatanodeManager(fsn, conf3);
|
||||
assertEquals(MockDfsNetworkTopology.class, dm3.getNetworkTopology()
|
||||
.getClass());
|
||||
|
||||
// case 4, dfs.use.dfs.network.topology=true, and specify the
|
||||
// dfs.net.topology.impl property.
|
||||
Configuration conf4 = new HdfsConfiguration();
|
||||
conf4.setClass(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY,
|
||||
MockDfsNetworkTopology.class, NetworkTopology.class);
|
||||
conf4.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true);
|
||||
DatanodeManager dm4 = mockDatanodeManager(fsn, conf4);
|
||||
assertEquals(MockDfsNetworkTopology.class, dm4.getNetworkTopology()
|
||||
.getClass());
|
||||
}
|
||||
|
||||
/**
|
||||
* A NetworkTopology implementation for test.
|
||||
*
|
||||
*/
|
||||
public static class MockDfsNetworkTopology extends DFSNetworkTopology {
|
||||
public MockDfsNetworkTopology(){
|
||||
super();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue