Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1801. Make Topology Aware Replication/Read non-default for ozone 0.4.1. Contributed by Xiaoyu Yao. 

Signed-off-by: Anu Engineer <aengineer@apache.org>
This commit is contained in:
Xiaoyu Yao 2019-07-15 14:28:53 -07:00 committed by Anu Engineer
parent c58e11bf52
commit eb0849b547
5 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
View File

@ -110,7 +110,8 @@ public class XceiverClientGrpc extends XceiverClientSpi {
public void connect() throws Exception {
// connect to the closest node, if closest node doesn't exist, delegate to
// first node, which is usually the leader in the pipeline.
DatanodeDetails dn = this.pipeline.getClosestNode();
DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
this.pipeline.getFirstNode();
// just make a connection to the picked datanode at the beginning
connectToDatanode(dn, null);
}
@ -122,8 +123,8 @@ public class XceiverClientGrpc extends XceiverClientSpi {
public void connect(String encodedToken) throws Exception {
// connect to the closest node, if closest node doesn't exist, delegate to
// first node, which is usually the leader in the pipeline.
DatanodeDetails dn;
dn = this.pipeline.getClosestNode();
DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
this.pipeline.getFirstNode();
// just make a connection to the picked datanode at the beginning
connectToDatanode(dn, encodedToken);
}

6
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View File

@ -317,10 +317,6 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
"ozone.scm.container.placement.impl";
public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT =
"org.apache.hadoop.hdds.scm.container.placement.algorithms." +
"SCMContainerPlacementRackAware";
public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
"ozone.scm.pipeline.owner.container.count";
public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
@ -371,7 +367,7 @@ public final class ScmConfigKeys {
public static final String DFS_NETWORK_TOPOLOGY_AWARE_READ_ENABLED =
"dfs.network.topology.aware.read.enable";
public static final String DFS_NETWORK_TOPOLOGY_AWARE_READ_ENABLED_DEFAULT =
"true";
"false";
public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
public static final boolean HDDS_TRACING_ENABLED_DEFAULT = true;

6
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -823,13 +823,13 @@
<property>
<name>ozone.scm.container.placement.impl</name>
<value>
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom
</value>
<tag>OZONE, MANAGEMENT</tag>
<description>
The full name of class which implements org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy.
The class decides which datanode will be used to host the container replica. If not set,
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware will be used as default value.
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom will be used as default value.
</description>
</property>
<property>
@ -2347,7 +2347,7 @@
</property>
<property>
<name>dfs.network.topology.aware.read.enable</name>
<value>true</value>
<value>false</value>
<tag>OZONE, PERFORMANCE</tag>
<description>
Whether to enable topology aware read to improve the read performance.

6
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
View File

@ -34,6 +34,10 @@ public final class ContainerPlacementPolicyFactory {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerPlacementPolicyFactory.class);
private static final Class<? extends ContainerPlacementPolicy>
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT =
SCMContainerPlacementRandom.class;
private ContainerPlacementPolicyFactory() {
}
@ -42,7 +46,7 @@ public final class ContainerPlacementPolicyFactory {
final boolean fallback) throws SCMException{
final Class<? extends ContainerPlacementPolicy> placementClass = conf
.getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
SCMContainerPlacementRackAware.class,
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
ContainerPlacementPolicy.class);
Constructor<? extends ContainerPlacementPolicy> constructor;
try {

7
hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config vendored
View File

@ -26,11 +26,12 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware
OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
OZONE-SITE.XML_dfs.network.topology.aware.read.enable=true
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
HDFS-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
HDFS-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
HDFS-SITE.XML_dfs.network.topology.aware.read.enable=true
ASYNC_PROFILER_HOME=/opt/profiler
LOG4J.PROPERTIES_log4j.rootLogger=DEBUG, ARF
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender