From 6f1452044a0283d9132fa83783800dde3561d5f1 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 16 Jul 2013 14:07:22 +0000 Subject: [PATCH] svn merge -c 1503720 from trunk for HDFS-4992. Make balancer's mover thread count and dispatcher thread count configurable. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1503721 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../hadoop/hdfs/server/balancer/Balancer.java | 17 ++++++++++------- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 43fe1b2bcb0..a376d93ae85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -223,6 +223,9 @@ Release 2.1.0-beta - 2013-07-02 HDFS-4903. Print trash configuration and trash emptier state in namenode log. (Arpit Agarwal via suresh) + HDFS-4992. Make balancer's mover thread count and dispatcher thread count + configurable. (Max Lapan via szetszwo) + OPTIMIZATIONS HDFS-4465. Optimize datanode ReplicasMap and ReplicaInfo. (atm) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 7ef6b31b8fd..f575d614dea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -276,6 +276,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT = false; public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth"; public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; + public static final String DFS_BALANCER_MOVERTHREADS_KEY = "dfs.balancer.moverThreads"; + public static final int DFS_BALANCER_MOVERTHREADS_DEFAULT = 1000; + public static final String DFS_BALANCER_DISPATCHERTHREADS_KEY = "dfs.balancer.dispatcherThreads"; + public static final int DFS_BALANCER_DISPATCHERTHREADS_DEFAULT = 200; public static final String DFS_DATANODE_ADDRESS_KEY = "dfs.datanode.address"; public static final int DFS_DATANODE_DEFAULT_PORT = 50010; public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_DEFAULT_PORT; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index cab6dd788f9..cac8bf31bc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -226,13 +226,9 @@ public class Balancer { = new HashMap(); private NetworkTopology cluster; - final static private int MOVER_THREAD_POOL_SIZE = 1000; - final private ExecutorService moverExecutor = - Executors.newFixedThreadPool(MOVER_THREAD_POOL_SIZE); - final static private int DISPATCHER_THREAD_POOL_SIZE = 200; - final private ExecutorService dispatcherExecutor = - Executors.newFixedThreadPool(DISPATCHER_THREAD_POOL_SIZE); - + + private final ExecutorService moverExecutor; + private final ExecutorService dispatcherExecutor; /* This class keeps track of a scheduled block move */ private class PendingBlockMove { @@ -830,6 +826,13 @@ public class Balancer { this.policy = p.policy; this.nnc = theblockpool; cluster = NetworkTopology.getInstance(conf); + + this.moverExecutor = Executors.newFixedThreadPool( + conf.getInt(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY, + DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_DEFAULT)); + this.dispatcherExecutor = Executors.newFixedThreadPool( + conf.getInt(DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_KEY, + DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_DEFAULT)); } /* Shuffle datanode array */