From a48e8c9bebf4ab7b46eccac07e9743d4038b45c2 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 14 Nov 2022 02:07:40 +0000 Subject: [PATCH] MAPREDUCE-5608. Replace and deprecate mapred.tasktracker.indexcache.mb (#5014) Co-authored-by: Ashutosh Gupta Signed-off-by: Akira Ajisaka --- .../src/site/markdown/DeprecatedProperties.md | 3 ++- .../java/org/apache/hadoop/mapred/IndexCache.java | 4 ++-- .../org/apache/hadoop/mapreduce/MRJobConfig.java | 2 ++ .../mapreduce/server/tasktracker/TTConfig.java | 6 ++++++ .../org/apache/hadoop/mapreduce/util/ConfigUtil.java | 6 ++++-- .../org/apache/hadoop/mapred/TestIndexCache.java | 12 ++++++------ 6 files changed, 22 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md index 281e42dad88..a00feb039d8 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md @@ -208,7 +208,8 @@ The following table lists the configuration property names that are deprecated i | mapred.task.profile.params | mapreduce.task.profile.params | | mapred.task.profile.reduces | mapreduce.task.profile.reduces | | mapred.task.timeout | mapreduce.task.timeout | -| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb | +| mapred.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb | +| mapreduce.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb | | mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum | | mapred.tasktracker.memory\_calculator\_plugin | mapreduce.tasktracker.resourcecalculatorplugin | | mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin | diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java index 0e24bbe5330..70328ea0075 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java @@ -23,7 +23,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +43,7 @@ class IndexCache { public IndexCache(JobConf conf) { this.conf = conf; totalMemoryAllowed = - conf.getInt(TTConfig.TT_INDEX_CACHE, 10) * 1024 * 1024; + conf.getInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 10) * 1024 * 1024; LOG.info("IndexCache created with max memory = " + totalMemoryAllowed); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 4523565f300..02fa4eca636 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -577,6 +577,8 @@ public interface MRJobConfig { public static final String MAX_SHUFFLE_FETCH_HOST_FAILURES = "mapreduce.reduce.shuffle.max-host-failures"; public static final int DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES = 5; + public static final String SHUFFLE_INDEX_CACHE = "mapreduce.reduce.shuffle.indexcache.mb"; + public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr"; public static final String REDUCE_SKIP_MAXGROUPS = "mapreduce.reduce.skip.maxgroups"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java index f75ad05d295..c0e91804dcb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java @@ -29,6 +29,12 @@ import org.apache.hadoop.mapreduce.MRConfig; @InterfaceStability.Evolving public interface TTConfig extends MRConfig { + /** + * @deprecated Use + * {@link org.apache.hadoop.mapreduce.MRJobConfig#SHUFFLE_INDEX_CACHE} + * instead + */ + @Deprecated public static final String TT_INDEX_CACHE = "mapreduce.tasktracker.indexcache.mb"; public static final String TT_MAP_SLOTS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java index cf120342dd1..95074de231c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java @@ -80,8 +80,6 @@ public class ConfigUtil { JTConfig.JT_TASKCACHE_LEVELS), new DeprecationDelta("mapred.job.tracker.retire.jobs", JTConfig.JT_RETIREJOBS), - new DeprecationDelta("mapred.tasktracker.indexcache.mb", - TTConfig.TT_INDEX_CACHE), new DeprecationDelta("mapred.tasktracker.map.tasks.maximum", TTConfig.TT_MAP_SLOTS), new DeprecationDelta("mapred.tasktracker.memory_calculator_plugin", @@ -290,6 +288,10 @@ public class ConfigUtil { MRJobConfig.REDUCE_LOG_LEVEL), new DeprecationDelta("mapreduce.job.counters.limit", MRJobConfig.COUNTERS_MAX_KEY), + new DeprecationDelta("mapred.tasktracker.indexcache.mb", + MRJobConfig.SHUFFLE_INDEX_CACHE), + new DeprecationDelta("mapreduce.tasktracker.indexcache.mb", + MRJobConfig.SHUFFLE_INDEX_CACHE), new DeprecationDelta("jobclient.completion.poll.interval", Job.COMPLETION_POLL_INTERVAL_KEY), new DeprecationDelta("jobclient.progress.monitor.poll.interval", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java index dabce770e82..2df5cc2dc06 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java @@ -30,7 +30,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.junit.Before; import org.junit.Test; @@ -56,7 +56,7 @@ public class TestIndexCache { r.setSeed(seed); System.out.println("seed: " + seed); fs.delete(p, true); - conf.setInt(TTConfig.TT_INDEX_CACHE, 1); + conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1); final int partsPerMap = 1000; final int bytesPerFile = partsPerMap * 24; IndexCache cache = new IndexCache(conf); @@ -127,7 +127,7 @@ public class TestIndexCache { public void testBadIndex() throws Exception { final int parts = 30; fs.delete(p, true); - conf.setInt(TTConfig.TT_INDEX_CACHE, 1); + conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1); IndexCache cache = new IndexCache(conf); Path f = new Path(p, "badindex"); @@ -159,7 +159,7 @@ public class TestIndexCache { @Test public void testInvalidReduceNumberOrLength() throws Exception { fs.delete(p, true); - conf.setInt(TTConfig.TT_INDEX_CACHE, 1); + conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1); final int partsPerMap = 1000; final int bytesPerFile = partsPerMap * 24; IndexCache cache = new IndexCache(conf); @@ -205,7 +205,7 @@ public class TestIndexCache { // fails with probability of 100% on code before MAPREDUCE-2541, // so it is repeatable in practice. fs.delete(p, true); - conf.setInt(TTConfig.TT_INDEX_CACHE, 10); + conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 10); // Make a big file so removeMapThread almost surely runs faster than // getInfoThread final int partsPerMap = 100000; @@ -251,7 +251,7 @@ public class TestIndexCache { @Test public void testCreateRace() throws Exception { fs.delete(p, true); - conf.setInt(TTConfig.TT_INDEX_CACHE, 1); + conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1); final int partsPerMap = 1000; final int bytesPerFile = partsPerMap * 24; final IndexCache cache = new IndexCache(conf);