From a04bee4567cfa0611023eff33f3dcad22fa265e5 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Sat, 2 Feb 2013 00:47:58 +0000 Subject: [PATCH] MAPREDUCE-4971. Minor extensibility enhancements to Counters & FileOutputFormat. Contributed by Arun C Murthy git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441670 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/mapred/Counters.java | 4 ++++ .../org/apache/hadoop/mapred/FileOutputCommitter.java | 2 +- .../org/apache/hadoop/mapred/FileOutputFormat.java | 5 +++-- .../main/java/org/apache/hadoop/mapred/JobConf.java | 4 +++- .../mapreduce/counters/FileSystemCounterGroup.java | 11 +++++++++++ .../mapreduce/counters/FrameworkCounterGroup.java | 11 +++++++++++ 7 files changed, 36 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e405fde57dc..e09203afb72 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -215,6 +215,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4838. Add additional fields like Locality, Avataar to the JobHistory logs. (Zhijie Shen via sseth) + MAPREDUCE-4971. Minor extensibility enhancements to Counters & + FileOutputFormat. (Arun C Murthy via sseth) + OPTIMIZATIONS MAPREDUCE-4893. Fixed MR ApplicationMaster to do optimal assignment of diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java index da25b2aa137..7d648a49126 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java @@ -230,6 +230,10 @@ public int hashCode() { public static class Group implements CounterGroupBase { private CounterGroupBase realGroup; + protected Group() { + realGroup = null; + } + Group(GenericGroup group) { this.realGroup = group; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java index a6190d2060d..496280a7371 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java @@ -92,7 +92,7 @@ Path getJobAttemptPath(JobContext context) { } @Private - Path getTaskAttemptPath(TaskAttemptContext context) throws IOException { + public Path getTaskAttemptPath(TaskAttemptContext context) throws IOException { Path out = getOutputPath(context); return out == null ? null : getTaskAttemptPath(context, out); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java index 892e6906b0c..9082de8ce50 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java @@ -22,6 +22,7 @@ import java.text.NumberFormat; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -152,8 +153,8 @@ public static void setOutputPath(JobConf conf, Path outputDir) { * @param outputDir the {@link Path} of the output directory * for the map-reduce job. */ - - static void setWorkOutputPath(JobConf conf, Path outputDir) { + @Private + public static void setWorkOutputPath(JobConf conf, Path outputDir) { outputDir = new Path(conf.getWorkingDirectory(), outputDir); conf.set(JobContext.TASK_OUTPUT_DIR, outputDir.toString()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 1304755e3c3..e76f62856da 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -419,7 +420,8 @@ public Credentials getCredentials() { return credentials; } - void setCredentials(Credentials credentials) { + @Private + public void setCredentials(Credentials credentials) { this.credentials = credentials; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java index 3f3729fb056..b1b67969cfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java @@ -35,6 +35,7 @@ import com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.FileSystemCounter; @@ -72,6 +73,16 @@ public FSCounter(String scheme, FileSystemCounter ref) { this.scheme = scheme; key = ref; } + + @Private + public String getScheme() { + return scheme; + } + + @Private + public FileSystemCounter getFileSystemCounter() { + return key; + } @Override public String getName() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java index 64e4cc81631..5ebed5a7319 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.util.ResourceBundles; @@ -66,7 +67,17 @@ public FrameworkCounter(T ref, String groupName) { key = ref; this.groupName = groupName; } + + @Private + public T getKey() { + return key; + } + @Private + public String getGroupName() { + return groupName; + } + @Override public String getName() { return key.name();