From 4394e5edb0e87879036b97da7db6a20be2e111d3 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 30 May 2013 23:53:21 +0000 Subject: [PATCH] MAPREDUCE-5228. Bring back FileInputFormat.Counter and FileOuputFormat.Counter for binary compatibility with 1.x mapred APIs. Contributed by Mayank Bansal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488060 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++ .../org/apache/hadoop/mapred/Counters.java | 35 +++++++++++++- .../apache/hadoop/mapred/FileInputFormat.java | 5 ++ .../mapred/FileInputFormat_Counter.properties | 18 +++++++ .../hadoop/mapred/FileOutputFormat.java | 5 ++ .../FileOutputFormat_Counter.properties | 18 +++++++ .../mapreduce/lib/input/FileInputFormat.java | 5 ++ .../input/FileInputFormat_Counter.properties | 18 +++++++ .../input/FileOutputFormat_Counter.properties | 18 +++++++ .../lib/output/FileOutputFormat.java | 5 ++ .../apache/hadoop/mapred/TestJobCounters.java | 48 ++++++++++++++++++- 11 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat_Counter.properties create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat_Counter.properties create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat_Counter.properties create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat_Counter.properties diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 51886210cc7..86ddbc7e7ba 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -245,6 +245,10 @@ Release 2.0.5-beta - UNRELEASED filecache.DistributedCache for binary compatibility with mapred in 1.x. (Zhijie Shen via vinodkv) + MAPREDUCE-5228. Bring back FileInputFormat.Counter and + FileOuputFormat.Counter for binary compatibility with 1.x mapred APIs. + (Mayank Bansal via vinodkv) + OPTIMIZATIONS MAPREDUCE-4974. Optimising the LineRecordReader initialize() method diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java index a35fdaf4c1e..4b277f80a2c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java @@ -26,6 +26,7 @@ import java.io.DataOutput; import java.io.IOException; import java.text.ParseException; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import org.apache.commons.collections.IteratorUtils; @@ -40,10 +41,9 @@ import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; import org.apache.hadoop.mapreduce.counters.GenericCounter; -import org.apache.hadoop.mapreduce.counters.LimitExceededException; import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; -import org.apache.hadoop.mapreduce.util.CountersStrings; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; import com.google.common.collect.Iterators; @@ -64,6 +64,12 @@ public class Counters public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX; public static int MAX_GROUP_LIMIT = Limits.GROUPS_MAX; + private static HashMap depricatedCounterMap = + new HashMap(); + + static { + initDepricatedMap(); + } public Counters() { super(groupFactory); @@ -73,6 +79,27 @@ public class Counters super(newCounters, groupFactory); } + @SuppressWarnings({ "deprecation" }) + private static void initDepricatedMap() { + depricatedCounterMap.put(FileInputFormat.Counter.class.getName(), + FileInputFormatCounter.class.getName()); + depricatedCounterMap.put(FileOutputFormat.Counter.class.getName(), + FileOutputFormatCounter.class.getName()); + depricatedCounterMap.put( + org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.class + .getName(), FileInputFormatCounter.class.getName()); + depricatedCounterMap.put( + org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.class + .getName(), FileOutputFormatCounter.class.getName()); + } + + private static String getNewGroupKey(String oldGroup) { + if (depricatedCounterMap.containsKey(oldGroup)) { + return depricatedCounterMap.get(oldGroup); + } + return null; + } + /** * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters * @param newCounters new Counters @@ -445,6 +472,10 @@ public class Counters " BYTES_READ as counter name instead"); return findCounter(FileInputFormatCounter.BYTES_READ); } + String newGroupKey = getNewGroupKey(group); + if (newGroupKey != null) { + group = newGroupKey; + } return getGroup(group).getCounterForName(name); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java index d06d6fb28bf..a68db672bfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java @@ -61,6 +61,11 @@ public abstract class FileInputFormat implements InputFormat { public static final Log LOG = LogFactory.getLog(FileInputFormat.class); + + @Deprecated + public static enum Counter { + BYTES_READ + } public static final String NUM_INPUT_FILES = org.apache.hadoop.mapreduce.lib.input.FileInputFormat.NUM_INPUT_FILES; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat_Counter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat_Counter.properties new file mode 100644 index 00000000000..7cf5d5dd062 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat_Counter.properties @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ResourceBundle properties file for file-input-format counters + +CounterGroupName= File Input Format Counters + +BYTES_READ.name= Bytes Read diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java index 9082de8ce50..0efcf9ddf9e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java @@ -36,6 +36,11 @@ import org.apache.hadoop.util.Progressable; @InterfaceStability.Stable public abstract class FileOutputFormat implements OutputFormat { + @Deprecated + public static enum Counter { + BYTES_WRITTEN + } + /** * Set whether the output of the job is compressed. * @param conf the {@link JobConf} to modify diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat_Counter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat_Counter.properties new file mode 100644 index 00000000000..ddcb608a53e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat_Counter.properties @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ResourceBundle properties file for file-output-format counters + +CounterGroupName= File Output Format Counters + +BYTES_WRITTEN.name= Bytes Written diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java index 9c4809a22e4..56c2cdcc879 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java @@ -68,6 +68,11 @@ public abstract class FileInputFormat extends InputFormat { private static final Log LOG = LogFactory.getLog(FileInputFormat.class); private static final double SPLIT_SLOP = 1.1; // 10% slop + + @Deprecated + public static enum Counter { + BYTES_READ + } private static final PathFilter hiddenFileFilter = new PathFilter(){ public boolean accept(Path p){ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat_Counter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat_Counter.properties new file mode 100644 index 00000000000..7cf5d5dd062 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat_Counter.properties @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ResourceBundle properties file for file-input-format counters + +CounterGroupName= File Input Format Counters + +BYTES_READ.name= Bytes Read diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat_Counter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat_Counter.properties new file mode 100644 index 00000000000..ddcb608a53e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat_Counter.properties @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ResourceBundle properties file for file-output-format counters + +CounterGroupName= File Output Format Counters + +BYTES_WRITTEN.name= Bytes Written diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java index 60345a426e4..fa3708e7948 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java @@ -60,6 +60,11 @@ public static final String COMPRESS_CODEC = public static final String COMPRESS_TYPE = "mapreduce.output.fileoutputformat.compress.type"; public static final String OUTDIR = "mapreduce.output.fileoutputformat.outputdir"; + @Deprecated + public static enum Counter { + BYTES_WRITTEN + } + /** * Set whether the output of the job is compressed. * @param job the job to modify diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java index d476ba98a4c..59e51ecdd8c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java @@ -80,6 +80,40 @@ public class TestJobCounters { } } + @SuppressWarnings("deprecation") + private void validateOldFileCounters(Counters counter, long fileBytesRead, + long fileBytesWritten, long mapOutputBytes, + long mapOutputMaterializedBytes) { + + assertEquals(fileBytesRead, + counter.findCounter(FileInputFormat.Counter.BYTES_READ).getValue()); + + assertEquals( + fileBytesRead, + counter + .findCounter( + org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.BYTES_READ) + .getValue()); + + assertEquals(fileBytesWritten, + counter.findCounter(FileOutputFormat.Counter.BYTES_WRITTEN).getValue()); + + assertEquals( + fileBytesWritten, + counter + .findCounter( + org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.BYTES_WRITTEN) + .getValue()); + + if (mapOutputBytes >= 0) { + assertTrue(counter.findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue() != 0); + } + if (mapOutputMaterializedBytes >= 0) { + assertTrue(counter.findCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES) + .getValue() != 0); + } + } + private void validateCounters(Counters counter, long spillRecCnt, long mapInputRecords, long mapOutputRecords) { // Check if the numer of Spilled Records is same as expected @@ -255,7 +289,7 @@ public class TestJobCounters { // 4 records/line = 61440 output records validateCounters(c1, 90112, 15360, 61440); validateFileCounters(c1, inputSize, 0, 0, 0); - + validateOldFileCounters(c1, inputSize, 61928, 0, 0); } @Test @@ -431,6 +465,18 @@ public class TestJobCounters { validateFileCounters(c1, inputSize, 0, -1, -1); } + @SuppressWarnings("deprecation") + @Test + public void testOldCounters() throws Exception { + Counters c1 = new Counters(); + c1.incrCounter(FileInputFormat.Counter.BYTES_READ, 100); + c1.incrCounter(FileOutputFormat.Counter.BYTES_WRITTEN, 200); + c1.incrCounter(TaskCounter.MAP_OUTPUT_BYTES, 100); + c1.incrCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES, 100); + validateFileCounters(c1, 100, 200, 100, 100); + validateOldFileCounters(c1, 100, 200, 100, 100); + } + /** * Increases the JVM's heap usage to the specified target value. */