MAPREDUCE-3794. Support mapred.Task.Counter and mapred.JobInProgress.Counter enums for compatibility (Tom White via mahadev) - Merging r1241299 from trunk.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1241300 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
0c7f4cddce
commit
8aa0153373
|
@ -685,6 +685,9 @@ Release 0.23.1 - Unreleased
|
|||
MAPREDUCE-3809. Ensure that there is no needless sleep in Task at the end
|
||||
of the task. (sseth via acmurthy)
|
||||
|
||||
MAPREDUCE-3794. Support mapred.Task.Counter and mapred.JobInProgress.Counter
|
||||
enums for compatibility (Tom White via mahadev)
|
||||
|
||||
Release 0.23.0 - 2011-11-01
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.mapred;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.mapreduce.JobCounter;
|
||||
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public class JobInProgress {
|
||||
|
||||
/**
|
||||
* @deprecated Provided for compatibility. Use {@link JobCounter} instead.
|
||||
*/
|
||||
@Deprecated
|
||||
public static enum Counter {
|
||||
NUM_FAILED_MAPS,
|
||||
NUM_FAILED_REDUCES,
|
||||
TOTAL_LAUNCHED_MAPS,
|
||||
TOTAL_LAUNCHED_REDUCES,
|
||||
OTHER_LOCAL_MAPS,
|
||||
DATA_LOCAL_MAPS,
|
||||
RACK_LOCAL_MAPS,
|
||||
SLOTS_MILLIS_MAPS,
|
||||
SLOTS_MILLIS_REDUCES,
|
||||
FALLOW_SLOTS_MILLIS_MAPS,
|
||||
FALLOW_SLOTS_MILLIS_REDUCES
|
||||
}
|
||||
|
||||
}
|
|
@ -80,6 +80,33 @@ abstract public class Task implements Writable, Configurable {
|
|||
|
||||
public static String MERGED_OUTPUT_PREFIX = ".merged";
|
||||
public static final long DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS = 10000;
|
||||
|
||||
/**
|
||||
* @deprecated Provided for compatibility. Use {@link TaskCounter} instead.
|
||||
*/
|
||||
@Deprecated
|
||||
public static enum Counter {
|
||||
MAP_INPUT_RECORDS,
|
||||
MAP_OUTPUT_RECORDS,
|
||||
MAP_SKIPPED_RECORDS,
|
||||
MAP_INPUT_BYTES,
|
||||
MAP_OUTPUT_BYTES,
|
||||
MAP_OUTPUT_MATERIALIZED_BYTES,
|
||||
COMBINE_INPUT_RECORDS,
|
||||
COMBINE_OUTPUT_RECORDS,
|
||||
REDUCE_INPUT_GROUPS,
|
||||
REDUCE_SHUFFLE_BYTES,
|
||||
REDUCE_INPUT_RECORDS,
|
||||
REDUCE_OUTPUT_RECORDS,
|
||||
REDUCE_SKIPPED_GROUPS,
|
||||
REDUCE_SKIPPED_RECORDS,
|
||||
SPILLED_RECORDS,
|
||||
SPLIT_RAW_BYTES,
|
||||
CPU_MILLISECONDS,
|
||||
PHYSICAL_MEMORY_BYTES,
|
||||
VIRTUAL_MEMORY_BYTES,
|
||||
COMMITTED_HEAP_BYTES
|
||||
}
|
||||
|
||||
/**
|
||||
* Counters to measure the usage of the different file systems.
|
||||
|
@ -826,7 +853,8 @@ abstract public class Task implements Writable, Configurable {
|
|||
return; // nothing to do.
|
||||
}
|
||||
|
||||
Counter gcCounter = counters.findCounter(TaskCounter.GC_TIME_MILLIS);
|
||||
org.apache.hadoop.mapred.Counters.Counter gcCounter =
|
||||
counters.findCounter(TaskCounter.GC_TIME_MILLIS);
|
||||
if (null != gcCounter) {
|
||||
gcCounter.increment(getElapsedGc());
|
||||
}
|
||||
|
|
|
@ -132,23 +132,43 @@ public class TestCounters {
|
|||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testLegacyNames() {
|
||||
public void testReadWithLegacyNames() {
|
||||
Counters counters = new Counters();
|
||||
counters.incrCounter(TaskCounter.MAP_INPUT_RECORDS, 1);
|
||||
counters.incrCounter(JobCounter.DATA_LOCAL_MAPS, 1);
|
||||
counters.findCounter("file", FileSystemCounter.BYTES_READ).increment(1);
|
||||
|
||||
checkLegacyNames(counters);
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testWriteWithLegacyNames() {
|
||||
Counters counters = new Counters();
|
||||
counters.incrCounter(Task.Counter.MAP_INPUT_RECORDS, 1);
|
||||
counters.incrCounter(JobInProgress.Counter.DATA_LOCAL_MAPS, 1);
|
||||
counters.findCounter("FileSystemCounter", "FILE_BYTES_READ").increment(1);
|
||||
|
||||
checkLegacyNames(counters);
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private void checkLegacyNames(Counters counters) {
|
||||
assertEquals("New name", 1, counters.findCounter(
|
||||
TaskCounter.class.getName(), "MAP_INPUT_RECORDS").getValue());
|
||||
assertEquals("Legacy name", 1, counters.findCounter(
|
||||
"org.apache.hadoop.mapred.Task$Counter",
|
||||
"MAP_INPUT_RECORDS").getValue());
|
||||
assertEquals("Legacy enum", 1,
|
||||
counters.findCounter(Task.Counter.MAP_INPUT_RECORDS).getValue());
|
||||
|
||||
assertEquals("New name", 1, counters.findCounter(
|
||||
JobCounter.class.getName(), "DATA_LOCAL_MAPS").getValue());
|
||||
assertEquals("Legacy name", 1, counters.findCounter(
|
||||
"org.apache.hadoop.mapred.JobInProgress$Counter",
|
||||
"DATA_LOCAL_MAPS").getValue());
|
||||
assertEquals("Legacy enum", 1,
|
||||
counters.findCounter(JobInProgress.Counter.DATA_LOCAL_MAPS).getValue());
|
||||
|
||||
assertEquals("New name", 1, counters.findCounter(
|
||||
FileSystemCounter.class.getName(), "FILE_BYTES_READ").getValue());
|
||||
|
|
Loading…
Reference in New Issue