MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again in Limits (Sandy Ryza)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1524408 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
5e18410e06
commit
c09f00771e
|
@ -164,6 +164,9 @@ Release 2.3.0 - UNRELEASED
|
|||
|
||||
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
|
||||
|
||||
MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again
|
||||
in Limits (Sandy Ryza)
|
||||
|
||||
BUG FIXES
|
||||
|
||||
MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal
|
||||
|
|
|
@ -41,6 +41,7 @@ import org.apache.hadoop.ipc.RPC;
|
|||
import org.apache.hadoop.mapreduce.MRConfig;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.apache.hadoop.mapreduce.TaskType;
|
||||
import org.apache.hadoop.mapreduce.counters.Limits;
|
||||
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
|
||||
import org.apache.hadoop.mapreduce.security.TokenCache;
|
||||
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
|
||||
|
@ -76,6 +77,8 @@ class YarnChild {
|
|||
LOG.debug("Child starting");
|
||||
|
||||
final JobConf job = new JobConf();
|
||||
// Initing with our JobConf allows us to avoid loading confs twice
|
||||
Limits.init(job);
|
||||
job.addResource(MRJobConfig.JOB_CONF_FILE);
|
||||
UserGroupInformation.setConfiguration(job);
|
||||
|
||||
|
|
|
@ -62,8 +62,8 @@ import com.google.common.collect.Iterators;
|
|||
public class Counters
|
||||
extends AbstractCounters<Counters.Counter, Counters.Group> {
|
||||
|
||||
public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
|
||||
public static int MAX_GROUP_LIMIT = Limits.GROUPS_MAX;
|
||||
public static int MAX_COUNTER_LIMIT = Limits.getCountersMax();
|
||||
public static int MAX_GROUP_LIMIT = Limits.getGroupsMax();
|
||||
private static HashMap<String, String> depricatedCounterMap =
|
||||
new HashMap<String, String>();
|
||||
|
||||
|
|
|
@ -28,37 +28,80 @@ import static org.apache.hadoop.mapreduce.MRJobConfig.*;
|
|||
public class Limits {
|
||||
|
||||
static final Configuration conf = new JobConf();
|
||||
public static final int GROUP_NAME_MAX =
|
||||
conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT);
|
||||
public static final int COUNTER_NAME_MAX =
|
||||
conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT);
|
||||
public static final int GROUPS_MAX =
|
||||
conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
|
||||
public static final int COUNTERS_MAX =
|
||||
conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
|
||||
|
||||
private int totalCounters;
|
||||
private LimitExceededException firstViolation;
|
||||
|
||||
private static boolean isInited;
|
||||
|
||||
private static int GROUP_NAME_MAX;
|
||||
private static int COUNTER_NAME_MAX;
|
||||
private static int GROUPS_MAX;
|
||||
private static int COUNTERS_MAX;
|
||||
|
||||
public synchronized static void init(Configuration conf) {
|
||||
if (!isInited) {
|
||||
if (conf == null) {
|
||||
conf = new JobConf();
|
||||
}
|
||||
GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY,
|
||||
COUNTER_GROUP_NAME_MAX_DEFAULT);
|
||||
COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY,
|
||||
COUNTER_NAME_MAX_DEFAULT);
|
||||
GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
|
||||
COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
|
||||
}
|
||||
isInited = true;
|
||||
}
|
||||
|
||||
public static int getGroupNameMax() {
|
||||
if (!isInited) {
|
||||
init(null);
|
||||
}
|
||||
return GROUP_NAME_MAX;
|
||||
}
|
||||
|
||||
public static int getCounterNameMax() {
|
||||
if (!isInited) {
|
||||
init(null);
|
||||
}
|
||||
return COUNTER_NAME_MAX;
|
||||
}
|
||||
|
||||
public static int getGroupsMax() {
|
||||
if (!isInited) {
|
||||
init(null);
|
||||
}
|
||||
return GROUPS_MAX;
|
||||
}
|
||||
|
||||
public static int getCountersMax() {
|
||||
if (!isInited) {
|
||||
init(null);
|
||||
}
|
||||
return COUNTERS_MAX;
|
||||
}
|
||||
|
||||
public static String filterName(String name, int maxLen) {
|
||||
return name.length() > maxLen ? name.substring(0, maxLen - 1) : name;
|
||||
}
|
||||
|
||||
public static String filterCounterName(String name) {
|
||||
return filterName(name, COUNTER_NAME_MAX);
|
||||
return filterName(name, getCounterNameMax());
|
||||
}
|
||||
|
||||
public static String filterGroupName(String name) {
|
||||
return filterName(name, GROUP_NAME_MAX);
|
||||
return filterName(name, getGroupNameMax());
|
||||
}
|
||||
|
||||
public synchronized void checkCounters(int size) {
|
||||
if (firstViolation != null) {
|
||||
throw new LimitExceededException(firstViolation);
|
||||
}
|
||||
if (size > COUNTERS_MAX) {
|
||||
int countersMax = getCountersMax();
|
||||
if (size > countersMax) {
|
||||
firstViolation = new LimitExceededException("Too many counters: "+ size +
|
||||
" max="+ COUNTERS_MAX);
|
||||
" max="+ countersMax);
|
||||
throw firstViolation;
|
||||
}
|
||||
}
|
||||
|
@ -72,9 +115,10 @@ public class Limits {
|
|||
if (firstViolation != null) {
|
||||
throw new LimitExceededException(firstViolation);
|
||||
}
|
||||
if (size > GROUPS_MAX) {
|
||||
int groupsMax = getGroupsMax();
|
||||
if (size > groupsMax) {
|
||||
firstViolation = new LimitExceededException("Too many counter groups: "+
|
||||
size +" max="+ GROUPS_MAX);
|
||||
size +" max="+ groupsMax);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -101,8 +101,8 @@ public class TestCounters {
|
|||
static final long FS_COUNTER_VALUE = 10;
|
||||
|
||||
private void testMaxCounters(final Counters counters) {
|
||||
LOG.info("counters max="+ Limits.COUNTERS_MAX);
|
||||
for (int i = 0; i < Limits.COUNTERS_MAX; ++i) {
|
||||
LOG.info("counters max="+ Limits.getCountersMax());
|
||||
for (int i = 0; i < Limits.getCountersMax(); ++i) {
|
||||
counters.findCounter("test", "test"+ i);
|
||||
}
|
||||
setExpected(counters);
|
||||
|
@ -115,8 +115,8 @@ public class TestCounters {
|
|||
}
|
||||
|
||||
private void testMaxGroups(final Counters counters) {
|
||||
LOG.info("counter groups max="+ Limits.GROUPS_MAX);
|
||||
for (int i = 0; i < Limits.GROUPS_MAX; ++i) {
|
||||
LOG.info("counter groups max="+ Limits.getGroupsMax());
|
||||
for (int i = 0; i < Limits.getGroupsMax(); ++i) {
|
||||
// assuming COUNTERS_MAX > GROUPS_MAX
|
||||
counters.findCounter("test"+ i, "test");
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue