MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1523607 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-09-16 12:20:30 +00:00
parent 3638f42a50
commit 7993c89ea8
4 changed files with 41 additions and 0 deletions

View File

@ -53,6 +53,8 @@ Release 2.1.1-beta - UNRELEASED
but just before ClientService to avoid race conditions during RM restart. but just before ClientService to avoid race conditions during RM restart.
(Jian He via vinodkv) (Jian He via vinodkv)
MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race

View File

@ -24,6 +24,7 @@ import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
@ -56,6 +57,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ReflectionUtils;
import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.JsonMappingException;
@ -405,6 +407,19 @@ class JobSubmitter {
// different job. // different job.
TokenCache.cleanUpTokenReferral(conf); TokenCache.cleanUpTokenReferral(conf);
if (conf.getBoolean(
MRJobConfig.JOB_TOKEN_TRACKING_IDS_ENABLED,
MRJobConfig.DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED)) {
// Add HDFS tracking ids
ArrayList<String> trackingIds = new ArrayList<String>();
for (Token<? extends TokenIdentifier> t :
job.getCredentials().getAllTokens()) {
trackingIds.add(t.decodeIdentifier().getTrackingId());
}
conf.setStrings(MRJobConfig.JOB_TOKEN_TRACKING_IDS,
trackingIds.toArray(new String[trackingIds.size()]));
}
// Write job file to submit dir // Write job file to submit dir
writeConf(conf, submitJobFile); writeConf(conf, submitJobFile);

View File

@ -308,6 +308,13 @@ public interface MRJobConfig {
public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY =
"mapreduce.job.credentials.binary"; "mapreduce.job.credentials.binary";
/* Configs for tracking ids of tokens used by a job */
public static final String JOB_TOKEN_TRACKING_IDS_ENABLED =
"mapreduce.job.token.tracking.ids.enabled";
public static final boolean DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED = false;
public static final String JOB_TOKEN_TRACKING_IDS =
"mapreduce.job.token.tracking.ids";
public static final String JOB_SUBMITHOST = public static final String JOB_SUBMITHOST =
"mapreduce.job.submithostname"; "mapreduce.job.submithostname";
public static final String JOB_SUBMITHOSTADDR = public static final String JOB_SUBMITHOSTADDR =

View File

@ -1130,6 +1130,23 @@
</description> </description>
</property> </property>
<property>
<name>mapreduce.job.token.tracking.ids.enabled</name>
<value>false</value>
<description>Whether to write tracking ids of tokens to
job-conf. When true, the configuration property
"mapreduce.job.token.tracking.ids" is set to the token-tracking-ids of
the job</description>
</property>
<property>
<name>mapreduce.job.token.tracking.ids</name>
<value></value>
<description>When mapreduce.job.token.tracking.ids.enabled is
set to true, this is set by the framework to the
token-tracking-ids used by the job.</description>
</property>
<property> <property>
<name>mapreduce.task.merge.progress.records</name> <name>mapreduce.task.merge.progress.records</name>
<value>10000</value> <value>10000</value>