diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0d448eabc56..7a2bf0f7b61 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -53,6 +53,8 @@ Release 2.1.1-beta - UNRELEASED but just before ClientService to avoid race conditions during RM restart. (Jian He via vinodkv) + MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu) + OPTIMIZATIONS MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index cee7a33d7e6..d83a3dd7ab9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -24,6 +24,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -56,6 +57,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.ReflectionUtils; import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.JsonMappingException; @@ -405,6 +407,19 @@ class JobSubmitter { // different job. TokenCache.cleanUpTokenReferral(conf); + if (conf.getBoolean( + MRJobConfig.JOB_TOKEN_TRACKING_IDS_ENABLED, + MRJobConfig.DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED)) { + // Add HDFS tracking ids + ArrayList trackingIds = new ArrayList(); + for (Token t : + job.getCredentials().getAllTokens()) { + trackingIds.add(t.decodeIdentifier().getTrackingId()); + } + conf.setStrings(MRJobConfig.JOB_TOKEN_TRACKING_IDS, + trackingIds.toArray(new String[trackingIds.size()])); + } + // Write job file to submit dir writeConf(conf, submitJobFile); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 142b5b7913f..694342ef2e1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -308,6 +308,13 @@ public interface MRJobConfig { public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; + /* Configs for tracking ids of tokens used by a job */ + public static final String JOB_TOKEN_TRACKING_IDS_ENABLED = + "mapreduce.job.token.tracking.ids.enabled"; + public static final boolean DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED = false; + public static final String JOB_TOKEN_TRACKING_IDS = + "mapreduce.job.token.tracking.ids"; + public static final String JOB_SUBMITHOST = "mapreduce.job.submithostname"; public static final String JOB_SUBMITHOSTADDR = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 34dece66448..f679ba19133 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1130,6 +1130,23 @@ + + mapreduce.job.token.tracking.ids.enabled + false + Whether to write tracking ids of tokens to + job-conf. When true, the configuration property + "mapreduce.job.token.tracking.ids" is set to the token-tracking-ids of + the job + + + + mapreduce.job.token.tracking.ids + + When mapreduce.job.token.tracking.ids.enabled is + set to true, this is set by the framework to the + token-tracking-ids used by the job. + + mapreduce.task.merge.progress.records 10000