diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 60facec1284..480f8a32030 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -810,7 +810,7 @@ public abstract class TaskAttemptImpl implements ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext( commonContainerSpec.getUser(), commonContainerSpec.getLocalResources(), myEnv, commands, - myServiceData, commonContainerSpec.getContainerTokens().duplicate(), + myServiceData, commonContainerSpec.getTokens().duplicate(), applicationACLs); return container; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java index 54be1d74f0c..6704e009f31 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java @@ -123,7 +123,7 @@ public class TestTaskAttemptContainerRequest { Credentials launchCredentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); - dibb.reset(launchCtx.getContainerTokens()); + dibb.reset(launchCtx.getTokens()); launchCredentials.readTokenStorageStream(dibb); // verify all tokens specified for the task attempt are in the launch context diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3530933a7a1..7cfb6c05e05 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -124,6 +124,9 @@ Release 2.0.5-beta - UNRELEASED YARN-630. Changed AMRMProtocol api to throw IOException and YarnRemoteException. (Xuan Gong via vinodkv) + YARN-615. Rename ContainerLaunchContext.containerTokens to tokens. + (Vinod Kumar Vavilapalli via sseth) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index 36cfdfbadd8..da6f6d42378 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -68,20 +68,24 @@ public interface ContainerLaunchContext { void setUser(String user); /** - * Get security tokens (if security is enabled). - * @return security tokens (if security is enabled) + * Get all the tokens needed by this container. It may include file-system + * tokens, ApplicationMaster related tokens if this container is an + * ApplicationMaster or framework level tokens needed by this container to + * communicate to various services in a secure manner. + * + * @return tokens needed by this container. */ @Public @Stable - ByteBuffer getContainerTokens(); + ByteBuffer getTokens(); /** - * Set security tokens (if security is enabled). - * @param containerToken security tokens + * Set security tokens needed by this container. + * @param tokens security tokens */ @Public @Stable - void setContainerTokens(ByteBuffer containerToken); + void setTokens(ByteBuffer tokens); /** * Get LocalResource required by the container. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index 6a26508f711..5ab7e12fa55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -47,7 +47,7 @@ implements ContainerLaunchContext { boolean viaProto = false; private Map localResources = null; - private ByteBuffer containerTokens = null; + private ByteBuffer tokens = null; private Map serviceData = null; private Map environment = null; private List commands = null; @@ -73,8 +73,8 @@ implements ContainerLaunchContext { if (this.localResources != null) { addLocalResourcesToProto(); } - if (this.containerTokens != null) { - builder.setContainerTokens(convertToProtoFormat(this.containerTokens)); + if (this.tokens != null) { + builder.setTokens(convertToProtoFormat(this.tokens)); } if (this.serviceData != null) { addServiceDataToProto(); @@ -226,25 +226,25 @@ implements ContainerLaunchContext { } @Override - public ByteBuffer getContainerTokens() { + public ByteBuffer getTokens() { ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerTokens != null) { - return this.containerTokens; + if (this.tokens != null) { + return this.tokens; } - if (!p.hasContainerTokens()) { + if (!p.hasTokens()) { return null; } - this.containerTokens = convertFromProtoFormat(p.getContainerTokens()); - return this.containerTokens; + this.tokens = convertFromProtoFormat(p.getTokens()); + return this.tokens; } @Override - public void setContainerTokens(ByteBuffer containerTokens) { + public void setTokens(ByteBuffer tokens) { maybeInitBuilder(); - if (containerTokens == null) { - builder.clearContainerTokens(); + if (tokens == null) { + builder.clearTokens(); } - this.containerTokens = containerTokens; + this.tokens = tokens; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 23afa1159c1..f69aefb29ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -269,7 +269,7 @@ message QueueUserACLInfoProto { message ContainerLaunchContextProto { optional string user = 1; repeated StringLocalResourceMapProto localResources = 2; - optional bytes container_tokens = 3; + optional bytes tokens = 3; repeated StringBytesMapProto service_data = 4; repeated StringStringMapProto environment = 5; repeated string command = 6; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index e6699f39278..3e19069db5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -288,7 +288,7 @@ public class BuilderUtils { public static ContainerLaunchContext newContainerLaunchContext( String user, Map localResources, Map environment, List commands, - Map serviceData, ByteBuffer containerTokens, + Map serviceData, ByteBuffer tokens, Map acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); @@ -297,7 +297,7 @@ public class BuilderUtils { container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); - container.setContainerTokens(containerTokens); + container.setTokens(tokens); container.setApplicationACLs(acls); return container; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 68891103ed8..63cb07eb5d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -427,7 +427,7 @@ public class ContainerManagerImpl extends CompositeService implements + launchContext.getUser()); // //////////// Parse credentials - ByteBuffer tokens = launchContext.getContainerTokens(); + ByteBuffer tokens = launchContext.getTokens(); Credentials credentials = new Credentials(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 369c10afa7f..ebe4562271e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -316,7 +316,7 @@ public class RMAppManager implements EventHandler, throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); - ByteBuffer tokens = application.getAMContainerSpec().getContainerTokens(); + ByteBuffer tokens = application.getAMContainerSpec().getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index b87046be4ca..b5af79b162b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -195,9 +195,9 @@ public class AMLauncher implements Runnable { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); - if (container.getContainerTokens() != null) { + if (container.getTokens() != null) { // TODO: Don't do this kind of checks everywhere. - dibb.reset(container.getContainerTokens()); + dibb.reset(container.getTokens()); credentials.readTokenStorageStream(dibb); } @@ -209,7 +209,7 @@ public class AMLauncher implements Runnable { } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); - container.setContainerTokens(ByteBuffer.wrap(dob.getData(), 0, + container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); SecretKey clientSecretKey = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 0855a6fba3d..5ddd5b4aa86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -184,7 +184,7 @@ public class MockRM extends ResourceManager { DataOutputBuffer dob = new DataOutputBuffer(); ts.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - clc.setContainerTokens(securityTokens); + clc.setTokens(securityTokens); } sub.setAMContainerSpec(clc); req.setApplicationSubmissionContext(sub); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index bfa20927b2e..23ee9fa4954 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -78,7 +78,7 @@ public class TestAMAuthorization { public StartContainerResponse startContainer(StartContainerRequest request) throws YarnRemoteException { - amTokens = request.getContainerLaunchContext().getContainerTokens(); + amTokens = request.getContainerLaunchContext().getTokens(); return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 30ae6ce5246..b2b5205abe3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -474,7 +474,7 @@ public class TestRMRestart { securityTokens.rewind(); Assert.assertEquals(securityTokens, appState .getApplicationSubmissionContext().getAMContainerSpec() - .getContainerTokens()); + .getTokens()); // start new RM MockRM rm2 = new TestSecurityMockRM(conf, memStore);