diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0542f376920..df145b199cb 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -130,6 +130,9 @@ Release 2.1.0-beta - UNRELEASED ClusterMetrics for binary compatibility with 1.x APIs. (Mayank Bansal via vinodkv) + MAPREDUCE-5289. Updated MR App to use Token directly after YARN-717. (Jian He + via vinodkv) + OPTIMIZATIONS MAPREDUCE-4974. Optimising the LineRecordReader initialize() method diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherEvent.java index ce720e1dea5..c663566a960 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherEvent.java @@ -18,10 +18,10 @@ package org.apache.hadoop.mapreduce.v2.app.launcher; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; -import org.apache.hadoop.yarn.event.AbstractEvent; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.event.AbstractEvent; public class ContainerLauncherEvent extends AbstractEvent { @@ -29,12 +29,12 @@ public class ContainerLauncherEvent private TaskAttemptId taskAttemptID; private ContainerId containerID; private String containerMgrAddress; - private ContainerToken containerToken; + private Token containerToken; public ContainerLauncherEvent(TaskAttemptId taskAttemptID, ContainerId containerID, String containerMgrAddress, - ContainerToken containerToken, + Token containerToken, ContainerLauncher.EventType type) { super(type); this.taskAttemptID = taskAttemptID; @@ -55,7 +55,7 @@ public String getContainerMgrAddress() { return containerMgrAddress; } - public ContainerToken getContainerToken() { + public Token getContainerToken() { return containerToken; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index 3e64ad4bbf4..5911f3cd865 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.service.AbstractService; @@ -115,10 +114,11 @@ private class Container { private TaskAttemptId taskAttemptID; private ContainerId containerID; final private String containerMgrAddress; - private ContainerToken containerToken; + private org.apache.hadoop.yarn.api.records.Token containerToken; public Container(TaskAttemptId taId, ContainerId containerID, - String containerMgrAddress, ContainerToken containerToken) { + String containerMgrAddress, + org.apache.hadoop.yarn.api.records.Token containerToken) { this.state = ContainerState.PREP; this.taskAttemptID = taId; this.containerMgrAddress = containerMgrAddress; @@ -345,7 +345,8 @@ protected EventProcessor createEventProcessor(ContainerLauncherEvent event) { } protected ContainerManager getCMProxy(ContainerId containerID, - final String containerManagerBindAddr, ContainerToken containerToken) + final String containerManagerBindAddr, + org.apache.hadoop.yarn.api.records.Token containerToken) throws IOException { final InetSocketAddress cmAddr = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 05ea74f3984..ebf38b9c922 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -90,7 +90,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; @@ -516,7 +516,7 @@ public void handle(ContainerAllocatorEvent event) { ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, nodeId.toString(), "user", resource, System.currentTimeMillis() + 10000, 42, 42); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); Container container = Container.newInstance(cId, nodeId, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java index 8248a27df44..05111f926e5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java @@ -46,7 +46,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.junit.Test; /** @@ -226,7 +226,7 @@ public void handle(ContainerLauncherEvent event) { @Override protected ContainerManager getCMProxy(ContainerId contianerID, - String containerManagerBindAddr, ContainerToken containerToken) + String containerManagerBindAddr, Token containerToken) throws IOException { try { synchronized (this) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index 2291a75c5c3..90fa5225982 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -61,7 +61,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -347,7 +347,7 @@ protected ContainerLauncher createContainerLauncher(AppContext context) { return new ContainerLauncherImpl(context) { @Override protected ContainerManager getCMProxy(ContainerId containerID, - String containerManagerBindAddr, ContainerToken containerToken) + String containerManagerBindAddr, Token containerToken) throws IOException { // make proxy connect to our local containerManager server ContainerManager proxy = (ContainerManager) rpc.getProxy( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java index 4b3c2b38a43..bd50b9c6eca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java @@ -55,9 +55,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -407,7 +407,7 @@ public void testContainerCleaned() throws Exception { } } - private ContainerToken createNewContainerToken(ContainerId contId, + private Token createNewContainerToken(ContainerId contId, String containerManagerAddr) { long currentTime = System.currentTimeMillis(); return BuilderUtils.newContainerToken(NodeId.newInstance("127.0.0.1", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java index c5d7837a95e..86cdf3387d7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java @@ -20,7 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.Token; /** * The request issued by the client to the {@code ResourceManager} to cancel a @@ -29,6 +29,6 @@ @Public @Evolving public interface CancelDelegationTokenRequest { - DelegationToken getDelegationToken(); - void setDelegationToken(DelegationToken dToken); + Token getDelegationToken(); + void setDelegationToken(Token dToken); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/GetDelegationTokenResponse.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/GetDelegationTokenResponse.java index 67aca616647..930c6746fb7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/GetDelegationTokenResponse.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/GetDelegationTokenResponse.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.mapreduce.v2.api.protocolrecords; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.Token; public interface GetDelegationTokenResponse { - void setDelegationToken(DelegationToken clientDToken); - DelegationToken getDelegationToken(); + void setDelegationToken(Token clientDToken); + Token getDelegationToken(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java index af062fdcd5d..99eb2828ab3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java @@ -20,7 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.Token; /** * The request issued by the client to renew a delegation token from @@ -29,6 +29,6 @@ @Public @Evolving public interface RenewDelegationTokenRequest { - DelegationToken getDelegationToken(); - void setDelegationToken(DelegationToken dToken); + Token getDelegationToken(); + void setDelegationToken(Token dToken); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java index 8a4045552e2..9414a9a769e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java @@ -21,9 +21,9 @@ import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; public class CancelDelegationTokenRequestPBImpl extends ProtoBase implements @@ -44,10 +44,10 @@ public CancelDelegationTokenRequestPBImpl ( this.viaProto = true; } - DelegationToken token; + Token token; @Override - public DelegationToken getDelegationToken() { + public Token getDelegationToken() { CancelDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.token != null) { return this.token; @@ -57,7 +57,7 @@ public DelegationToken getDelegationToken() { } @Override - public void setDelegationToken(DelegationToken token) { + public void setDelegationToken(Token token) { maybeInitBuilder(); if (token == null) builder.clearToken(); @@ -95,11 +95,11 @@ private void maybeInitBuilder() { } - private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new DelegationTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(DelegationToken t) { - return ((DelegationTokenPBImpl)t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java index 69111645d99..bff52eab583 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java @@ -21,14 +21,14 @@ import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; public class GetDelegationTokenResponsePBImpl extends ProtoBase implements GetDelegationTokenResponse { - DelegationToken mrToken; + Token mrToken; GetDelegationTokenResponseProto proto = @@ -47,7 +47,7 @@ public GetDelegationTokenResponsePBImpl ( } @Override - public DelegationToken getDelegationToken() { + public Token getDelegationToken() { GetDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.mrToken != null) { return this.mrToken; @@ -60,7 +60,7 @@ public DelegationToken getDelegationToken() { } @Override - public void setDelegationToken(DelegationToken mrToken) { + public void setDelegationToken(Token mrToken) { maybeInitBuilder(); if (mrToken == null) builder.getToken(); @@ -97,11 +97,11 @@ private void maybeInitBuilder() { viaProto = false; } - private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new DelegationTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(DelegationToken t) { - return ((DelegationTokenPBImpl)t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java index 5b616b6b9a1..7abe34732d0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java @@ -21,9 +21,9 @@ import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; public class RenewDelegationTokenRequestPBImpl extends ProtoBase implements @@ -44,10 +44,10 @@ public RenewDelegationTokenRequestPBImpl( this.viaProto = true; } - DelegationToken token; + Token token; @Override - public DelegationToken getDelegationToken() { + public Token getDelegationToken() { RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.token != null) { return this.token; @@ -57,7 +57,7 @@ public DelegationToken getDelegationToken() { } @Override - public void setDelegationToken(DelegationToken token) { + public void setDelegationToken(Token token) { maybeInitBuilder(); if (token == null) builder.clearToken(); @@ -93,11 +93,11 @@ private void maybeInitBuilder() { viaProto = false; } - private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new DelegationTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(DelegationToken t) { - return ((DelegationTokenPBImpl) t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl) t).getProto(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java index 2415e45801c..723487b8a84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java @@ -36,7 +36,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; @@ -56,7 +55,7 @@ public boolean handleKind(Text kind) { public long renew(Token token, Configuration conf) throws IOException, InterruptedException { - DelegationToken dToken = BuilderUtils.newDelegationToken( + org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken( token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString()); @@ -77,7 +76,7 @@ public long renew(Token token, Configuration conf) throws IOException, public void cancel(Token token, Configuration conf) throws IOException, InterruptedException { - DelegationToken dToken = BuilderUtils.newDelegationToken( + org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken( token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index b5add9bbda2..e1d56f865c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -79,7 +79,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -344,7 +343,7 @@ public GetDelegationTokenResponse getDelegationToken( Token realJHSToken = new Token(tokenIdentifier, jhsDTSecretManager); - DelegationToken mrDToken = BuilderUtils.newDelegationToken( + org.apache.hadoop.yarn.api.records.Token mrDToken = BuilderUtils.newDelegationToken( realJHSToken.getIdentifier(), realJHSToken.getKind().toString(), realJHSToken.getPassword(), realJHSToken.getService().toString()); response.setDelegationToken(mrDToken); @@ -359,7 +358,7 @@ public RenewDelegationTokenResponse renewDelegationToken( "Delegation Token can be renewed only with kerberos authentication"); } - DelegationToken protoToken = request.getDelegationToken(); + org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken(); Token token = new Token( protoToken.getIdentifier().array(), protoToken.getPassword() @@ -382,7 +381,7 @@ public CancelDelegationTokenResponse cancelDelegationToken( "Delegation Token can be cancelled only with kerberos authentication"); } - DelegationToken protoToken = request.getDelegationToken(); + org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken(); Token token = new Token( protoToken.getIdentifier().array(), protoToken.getPassword() diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index f216d3d3f27..1e5cd301b50 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -33,7 +33,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -64,13 +63,11 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -78,7 +75,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ProtoUtils; public class ClientServiceDelegate { @@ -184,7 +180,7 @@ private MRClientProtocol getProxy() throws IOException { serviceAddr = NetUtils.createSocketAddrForHost( application.getHost(), application.getRpcPort()); if (UserGroupInformation.isSecurityEnabled()) { - ClientToken clientToken = application.getClientToken(); + org.apache.hadoop.yarn.api.records.Token clientToken = application.getClientToken(); Token token = ProtoUtils.convertFromProtoFormat(clientToken, serviceAddr); newUgi.addToken(token); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 14b12b05a65..4d950cd6e06 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -72,7 +72,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -207,7 +206,7 @@ Token getDelegationTokenFromHS(MRClientProtocol hsProxy) GetDelegationTokenRequest request = recordFactory .newRecordInstance(GetDelegationTokenRequest.class); request.setRenewer(Master.getMasterPrincipal(conf)); - DelegationToken mrDelegationToken; + org.apache.hadoop.yarn.api.records.Token mrDelegationToken; mrDelegationToken = hsProxy.getDelegationToken(request) .getDelegationToken(); return ProtoUtils.convertFromProtoFormat(mrDelegationToken, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 4063f38e81d..6b652037ccb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -79,7 +79,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -283,7 +282,7 @@ public void testGetHSDelegationToken() throws Exception { token.setKind(RMDelegationTokenIdentifier.KIND_NAME); // Setup mock history token - DelegationToken historyToken = BuilderUtils.newDelegationToken( + org.apache.hadoop.yarn.api.records.Token historyToken = BuilderUtils.newDelegationToken( new byte[0], MRDelegationTokenIdentifier.KIND_NAME.toString(), new byte[0], hsTokenSevice.toString()); GetDelegationTokenResponse getDtResponse = Records @@ -359,8 +358,8 @@ public GetDelegationTokenResponse answer(InvocationOnMock invocation) { // check that the renewer matches the cluster's RM principal assertEquals(masterPrincipal, request.getRenewer() ); - DelegationToken token = - recordFactory.newRecordInstance(DelegationToken.class); + org.apache.hadoop.yarn.api.records.Token token = + recordFactory.newRecordInstance(org.apache.hadoop.yarn.api.records.Token.class); // none of these fields matter for the sake of the test token.setKind(""); token.setService(""); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java index 49c5dc88a64..cc23475fbef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -98,8 +97,8 @@ public void testClusterGetDelegationToken() throws Exception { YARNRunner yrunner = (YARNRunner) cluster.getClient(); GetDelegationTokenResponse getDTResponse = recordFactory.newRecordInstance(GetDelegationTokenResponse.class); - DelegationToken rmDTToken = recordFactory.newRecordInstance( - DelegationToken.class); + org.apache.hadoop.yarn.api.records.Token rmDTToken = recordFactory.newRecordInstance( + org.apache.hadoop.yarn.api.records.Token.class); rmDTToken.setIdentifier(ByteBuffer.wrap(new byte[2])); rmDTToken.setKind("Testclusterkind"); rmDTToken.setPassword(ByteBuffer.wrap("testcluster".getBytes())); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java index 5f37b300c0c..2181b6b0402 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java @@ -45,7 +45,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.util.ProtoUtils; @@ -107,7 +107,7 @@ protected JHSDelegationTokenSecretManager createJHSSecretManager( loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); - DelegationToken token = getDelegationToken(loggedInUser, hsService, + Token token = getDelegationToken(loggedInUser, hsService, loggedInUser.getShortUserName()); tokenFetchTime = System.currentTimeMillis(); LOG.info("Got delegation token at: " + tokenFetchTime); @@ -207,16 +207,16 @@ protected JHSDelegationTokenSecretManager createJHSSecretManager( } } - private DelegationToken getDelegationToken( + private Token getDelegationToken( final UserGroupInformation loggedInUser, final MRClientProtocol hsService, final String renewerString) throws IOException, InterruptedException { // Get the delegation token directly as it is a little difficult to setup // the kerberos based rpc. - DelegationToken token = loggedInUser - .doAs(new PrivilegedExceptionAction() { + Token token = loggedInUser + .doAs(new PrivilegedExceptionAction() { @Override - public DelegationToken run() throws IOException { + public Token run() throws IOException { GetDelegationTokenRequest request = Records .newRecord(GetDelegationTokenRequest.class); request.setRenewer(renewerString); @@ -228,7 +228,7 @@ public DelegationToken run() throws IOException { } private long renewDelegationToken(final UserGroupInformation loggedInUser, - final MRClientProtocol hsService, final DelegationToken dToken) + final MRClientProtocol hsService, final Token dToken) throws IOException, InterruptedException { long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction() { @@ -244,7 +244,7 @@ public Long run() throws IOException { } private void cancelDelegationToken(final UserGroupInformation loggedInUser, - final MRClientProtocol hsService, final DelegationToken dToken) + final MRClientProtocol hsService, final Token dToken) throws IOException, InterruptedException { loggedInUser.doAs(new PrivilegedExceptionAction() { @@ -259,7 +259,7 @@ public Void run() throws IOException { }); } - private MRClientProtocol getMRClientProtocol(DelegationToken token, + private MRClientProtocol getMRClientProtocol(Token token, final InetSocketAddress hsAddress, String user, final Configuration conf) { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.addToken(ProtoUtils.convertFromProtoFormat(token, hsAddress)); diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f9a27d00afc..902554372ab 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -221,6 +221,9 @@ Release 2.1.0-beta - UNRELEASED YARN-660. Improve AMRMClient with matching requests (bikas) + YARN-717. Put object creation factories for Token in the class itself and + remove useless derivations for specific tokens. (Jian He via vinodkv) + OPTIMIZATIONS YARN-512. Log aggregation root directory check is more expensive than it diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java index 5824952f4a0..6bef8cb417f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java @@ -50,9 +50,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -273,7 +273,7 @@ public GetQueueUserAclsInfoResponse getQueueUserAcls( * containers to be able to talk to the service using those tokens. * *

The ResourceManager responds with the delegation token - * {@link DelegationToken} that can be used by the client to speak to this + * {@link Token} that can be used by the client to speak to this * service. * @param request request to get a delegation token for the client. * @return delegation token that can be used to talk to this service diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CancelDelegationTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CancelDelegationTokenRequest.java index d9832ba73ba..1217878ec08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CancelDelegationTokenRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CancelDelegationTokenRequest.java @@ -20,7 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.Token; /** * The request issued by the client to the {@code ResourceManager} to cancel a @@ -29,6 +29,6 @@ @Public @Evolving public interface CancelDelegationTokenRequest { - DelegationToken getDelegationToken(); - void setDelegationToken(DelegationToken dToken); + Token getDelegationToken(); + void setDelegationToken(Token dToken); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenResponse.java index 7eb75c55e2f..776361eb113 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenResponse.java @@ -20,7 +20,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.yarn.api.records.Token; /** @@ -32,6 +33,12 @@ @Public @Evolving public interface GetDelegationTokenResponse { - DelegationToken getRMDelegationToken(); - void setRMDelegationToken(DelegationToken rmDTToken); + + /** + * The Delegation tokens have a identifier which maps to + * {@link AbstractDelegationTokenIdentifier}. + * + */ + Token getRMDelegationToken(); + void setRMDelegationToken(Token rmDTToken); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RenewDelegationTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RenewDelegationTokenRequest.java index b89f18d9317..1eb768c832a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RenewDelegationTokenRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RenewDelegationTokenRequest.java @@ -20,7 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.Token; /** * The request issued by the client to renew a delegation token from @@ -29,6 +29,6 @@ @Public @Evolving public interface RenewDelegationTokenRequest { - DelegationToken getDelegationToken(); - void setDelegationToken(DelegationToken dToken); + Token getDelegationToken(); + void setDelegationToken(Token dToken); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java index ca161d586b8..6b73d142082 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; /** *

The request sent by the ApplicationMaster to the @@ -62,9 +62,9 @@ public interface StartContainerRequest { @Public @Stable - public ContainerToken getContainerToken(); + public Token getContainerToken(); @Public @Stable - public void setContainerToken(ContainerToken container); + public void setContainerToken(Token container); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java index a38f94a7ede..e1d87289dae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java @@ -21,9 +21,9 @@ import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; public class CancelDelegationTokenRequestPBImpl extends ProtoBase implements @@ -44,10 +44,10 @@ public CancelDelegationTokenRequestPBImpl( viaProto = true; } - DelegationToken token; + Token token; @Override - public DelegationToken getDelegationToken() { + public Token getDelegationToken() { CancelDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.token != null) { return this.token; @@ -57,7 +57,7 @@ public DelegationToken getDelegationToken() { } @Override - public void setDelegationToken(DelegationToken token) { + public void setDelegationToken(Token token) { maybeInitBuilder(); if (token == null) builder.clearToken(); @@ -93,11 +93,11 @@ private void maybeInitBuilder() { viaProto = false; } - private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new DelegationTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(DelegationToken t) { - return ((DelegationTokenPBImpl) t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl) t).getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java index 76c31b40c8b..59651ff0ed5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java @@ -22,14 +22,14 @@ import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; public class GetDelegationTokenResponsePBImpl extends ProtoBase implements GetDelegationTokenResponse { - DelegationToken appToken; + Token appToken; GetDelegationTokenResponseProto proto = @@ -48,7 +48,7 @@ public GetDelegationTokenResponsePBImpl ( } @Override - public DelegationToken getRMDelegationToken() { + public Token getRMDelegationToken() { GetDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.appToken != null) { return this.appToken; @@ -61,7 +61,7 @@ public DelegationToken getRMDelegationToken() { } @Override - public void setRMDelegationToken(DelegationToken appToken) { + public void setRMDelegationToken(Token appToken) { maybeInitBuilder(); if (appToken == null) builder.clearToken(); @@ -99,11 +99,11 @@ private void maybeInitBuilder() { } - private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new DelegationTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(DelegationToken t) { - return ((DelegationTokenPBImpl)t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java index cf02e849085..963a714e8f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java @@ -21,9 +21,9 @@ import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; public class RenewDelegationTokenRequestPBImpl extends ProtoBase implements @@ -43,10 +43,10 @@ public RenewDelegationTokenRequestPBImpl ( this.viaProto = true; } - DelegationToken token; + Token token; @Override - public DelegationToken getDelegationToken() { + public Token getDelegationToken() { RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.token != null) { return this.token; @@ -56,7 +56,7 @@ public DelegationToken getDelegationToken() { } @Override - public void setDelegationToken(DelegationToken token) { + public void setDelegationToken(Token token) { maybeInitBuilder(); if (token == null) builder.clearToken(); @@ -94,11 +94,11 @@ private void maybeInitBuilder() { } - private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new DelegationTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(DelegationToken t) { - return ((DelegationTokenPBImpl)t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java index 5073f0f02c9..fdb7e8854bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java @@ -22,10 +22,10 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.ProtoBase; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerTokenPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProtoOrBuilder; @@ -39,7 +39,7 @@ public class StartContainerRequestPBImpl extends ProtoBaseclient token for communicating with the * ApplicationMaster. + *

+ * ClientToken is the security token used by the AMs to verify + * authenticity of any client. + *

+ * + *

+ * The ResourceManager, provides a secure token (via + * {@link ApplicationReport#getClientToken()}) which is verified by the + * ApplicationMaster when the client directly talks to an AM. + *

* @return client token for communicating with the * ApplicationMaster */ @Public @Stable - public abstract ClientToken getClientToken(); + public abstract Token getClientToken(); @Private @Unstable - public abstract void setClientToken(ClientToken clientToken); + public abstract void setClientToken(Token clientToken); /** * Get the YarnApplicationState of the application. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ClientToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ClientToken.java deleted file mode 100644 index 92c4d9eb5d3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ClientToken.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; - -/** - *

- * ClientToken is the security token used by the AMs to verify - * authenticity of any client. - *

- * - *

- * The ResourceManager, provides a secure token (via - * {@link ApplicationReport#getClientToken()}) which is verified by the - * ApplicationMaster when the client directly talks to an AM. - *

- * - */ -@Public -@Stable -public interface ClientToken extends Token {} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 1de48b0026c..2a2050a754f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -46,7 +46,7 @@ *
  • {@link Priority} at which the container was allocated.
    • *
  • {@link ContainerState} of the container.
    • *
  • - * {@link ContainerToken} of the container, used to securely verify + * Container Token {@link Token} of the container, used to securely verify * authenticity of the allocation. *
    • *
  • {@link ContainerStatus} of the container.
    • @@ -69,7 +69,7 @@ public abstract class Container implements Comparable { @Private public static Container newInstance(ContainerId containerId, NodeId nodeId, String nodeHttpAddress, Resource resource, Priority priority, - ContainerToken containerToken) { + Token containerToken) { Container container = Records.newRecord(Container.class); container.setId(containerId); container.setNodeId(nodeId); @@ -142,13 +142,27 @@ public static Container newInstance(ContainerId containerId, NodeId nodeId, /** * Get the ContainerToken for the container. + *

    ContainerToken is the security token used by the framework + * to verify authenticity of any Container.

    + * + *

    The ResourceManager, on container allocation provides a + * secure token which is verified by the NodeManager on + * container launch.

    + * + *

    Applications do not need to care about ContainerToken, they + * are transparently handled by the framework - the allocated + * Container includes the ContainerToken.

    + * + * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) + * * @return ContainerToken for the container */ @Public @Stable - public abstract ContainerToken getContainerToken(); + public abstract Token getContainerToken(); @Private @Unstable - public abstract void setContainerToken(ContainerToken containerToken); + public abstract void setContainerToken(Token containerToken); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java deleted file mode 100644 index 76ea73d8aac..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.AMRMProtocol; -import org.apache.hadoop.yarn.api.ContainerManager; - -/** - *

    ContainerToken is the security token used by the framework - * to verify authenticity of any Container.

    - * - *

    The ResourceManager, on container allocation provides a - * secure token which is verified by the NodeManager on - * container launch.

    - * - *

    Applications do not need to care about ContainerToken, they - * are transparently handled by the framework - the allocated - * Container includes the ContainerToken.

    - * - * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) - * @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) - */ -@Public -@Stable -public interface ContainerToken extends Token {} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DelegationToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DelegationToken.java deleted file mode 100644 index a7870df930b..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DelegationToken.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; - -/** - * The Delegation tokens have a identifier which maps to - * {@link AbstractDelegationTokenIdentifier}. - * - */ -@Public -@Evolving -public interface DelegationToken extends Token {} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Token.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Token.java index addc234e036..8dfa376576f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Token.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Token.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.util.Records; /** *

    Token is the security entity used by the framework @@ -30,18 +31,30 @@ */ @Public @Stable -public interface Token { +public abstract class Token { + + @Private + public static Token newInstance(byte[] identifier, String kind, byte[] password, + String service) { + Token token = Records.newRecord(Token.class); + token.setIdentifier(ByteBuffer.wrap(identifier)); + token.setKind(kind); + token.setPassword(ByteBuffer.wrap(password)); + token.setService(service); + return token; + } + /** * Get the token identifier. * @return token identifier */ @Public @Stable - ByteBuffer getIdentifier(); + public abstract ByteBuffer getIdentifier(); @Private @Stable - void setIdentifier(ByteBuffer identifier); + public abstract void setIdentifier(ByteBuffer identifier); /** * Get the token password @@ -49,11 +62,11 @@ public interface Token { */ @Public @Stable - ByteBuffer getPassword(); + public abstract ByteBuffer getPassword(); @Private @Stable - void setPassword(ByteBuffer password); + public abstract void setPassword(ByteBuffer password); /** * Get the token kind. @@ -61,11 +74,11 @@ public interface Token { */ @Public @Stable - String getKind(); + public abstract String getKind(); @Private @Stable - void setKind(String kind); + public abstract void setKind(String kind); /** * Get the service to which the token is allocated. @@ -73,10 +86,10 @@ public interface Token { */ @Public @Stable - String getService(); + public abstract String getService(); @Private @Stable - void setService(String service); + public abstract void setService(String service); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index a4ed75041cc..ad8a7912f68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -23,8 +23,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; @@ -42,7 +42,7 @@ public class ApplicationReportPBImpl extends ApplicationReport { private ApplicationId applicationId; private ApplicationAttemptId currentApplicationAttemptId; - private ClientToken clientToken = null; + private Token clientToken = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -160,7 +160,7 @@ public int getRpcPort() { } @Override - public ClientToken getClientToken() { + public Token getClientToken() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; if (this.clientToken != null) { return this.clientToken; @@ -309,7 +309,7 @@ public void setRpcPort(int rpcPort) { } @Override - public void setClientToken(ClientToken clientToken) { + public void setClientToken(Token clientToken) { maybeInitBuilder(); if (clientToken == null) builder.clearClientToken(); @@ -413,7 +413,7 @@ private void mergeLocalToBuilder() { builder.setCurrentApplicationAttemptId(convertToProtoFormat(this.currentApplicationAttemptId)); } if (this.clientToken != null - && !((ClientTokenPBImpl) this.clientToken).getProto().equals( + && !((TokenPBImpl) this.clientToken).getProto().equals( builder.getClientToken())) { builder.setClientToken(convertToProtoFormat(this.clientToken)); } @@ -476,11 +476,11 @@ private FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus return ProtoUtils.convertToProtoFormat(s); } - private ClientTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new ClientTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(ClientToken t) { - return ((ClientTokenPBImpl)t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ClientTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ClientTokenPBImpl.java deleted file mode 100644 index 6f5c52857e7..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ClientTokenPBImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records.impl.pb; - -import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; -import org.apache.hadoop.yarn.api.records.ClientToken; - -public class ClientTokenPBImpl extends TokenPBImpl implements ClientToken { - - public ClientTokenPBImpl() { - super(); - } - - public ClientTokenPBImpl(TokenProto p) { - super(p); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index ae4dadad45a..22bac0298bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -21,10 +21,10 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProtoOrBuilder; @@ -42,7 +42,7 @@ public class ContainerPBImpl extends Container { private NodeId nodeId = null; private Resource resource = null; private Priority priority = null; - private ContainerToken containerToken = null; + private Token containerToken = null; public ContainerPBImpl() { builder = ContainerProto.newBuilder(); @@ -98,7 +98,7 @@ private void mergeLocalToBuilder() { builder.setPriority(convertToProtoFormat(this.priority)); } if (this.containerToken != null - && !((ContainerTokenPBImpl) this.containerToken).getProto().equals( + && !((TokenPBImpl) this.containerToken).getProto().equals( builder.getContainerToken())) { builder.setContainerToken(convertToProtoFormat(this.containerToken)); } @@ -224,7 +224,7 @@ public void setPriority(Priority priority) { } @Override - public ContainerToken getContainerToken() { + public Token getContainerToken() { ContainerProtoOrBuilder p = viaProto ? proto : builder; if (this.containerToken != null) { return this.containerToken; @@ -237,7 +237,7 @@ public ContainerToken getContainerToken() { } @Override - public void setContainerToken(ContainerToken containerToken) { + public void setContainerToken(Token containerToken) { maybeInitBuilder(); if (containerToken == null) builder.clearContainerToken(); @@ -276,12 +276,12 @@ private PriorityProto convertToProtoFormat(Priority p) { return ((PriorityPBImpl)p).getProto(); } - private ContainerTokenPBImpl convertFromProtoFormat(TokenProto p) { - return new ContainerTokenPBImpl(p); + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); } - private TokenProto convertToProtoFormat(ContainerToken t) { - return ((ContainerTokenPBImpl)t).getProto(); + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); } public String toString() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java deleted file mode 100644 index 87676c59613..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records.impl.pb; - -import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; -import org.apache.hadoop.yarn.api.records.ContainerToken; - -public class ContainerTokenPBImpl extends TokenPBImpl implements ContainerToken { - - public ContainerTokenPBImpl() { - super(); - } - - public ContainerTokenPBImpl(TokenProto p) { - super(p); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/DelegationTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/DelegationTokenPBImpl.java deleted file mode 100644 index 4901cde7d00..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/DelegationTokenPBImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records.impl.pb; - -import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; -import org.apache.hadoop.yarn.api.records.DelegationToken; - -public class DelegationTokenPBImpl extends TokenPBImpl implements - DelegationToken { - - public DelegationTokenPBImpl() { - super(); - } - - public DelegationTokenPBImpl(TokenProto p) { - super(p); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java index fe304e03ada..b58df8267ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java @@ -22,11 +22,12 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.proto.SecurityProtos.TokenProtoOrBuilder; -import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.util.ProtoUtils; -public class TokenPBImpl extends ProtoBase implements - Token { +import com.google.protobuf.ByteString; + +public class TokenPBImpl extends Token { private TokenProto proto = TokenProto.getDefaultInstance(); private TokenProto.Builder builder = null; private boolean viaProto = false; @@ -50,6 +51,29 @@ public synchronized TokenProto getProto() { return proto; } + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { + return ProtoUtils.convertFromProtoFormat(byteString); + } + + protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { + return ProtoUtils.convertToProtoFormat(byteBuffer); + } + private synchronized void mergeLocalToBuilder() { if (this.identifier != null) { builder.setIdentifier(convertToProtoFormat(this.identifier)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClient.java index b3563a0c245..d5e94d59585 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClient.java @@ -28,8 +28,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.service.Service; @@ -69,7 +69,7 @@ Map startContainer(Container container, * @throws IOException */ void stopContainer(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) throws YarnRemoteException, IOException; + Token containerToken) throws YarnRemoteException, IOException; /** *

    Query the status of a container.

    @@ -83,7 +83,7 @@ void stopContainer(ContainerId containerId, NodeId nodeId, * @throws IOException */ ContainerStatus getContainerStatus(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) throws YarnRemoteException, IOException; + Token containerToken) throws YarnRemoteException, IOException; /** *

    Set whether the containers that are started by this client, and are diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientAsync.java index 44d6f345d8c..becb3571043 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientAsync.java @@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AbstractEvent; @@ -288,7 +288,7 @@ public void startContainer( } public void stopContainer(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) { + Token containerToken) { if (containers.get(containerId) == null) { callbackHandler.onStopContainerError(containerId, RPCUtil.getRemoteException("Container " + containerId + @@ -305,7 +305,7 @@ public void stopContainer(ContainerId containerId, NodeId nodeId, } public void getContainerStatus(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) { + Token containerToken) { try { events.put(new ContainerEvent(containerId, nodeId, containerToken, ContainerEventType.QUERY_CONTAINER)); @@ -343,10 +343,10 @@ protected static class ContainerEvent extends AbstractEvent{ private ContainerId containerId; private NodeId nodeId; - private ContainerToken containerToken; + private Token containerToken; public ContainerEvent(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken, ContainerEventType type) { + Token containerToken, ContainerEventType type) { super(type); this.containerId = containerId; this.nodeId = nodeId; @@ -361,7 +361,7 @@ public NodeId getNodeId() { return nodeId; } - public ContainerToken getContainerToken() { + public Token getContainerToken() { return containerToken; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java index 2bf9c93a1bc..4cf12b07d6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java @@ -32,7 +32,6 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; @@ -43,7 +42,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -122,11 +121,11 @@ public void cleanupRunningContainersOnStop(boolean enabled) { protected static class StartedContainer { private ContainerId containerId; private NodeId nodeId; - private ContainerToken containerToken; + private Token containerToken; private boolean stopped; public StartedContainer(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) { + Token containerToken) { this.containerId = containerId; this.nodeId = nodeId; this.containerToken = containerToken; @@ -141,7 +140,7 @@ public NodeId getNodeId() { return nodeId; } - public ContainerToken getContainerToken() { + public Token getContainerToken() { return containerToken; } } @@ -149,11 +148,11 @@ public ContainerToken getContainerToken() { protected static final class NMCommunicator extends AbstractService { private ContainerId containerId; private NodeId nodeId; - private ContainerToken containerToken; + private Token containerToken; private ContainerManager containerManager; public NMCommunicator(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) { + Token containerToken) { super(NMCommunicator.class.getName()); this.containerId = containerId; this.nodeId = nodeId; @@ -171,7 +170,7 @@ public synchronized void start() { UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(containerId.toString()); - Token token = + org.apache.hadoop.security.token.Token token = ProtoUtils.convertFromProtoFormat(containerToken, containerAddress); currentUser.addToken(token); @@ -316,7 +315,7 @@ public Map startContainer( @Override public void stopContainer(ContainerId containerId, NodeId nodeId, - ContainerToken containerToken) throws YarnRemoteException, IOException { + Token containerToken) throws YarnRemoteException, IOException { StartedContainer startedContainer = getStartedContainer(containerId); if (startedContainer == null) { throw RPCUtil.getRemoteException("Container " + containerId + @@ -348,7 +347,7 @@ public void stopContainer(ContainerId containerId, NodeId nodeId, @Override public ContainerStatus getContainerStatus(ContainerId containerId, - NodeId nodeId, ContainerToken containerToken) + NodeId nodeId, Token containerToken) throws YarnRemoteException, IOException { NMCommunicator nmCommunicator = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java index adbcf677e6b..b7cf5a07171 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java @@ -28,10 +28,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.service.Service; @@ -179,12 +179,12 @@ YarnClusterMetrics getYarnClusterMetrics() throws YarnRemoteException, * @param renewer * Address of the renewer who can renew these tokens when needed by * securely talking to YARN. - * @return a delegation token ({@link DelegationToken}) that can be used to + * @return a delegation token ({@link Token}) that can be used to * talk to YARN * @throws YarnRemoteException * @throws IOException */ - DelegationToken getRMDelegationToken(Text renewer) + Token getRMDelegationToken(Text renewer) throws YarnRemoteException, IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java index 556cb357a0a..4eb88850c7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java @@ -50,15 +50,14 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; -import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.util.Records; @@ -215,7 +214,7 @@ public List getNodeReports() throws YarnRemoteException, } @Override - public DelegationToken getRMDelegationToken(Text renewer) + public Token getRMDelegationToken(Text renewer) throws YarnRemoteException, IOException { /* get the token from RM */ GetDelegationTokenRequest rmDTRequest = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestNMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestNMClientAsync.java index 0e4c90a1a7f..bcbf94d7738 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestNMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestNMClientAsync.java @@ -45,7 +45,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -63,7 +63,7 @@ public class TestNMClientAsync { private NMClientAsync asyncClient; private NodeId nodeId; - private ContainerToken containerToken; + private Token containerToken; @Test (timeout = 30000) public void testNMClientAsync() throws Exception { @@ -368,10 +368,10 @@ private NMClient mockNMClient(int mode) any(ContainerLaunchContext.class))).thenReturn( Collections.emptyMap()); when(client.getContainerStatus(any(ContainerId.class), any(NodeId.class), - any(ContainerToken.class))).thenReturn( + any(Token.class))).thenReturn( recordFactory.newRecordInstance(ContainerStatus.class)); doNothing().when(client).stopContainer(any(ContainerId.class), - any(NodeId.class), any(ContainerToken.class)); + any(NodeId.class), any(Token.class)); break; case 1: doThrow(RPCUtil.getRemoteException("Start Exception")).when(client) @@ -379,21 +379,21 @@ private NMClient mockNMClient(int mode) any(ContainerLaunchContext.class)); doThrow(RPCUtil.getRemoteException("Query Exception")).when(client) .getContainerStatus(any(ContainerId.class), any(NodeId.class), - any(ContainerToken.class)); + any(Token.class)); doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client) .stopContainer(any(ContainerId.class), any(NodeId.class), - any(ContainerToken.class)); + any(Token.class)); break; case 2: when(client.startContainer(any(Container.class), any(ContainerLaunchContext.class))).thenReturn( Collections.emptyMap()); when(client.getContainerStatus(any(ContainerId.class), any(NodeId.class), - any(ContainerToken.class))).thenReturn( + any(Token.class))).thenReturn( recordFactory.newRecordInstance(ContainerStatus.class)); doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client) .stopContainer(any(ContainerId.class), any(NodeId.class), - any(ContainerToken.class)); + any(Token.class)); } return client; } @@ -532,9 +532,8 @@ private Container mockContainer(int i) { ContainerId containerId = ContainerId.newInstance(attemptId, i); nodeId = NodeId.newInstance("localhost", 0); // Create an empty record - containerToken = recordFactory.newRecordInstance(ContainerToken.class); + containerToken = recordFactory.newRecordInstance(Token.class); return BuilderUtils.newContainer(containerId, nodeId, null, null, null, containerToken); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java index 79267952378..54a8117e656 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.util.BuilderUtils; @@ -164,7 +163,8 @@ private static String getRenewer(Token token) throws IOException { .getRenewer().toString(); } - private static DelegationToken convertToProtoToken(Token token) { + private static org.apache.hadoop.yarn.api.records.Token + convertToProtoToken(Token token) { return BuilderUtils.newDelegationToken( token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 943b7c1e0ab..4793288cdcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -38,14 +38,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -169,7 +166,7 @@ public static ContainerId newContainerId(int appId, int appAttemptId, return cId; } - public static ContainerToken newContainerToken(ContainerId cId, String host, + public static Token newContainerToken(ContainerId cId, String host, int port, String user, Resource r, long expiryTime, int masterKeyId, byte[] password, long rmIdentifier) throws IOException { ContainerTokenIdentifier identifier = @@ -217,7 +214,7 @@ public static ContainerStatus newContainerStatus(ContainerId containerId, public static Container newContainer(ContainerId containerId, NodeId nodeId, String nodeHttpAddress, Resource resource, Priority priority, - ContainerToken containerToken) { + Token containerToken) { Container container = recordFactory.newRecordInstance(Container.class); container.setId(containerId); container.setNodeId(nodeId); @@ -238,31 +235,31 @@ public static T newToken(Class tokenClass, return token; } - public static DelegationToken newDelegationToken(byte[] identifier, + public static Token newDelegationToken(byte[] identifier, String kind, byte[] password, String service) { - return newToken(DelegationToken.class, identifier, kind, password, service); + return newToken(Token.class, identifier, kind, password, service); } - public static ClientToken newClientToken(byte[] identifier, String kind, + public static Token newClientToken(byte[] identifier, String kind, byte[] password, String service) { - return newToken(ClientToken.class, identifier, kind, password, service); + return newToken(Token.class, identifier, kind, password, service); } - public static ContainerToken newContainerToken(NodeId nodeId, + public static Token newContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens InetSocketAddress addr = NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token - ContainerToken containerToken = - newToken(ContainerToken.class, tokenIdentifier.getBytes(), + Token containerToken = + newToken(Token.class, tokenIdentifier.getBytes(), ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil .buildTokenService(addr).toString()); return containerToken; } public static ContainerTokenIdentifier newContainerTokenIdentifier( - ContainerToken containerToken) throws IOException { + Token containerToken) throws IOException { org.apache.hadoop.security.token.Token token = new org.apache.hadoop.security.token.Token( containerToken.getIdentifier() @@ -318,7 +315,7 @@ public static ResourceRequest newResourceRequest(ResourceRequest r) { public static ApplicationReport newApplicationReport( ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, - ClientToken clientToken, YarnApplicationState state, String diagnostics, + Token clientToken, YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index 10d24472052..e3a5d2d82f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -42,9 +42,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -101,7 +101,7 @@ private void testRPCTimeout(String rpcClass) throws Exception { ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 013c9b715d8..050879a71db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -44,9 +44,9 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -129,7 +129,7 @@ private void test(String rpcClass) throws Exception { ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); scRequest.setContainerToken(containerToken); @@ -179,7 +179,7 @@ public GetContainerStatusResponse getContainerStatus( @Override public StartContainerResponse startContainer(StartContainerRequest request) throws YarnRemoteException { - ContainerToken containerToken = request.getContainerToken(); + Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 6a7bd683f3d..cb9b98849a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -401,7 +400,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) } ContainerLaunchContext launchContext = request.getContainerLaunchContext(); - ContainerToken token = request.getContainerToken(); + org.apache.hadoop.yarn.api.records.Token token = request.getContainerToken(); ContainerTokenIdentifier tokenIdentifier = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index b4695960a99..44a51858b2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -135,7 +135,7 @@ public long getRMIdentifier() { String user = "testing"; String host = "127.0.0.1"; int port = 1234; - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cID, host, port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), SIMULATED_RM_IDENTIFIER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index f9f7e2339d5..26ff794a948 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -46,12 +46,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -134,7 +134,7 @@ public void testClearLocalDirWhenNodeReboot() throws IOException, Resource resource = Records.newRecord(Resource.class); resource.setMemory(1024); NodeId nodeId = BuilderUtils.newNodeId("127.0.0.1", 12345); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(), user, resource, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), 0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index ca87d624eae..6e6912ed009 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -48,12 +48,12 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -177,7 +177,7 @@ public static void startContainer(NodeManager nm, ContainerId cId, List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Resource resource = BuilderUtils.newResource(1024, 1); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(), user, resource, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), 0); @@ -257,7 +257,7 @@ private static File createUnhaltingScriptFile(ContainerId cId, fileWriter.close(); return scriptFile; } - + class TestNodeManager extends NodeManager { @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 6b3e27baeaf..976e94cc42b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -44,11 +44,11 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.api.ResourceManagerConstants; @@ -136,7 +136,7 @@ public void testContainerSetup() throws IOException, InterruptedException, containerLaunchContext.setLocalResources(localResources); Resource r = BuilderUtils.newResource(512, 1); int port = 12345; - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); @@ -229,7 +229,7 @@ public void testContainerLaunchAndStop() throws IOException, containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(100, 1); int port = 12345; - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); @@ -337,7 +337,7 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(100, 1); int port = 12345; - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); @@ -426,7 +426,7 @@ public void testLocalFilesCleanup() throws InterruptedException, Resource r = BuilderUtils.newResource(100, 1); int port = 12345; - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); @@ -516,7 +516,7 @@ public void testContainerLaunchFromPreviousRM() throws IOException, recordFactory.newRecordInstance(StartContainerRequest.class); startRequest1.setContainerLaunchContext(containerLaunchContext); - ContainerToken containerToken1 = + Token containerToken1 = BuilderUtils.newContainerToken(cId1, host, port, user, mockResource, System.currentTimeMillis() + 10000, 123, "password".getBytes(), (long) ResourceManagerConstants.RM_INVALID_IDENTIFIER); @@ -541,7 +541,7 @@ public void testContainerLaunchFromPreviousRM() throws IOException, StartContainerRequest startRequest2 = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest2.setContainerLaunchContext(containerLaunchContext); - ContainerToken containerToken2 = + Token containerToken2 = BuilderUtils.newContainerToken(cId1, host, port, user, mockResource, System.currentTimeMillis() + 10000, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 5e4662db269..e9aac112fa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -45,11 +45,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -578,7 +578,7 @@ private class WrappedContainer { ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource, currentTime + 10000L, 123, currentTime); - ContainerToken token = + Token token = BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port), "password".getBytes(), identifier); when(mockContainer.getContainerToken()).thenReturn(token); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 1941e4e6f30..904f16def40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -49,11 +49,11 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; @@ -233,7 +233,7 @@ public void testContainerEnvVariables() throws Exception { Resource r = BuilderUtils.newResource(1024, 1); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 1234, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); @@ -369,7 +369,7 @@ public void testDelayedKill() throws Exception { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(1024, 1); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 49be907aa7c..f8384f19634 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -18,10 +18,19 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -58,12 +67,12 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -742,7 +751,7 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 13fbf98855e..bd8a5f655dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -48,11 +48,11 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -229,7 +229,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException, StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index cab676a78cb..28a9b4fbf92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -179,7 +179,7 @@ public boolean isPmemCheckEnabled() { ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); long currentTime = System.currentTimeMillis(); - ContainerToken containerToken = + Token containerToken = BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 1166c5f9c58..bbb3ab729f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -67,7 +67,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -538,7 +537,7 @@ public RenewDelegationTokenResponse renewDelegationToken( "Delegation Token can be renewed only with kerberos authentication"); } - DelegationToken protoToken = request.getDelegationToken(); + org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken(); Token token = new Token( protoToken.getIdentifier().array(), protoToken.getPassword().array(), new Text(protoToken.getKind()), new Text(protoToken.getService())); @@ -562,7 +561,7 @@ public CancelDelegationTokenResponse cancelDelegationToken( throw new IOException( "Delegation Token can be cancelled only with kerberos authentication"); } - DelegationToken protoToken = request.getDelegationToken(); + org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken(); Token token = new Token( protoToken.getIdentifier().array(), protoToken.getPassword().array(), new Text(protoToken.getKind()), new Text(protoToken.getService())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index b5c5215bc94..d713efdab6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -431,7 +430,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { try { ApplicationAttemptId currentApplicationAttemptId = null; - ClientToken clientToken = null; + org.apache.hadoop.yarn.api.records.Token clientToken = null; String trackingUrl = UNAVAILABLE; String host = UNAVAILABLE; String origTrackingUrl = UNAVAILABLE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 9855ab51bae..57797ed7ae8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -51,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; @@ -1251,7 +1251,7 @@ Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, /** * Create ContainerToken, only in secure-mode */ - ContainerToken createContainerToken( + Token createContainerToken( FiCaSchedulerApp application, Container container) { return containerTokenSecretManager.createContainerToken( container.getId(), container.getNodeId(), @@ -1295,7 +1295,7 @@ private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode nod unreserve(application, priority, node, rmContainer); } - ContainerToken containerToken = + Token containerToken = createContainerToken(application, container); if (containerToken == null) { // Something went wrong... diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index e5a5d69ba80..0d38de6a0cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -157,7 +156,7 @@ public Container createContainer( NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - ContainerToken containerToken = + org.apache.hadoop.yarn.api.records.Token containerToken = containerTokenSecretManager.createContainerToken(containerId, nodeId, application.getUser(), capability); if (containerToken == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 935a7290693..2f4e70d19d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -549,7 +549,7 @@ private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - ContainerToken containerToken = null; + Token containerToken = null; containerToken = this.rmContext.getContainerTokenSecretManager() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index 2bafd5750bd..fe9e267cac3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -26,9 +26,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; @@ -170,7 +170,7 @@ public void run() { * @param capability * @return the container-token */ - public ContainerToken + public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability) { byte[] password; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index a7035a8a3dc..5f253969460 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -42,10 +42,10 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -162,7 +162,7 @@ synchronized public StartContainerResponse startContainer( StartContainerRequest request) throws YarnRemoteException { - ContainerToken containerToken = request.getContainerToken(); + Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 33261add5cf..b4f67e895c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -74,7 +74,7 @@ private static final class MyContainerManagerImpl implements Map env = request.getContainerLaunchContext().getEnvironment(); - ContainerToken containerToken = request.getContainerToken(); + Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 5588290eb1b..661fbc25729 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -244,7 +243,7 @@ private void checkTokenRenewal(UserGroupInformation owner, new Text(owner.getUserName()), new Text(renewer.getUserName()), null); Token token = new Token(tokenIdentifier, dtsm); - DelegationToken dToken = BuilderUtils.newDelegationToken( + org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken( token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString()); RenewDelegationTokenRequest request = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index 9ee72a79c14..fa19b656086 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -120,7 +119,7 @@ public void testDelegationToken() throws IOException, InterruptedException { loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); - DelegationToken token = getDelegationToken(loggedInUser, clientRMService, + org.apache.hadoop.yarn.api.records.Token token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName()); long tokenFetchTime = System.currentTimeMillis(); LOG.info("Got delegation token at: " + tokenFetchTime); @@ -350,14 +349,15 @@ public Server getServer(Class protocol, Object instance, // Get the delegation token directly as it is a little difficult to setup // the kerberos based rpc. - private DelegationToken getDelegationToken( + private org.apache.hadoop.yarn.api.records.Token getDelegationToken( final UserGroupInformation loggedInUser, final ClientRMProtocol clientRMService, final String renewerString) throws IOException, InterruptedException { - DelegationToken token = loggedInUser - .doAs(new PrivilegedExceptionAction() { + org.apache.hadoop.yarn.api.records.Token token = loggedInUser + .doAs(new PrivilegedExceptionAction() { @Override - public DelegationToken run() throws YarnRemoteException, IOException { + public org.apache.hadoop.yarn.api.records.Token run() + throws YarnRemoteException, IOException { GetDelegationTokenRequest request = Records .newRecord(GetDelegationTokenRequest.class); request.setRenewer(renewerString); @@ -369,7 +369,8 @@ public DelegationToken run() throws YarnRemoteException, IOException { } private long renewDelegationToken(final UserGroupInformation loggedInUser, - final ClientRMProtocol clientRMService, final DelegationToken dToken) + final ClientRMProtocol clientRMService, + final org.apache.hadoop.yarn.api.records.Token dToken) throws IOException, InterruptedException { long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction() { @Override @@ -385,7 +386,8 @@ public Long run() throws YarnRemoteException, IOException { } private void cancelDelegationToken(final UserGroupInformation loggedInUser, - final ClientRMProtocol clientRMService, final DelegationToken dToken) + final ClientRMProtocol clientRMService, + final org.apache.hadoop.yarn.api.records.Token dToken) throws IOException, InterruptedException { loggedInUser.doAs(new PrivilegedExceptionAction() { @Override @@ -399,7 +401,8 @@ public Void run() throws YarnRemoteException, IOException { }); } - private ClientRMProtocol getClientRMProtocolWithDT(DelegationToken token, + private ClientRMProtocol getClientRMProtocolWithDT( + org.apache.hadoop.yarn.api.records.Token token, final InetSocketAddress rmAddress, String user, final Configuration conf) { // Maybe consider converting to Hadoop token, serialize de-serialize etc // before trying to renew the token. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index a54a277a278..9fe0f084572 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -48,7 +48,6 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; @@ -597,7 +596,8 @@ public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { when(request1.getRenewer()).thenReturn("renewer1"); GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1); - DelegationToken delegationToken1 = response1.getRMDelegationToken(); + org.apache.hadoop.yarn.api.records.Token delegationToken1 = + response1.getRMDelegationToken(); Token token1 = ProtoUtils.convertFromProtoFormat(delegationToken1, null); RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier(); @@ -635,7 +635,8 @@ public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { when(request2.getRenewer()).thenReturn("renewer2"); GetDelegationTokenResponse response2 = rm1.getClientRMService().getDelegationToken(request2); - DelegationToken delegationToken2 = response2.getRMDelegationToken(); + org.apache.hadoop.yarn.api.records.Token delegationToken2 = + response2.getRMDelegationToken(); Token token2 = ProtoUtils.convertFromProtoFormat(delegationToken2, null); RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 116658f7b3a..71817059b5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java index 9cdd5444360..e90b8bde4c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java @@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -222,7 +221,7 @@ protected void doSecureLogin() throws IOException { GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request); ApplicationReport appReport = reportResponse.getApplicationReport(); - ClientToken clientToken = appReport.getClientToken(); + org.apache.hadoop.yarn.api.records.Token clientToken = appReport.getClientToken(); // Wait till AM is 'launched' int waitTime = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java index e72f7f56770..684320de32c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java @@ -33,7 +33,6 @@ import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; -import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -99,7 +98,8 @@ public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { when(request.getRenewer()).thenReturn("renewer1"); GetDelegationTokenResponse response = rm1.getClientRMService().getDelegationToken(request); - DelegationToken delegationToken = response.getRMDelegationToken(); + org.apache.hadoop.yarn.api.records.Token delegationToken = + response.getRMDelegationToken(); Token token1 = ProtoUtils.convertFromProtoFormat(delegationToken, null); RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index b1f0ca2f38f..c83ba184486 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -153,7 +152,8 @@ private void testAuthenticatedUser() throws IOException, final ContainerId containerID = allocatedContainer.getId(); UserGroupInformation authenticatedUser = UserGroupInformation .createRemoteUser(containerID.toString()); - ContainerToken containerToken = allocatedContainer.getContainerToken(); + org.apache.hadoop.yarn.api.records.Token containerToken = + allocatedContainer.getContainerToken(); Token token = new Token( containerToken.getIdentifier().array(), containerToken.getPassword() .array(), new Text(containerToken.getKind()), new Text( @@ -212,7 +212,8 @@ private void testMaliceUser() throws IOException, InterruptedException, // Now talk to the NM for launching the container with modified resource - ContainerToken containerToken = allocatedContainer.getContainerToken(); + org.apache.hadoop.yarn.api.records.Token containerToken = + allocatedContainer.getContainerToken(); ContainerTokenIdentifier originalContainerTokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); @@ -327,7 +328,8 @@ private void testExpiredTokens() throws IOException, InterruptedException, // Now talk to the NM for launching the container with modified containerID final ContainerId containerID = allocatedContainer.getId(); - ContainerToken containerToken = allocatedContainer.getContainerToken(); + org.apache.hadoop.yarn.api.records.Token containerToken = + allocatedContainer.getContainerToken(); final ContainerTokenIdentifier tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);