From 2cb090a07510dd35d0c59c4317dea6887a9e8ca3 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 4 Jan 2013 20:31:22 +0000 Subject: [PATCH] MAPREDUCE-4894. Renewal / cancellation of JobHistory tokens (Siddharth Seth via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1429105 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/client/MRClientService.java | 18 ++ .../mapreduce/v2/api/HSClientProtocol.java | 1 + .../mapreduce/v2/api/MRClientProtocol.java | 24 ++ .../v2/api/MRDelegationTokenIdentifier.java | 3 + .../client/MRClientProtocolPBClientImpl.java | 39 ++- .../MRClientProtocolPBServiceImpl.java | 38 +++ .../CancelDelegationTokenRequest.java | 34 +++ .../CancelDelegationTokenResponse.java | 31 +++ .../RenewDelegationTokenRequest.java | 34 +++ .../RenewDelegationTokenResponse.java | 32 +++ .../CancelDelegationTokenRequestPBImpl.java | 108 ++++++++ .../CancelDelegationTokenResponsePBImpl.java | 44 +++ .../pb/RenewDelegationTokenRequestPBImpl.java | 106 ++++++++ .../RenewDelegationTokenResponsePBImpl.java | 70 +++++ .../v2/security/MRDelegationTokenRenewer.java | 121 +++++++++ .../src/main/proto/MRClientProtocol.proto | 2 + .../src/main/proto/mr_service_protos.proto | 15 + ....apache.hadoop.security.token.TokenRenewer | 1 + .../hadoop/mapreduce/v2/TestRPCFactories.java | 16 ++ .../org/apache/hadoop/mapred/JobClient.java | 30 -- .../org/apache/hadoop/mapreduce/Cluster.java | 16 +- ....apache.hadoop.security.token.TokenRenewer | 1 - .../mapreduce/v2/hs/HistoryClientService.java | 80 +++++- .../apache/hadoop/mapred/NotRunningJob.java | 20 +- .../hadoop/mapred/ResourceMgrDelegate.java | 15 +- .../org/apache/hadoop/mapred/YARNRunner.java | 6 +- .../hadoop/mapred/TestClientRedirect.java | 32 ++- .../mapreduce/security/TestJHSSecurity.java | 257 ++++++++++++++---- 29 files changed, 1077 insertions(+), 120 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenResponse.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenResponse.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4f422311b02..1ca70ce6146 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -501,6 +501,9 @@ Release 0.23.6 - UNRELEASED MAPREDUCE-4832. MR AM can get in a split brain situation (jlowe) + MAPREDUCE-4894. Renewal / cancellation of JobHistory tokens (Siddharth + Seth via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java index d756480f97b..03811026244 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java @@ -31,6 +31,8 @@ import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -55,6 +57,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; @@ -386,5 +390,19 @@ public GetDelegationTokenResponse getDelegationToken( throw RPCUtil.getRemoteException("MR AM not authorized to issue delegation" + " token"); } + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + throw RPCUtil.getRemoteException("MR AM not authorized to renew delegation" + + " token"); + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + throw RPCUtil.getRemoteException("MR AM not authorized to cancel delegation" + + " token"); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSClientProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSClientProtocol.java index f1ea91514a7..71f9dc2575c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSClientProtocol.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSClientProtocol.java @@ -18,5 +18,6 @@ package org.apache.hadoop.mapreduce.v2.api; + public interface HSClientProtocol extends MRClientProtocol { } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java index 08166b96b18..2c51b4be42b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java @@ -20,6 +20,8 @@ import java.net.InetSocketAddress; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -44,6 +46,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; public interface MRClientProtocol { @@ -64,4 +68,24 @@ public interface MRClientProtocol { public KillTaskAttemptResponse killTaskAttempt(KillTaskAttemptRequest request) throws YarnRemoteException; public FailTaskAttemptResponse failTaskAttempt(FailTaskAttemptRequest request) throws YarnRemoteException; public GetDelegationTokenResponse getDelegationToken(GetDelegationTokenRequest request) throws YarnRemoteException; + + /** + * Renew an existing delegation token. + * + * @param request the delegation token to be renewed. + * @return the new expiry time for the delegation token. + * @throws YarnRemoteException + */ + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException; + + /** + * Cancel an existing delegation token. + * + * @param request the delegation token to be cancelled. + * @return an empty response. + * @throws YarnRemoteException + */ + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRDelegationTokenIdentifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRDelegationTokenIdentifier.java index d2c86226a41..a63872aabde 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRDelegationTokenIdentifier.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRDelegationTokenIdentifier.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.api; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -30,6 +31,8 @@ * issued by JobHistoryServer to delegate * MR tasks talking to the JobHistoryServer. */ +@Private +// TODO Move to a different package. public class MRDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { public static final Text KIND_NAME = new Text("MR_DELEGATION_TOKEN"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java index 3ab3f0c3b8b..930163a56ec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java @@ -19,7 +19,6 @@ package org.apache.hadoop.mapreduce.v2.api.impl.pb.client; import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; @@ -27,6 +26,8 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -51,6 +52,10 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.FailTaskAttemptRequestPBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.FailTaskAttemptResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.GetCountersRequestPBImpl; @@ -75,6 +80,9 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskAttemptResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskRequestPBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto; @@ -87,6 +95,7 @@ import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillJobRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl; @@ -242,5 +251,31 @@ public FailTaskAttemptResponse failTaskAttempt(FailTaskAttemptRequest request) throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e); } } - + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + RenewDelegationTokenRequestProto requestProto = + ((RenewDelegationTokenRequestPBImpl) request).getProto(); + try { + return new RenewDelegationTokenResponsePBImpl(proxy.renewDelegationToken( + null, requestProto)); + } catch (ServiceException e) { + throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e); + } + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + CancelDelegationTokenRequestProto requestProto = + ((CancelDelegationTokenRequestPBImpl) request).getProto(); + try { + return new CancelDelegationTokenResponsePBImpl( + proxy.cancelDelegationToken(null, requestProto)); + + } catch (ServiceException e) { + throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e); + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java index 90881215fac..492272ce8e6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java @@ -20,6 +20,7 @@ import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -43,6 +44,9 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.FailTaskAttemptRequestPBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.FailTaskAttemptResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.GetCountersRequestPBImpl; @@ -67,6 +71,10 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskAttemptResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskRequestPBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto; @@ -91,6 +99,8 @@ import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskResponseProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import com.google.protobuf.RpcController; @@ -252,4 +262,32 @@ public FailTaskAttemptResponseProto failTaskAttempt(RpcController controller, } } + @Override + public RenewDelegationTokenResponseProto renewDelegationToken( + RpcController controller, RenewDelegationTokenRequestProto proto) + throws ServiceException { + RenewDelegationTokenRequestPBImpl request = + new RenewDelegationTokenRequestPBImpl(proto); + try { + RenewDelegationTokenResponse response = real.renewDelegationToken(request); + return ((RenewDelegationTokenResponsePBImpl)response).getProto(); + } catch (YarnRemoteException e) { + throw new ServiceException(e); + } + } + + @Override + public CancelDelegationTokenResponseProto cancelDelegationToken( + RpcController controller, CancelDelegationTokenRequestProto proto) + throws ServiceException { + CancelDelegationTokenRequestPBImpl request = + new CancelDelegationTokenRequestPBImpl(proto); + try { + CancelDelegationTokenResponse response = real.cancelDelegationToken(request); + return ((CancelDelegationTokenResponsePBImpl)response).getProto(); + } catch (YarnRemoteException e) { + throw new ServiceException(e); + } + } + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java new file mode 100644 index 00000000000..c5d7837a95e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenRequest.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.records.DelegationToken; + +/** + * The request issued by the client to the {@code ResourceManager} to cancel a + * delegation token. + */ +@Public +@Evolving +public interface CancelDelegationTokenRequest { + DelegationToken getDelegationToken(); + void setDelegationToken(DelegationToken dToken); +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenResponse.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenResponse.java new file mode 100644 index 00000000000..80d10fc7080 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/CancelDelegationTokenResponse.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; + +/** + * The response from the {@code ResourceManager} to a cancelDelegationToken + * request. + */ +@Public +@Evolving +public interface CancelDelegationTokenResponse { +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java new file mode 100644 index 00000000000..af062fdcd5d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenRequest.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.records.DelegationToken; + +/** + * The request issued by the client to renew a delegation token from + * the {@code ResourceManager}. + */ +@Public +@Evolving +public interface RenewDelegationTokenRequest { + DelegationToken getDelegationToken(); + void setDelegationToken(DelegationToken dToken); +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenResponse.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenResponse.java new file mode 100644 index 00000000000..e36d2616da1 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/RenewDelegationTokenResponse.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; + +/** + * The response to a renewDelegationToken call to the {@code ResourceManager}. + */ +@Public +@Evolving +public interface RenewDelegationTokenResponse { + long getNextExpirationTime(); + void setNextExpirationTime(long expTime); +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java new file mode 100644 index 00000000000..e9e196bbbf0 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; + +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProtoOrBuilder; +import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.ProtoBase; +import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.DelegationTokenProto; + +public class CancelDelegationTokenRequestPBImpl extends + ProtoBase implements + CancelDelegationTokenRequest { + + CancelDelegationTokenRequestProto proto = + CancelDelegationTokenRequestProto.getDefaultInstance(); + CancelDelegationTokenRequestProto.Builder builder = null; + boolean viaProto = false; + + public CancelDelegationTokenRequestPBImpl() { + this.builder = CancelDelegationTokenRequestProto.newBuilder(); + } + + public CancelDelegationTokenRequestPBImpl ( + CancelDelegationTokenRequestProto proto) { + this.proto = proto; + this.viaProto = true; + } + + DelegationToken token; + + @Override + public DelegationToken getDelegationToken() { + CancelDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.token != null) { + return this.token; + } + if (!p.hasDelegationToken()) { + return null; + } + this.token = convertFromProtoFormat(p.getDelegationToken()); + return this.token; + } + + @Override + public void setDelegationToken(DelegationToken token) { + maybeInitBuilder(); + if (token == null) + builder.clearDelegationToken(); + this.token = token; + } + + @Override + public CancelDelegationTokenRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + + private void mergeLocalToBuilder() { + if (token != null) { + builder.setDelegationToken(convertToProtoFormat(this.token)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = CancelDelegationTokenRequestProto.newBuilder(proto); + } + viaProto = false; + } + + + private DelegationTokenPBImpl convertFromProtoFormat(DelegationTokenProto p) { + return new DelegationTokenPBImpl(p); + } + + private DelegationTokenProto convertToProtoFormat(DelegationToken t) { + return ((DelegationTokenPBImpl)t).getProto(); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java new file mode 100644 index 00000000000..4c7989aad17 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; + +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto; +import org.apache.hadoop.yarn.api.records.ProtoBase; + +public class CancelDelegationTokenResponsePBImpl extends + ProtoBase implements + CancelDelegationTokenResponse { + + CancelDelegationTokenResponseProto proto = CancelDelegationTokenResponseProto + .getDefaultInstance(); + + public CancelDelegationTokenResponsePBImpl() { + } + + public CancelDelegationTokenResponsePBImpl( + CancelDelegationTokenResponseProto proto) { + this.proto = proto; + } + + @Override + public CancelDelegationTokenResponseProto getProto() { + return proto; + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java new file mode 100644 index 00000000000..26777c6b1e6 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; + +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProtoOrBuilder; +import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.api.records.ProtoBase; +import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.DelegationTokenProto; + +public class RenewDelegationTokenRequestPBImpl extends + ProtoBase implements + RenewDelegationTokenRequest { + + RenewDelegationTokenRequestProto proto = RenewDelegationTokenRequestProto + .getDefaultInstance(); + RenewDelegationTokenRequestProto.Builder builder = null; + boolean viaProto = false; + + public RenewDelegationTokenRequestPBImpl() { + this.builder = RenewDelegationTokenRequestProto.newBuilder(); + } + + public RenewDelegationTokenRequestPBImpl( + RenewDelegationTokenRequestProto proto) { + this.proto = proto; + this.viaProto = true; + } + + DelegationToken token; + + @Override + public DelegationToken getDelegationToken() { + RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.token != null) { + return this.token; + } + if (!p.hasDelegationToken()) { + return null; + } + this.token = convertFromProtoFormat(p.getDelegationToken()); + return this.token; + } + + @Override + public void setDelegationToken(DelegationToken token) { + maybeInitBuilder(); + if (token == null) + builder.clearDelegationToken(); + this.token = token; + } + + @Override + public RenewDelegationTokenRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (token != null) { + builder.setDelegationToken(convertToProtoFormat(this.token)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RenewDelegationTokenRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private DelegationTokenPBImpl convertFromProtoFormat(DelegationTokenProto p) { + return new DelegationTokenPBImpl(p); + } + + private DelegationTokenProto convertToProtoFormat(DelegationToken t) { + return ((DelegationTokenPBImpl) t).getProto(); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java new file mode 100644 index 00000000000..0a9e1275e60 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; + + +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto; +import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProtoOrBuilder; +import org.apache.hadoop.yarn.api.records.ProtoBase; + +public class RenewDelegationTokenResponsePBImpl extends + ProtoBase implements + RenewDelegationTokenResponse { + + RenewDelegationTokenResponseProto proto = + RenewDelegationTokenResponseProto.getDefaultInstance(); + RenewDelegationTokenResponseProto.Builder builder = null; + boolean viaProto = false; + + public RenewDelegationTokenResponsePBImpl() { + this.builder = RenewDelegationTokenResponseProto.newBuilder(); + } + + public RenewDelegationTokenResponsePBImpl ( + RenewDelegationTokenResponseProto proto) { + this.proto = proto; + this.viaProto = true; + } + + @Override + public RenewDelegationTokenResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RenewDelegationTokenResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public long getNextExpirationTime() { + RenewDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder; + return p.getNextExpiryTs(); + } + + @Override + public void setNextExpirationTime(long expTime) { + maybeInitBuilder(); + builder.setNextExpiryTs(expTime); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java new file mode 100644 index 00000000000..2415e45801c --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.mapreduce.v2.security; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.PrivilegedAction; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenRenewer; +import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +public class MRDelegationTokenRenewer extends TokenRenewer { + + private static final Log LOG = LogFactory + .getLog(MRDelegationTokenRenewer.class); + + @Override + public boolean handleKind(Text kind) { + return MRDelegationTokenIdentifier.KIND_NAME.equals(kind); + } + + @Override + public long renew(Token token, Configuration conf) throws IOException, + InterruptedException { + + DelegationToken dToken = BuilderUtils.newDelegationToken( + token.getIdentifier(), token.getKind().toString(), token.getPassword(), + token.getService().toString()); + + MRClientProtocol histProxy = instantiateHistoryProxy(conf, + SecurityUtil.getTokenServiceAddr(token)); + try { + RenewDelegationTokenRequest request = Records + .newRecord(RenewDelegationTokenRequest.class); + request.setDelegationToken(dToken); + return histProxy.renewDelegationToken(request).getNextExpirationTime(); + } finally { + stopHistoryProxy(histProxy); + } + + } + + @Override + public void cancel(Token token, Configuration conf) throws IOException, + InterruptedException { + + DelegationToken dToken = BuilderUtils.newDelegationToken( + token.getIdentifier(), token.getKind().toString(), token.getPassword(), + token.getService().toString()); + + MRClientProtocol histProxy = instantiateHistoryProxy(conf, + SecurityUtil.getTokenServiceAddr(token)); + try { + CancelDelegationTokenRequest request = Records + .newRecord(CancelDelegationTokenRequest.class); + request.setDelegationToken(dToken); + histProxy.cancelDelegationToken(request); + } finally { + stopHistoryProxy(histProxy); + } + } + + @Override + public boolean isManaged(Token token) throws IOException { + return true; + } + + protected void stopHistoryProxy(MRClientProtocol proxy) { + RPC.stopProxy(proxy); + } + + protected MRClientProtocol instantiateHistoryProxy(final Configuration conf, + final InetSocketAddress hsAddress) throws IOException { + + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to MRHistoryServer at: " + hsAddress); + } + final YarnRPC rpc = YarnRPC.create(conf); + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + return currentUser.doAs(new PrivilegedAction() { + @Override + public MRClientProtocol run() { + return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, + hsAddress, conf); + } + }); + } +} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto index 28c8e6b7a21..f9de094f43d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto @@ -36,4 +36,6 @@ service MRClientProtocolService { rpc killTask (KillTaskRequestProto) returns (KillTaskResponseProto); rpc killTaskAttempt (KillTaskAttemptRequestProto) returns (KillTaskAttemptResponseProto); rpc failTaskAttempt (FailTaskAttemptRequestProto) returns (FailTaskAttemptResponseProto); + rpc renewDelegationToken(RenewDelegationTokenRequestProto) returns (RenewDelegationTokenResponseProto); + rpc cancelDelegationToken(CancelDelegationTokenRequestProto) returns (CancelDelegationTokenResponseProto); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto index 47a08aba6c4..f875fcb26bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto @@ -107,3 +107,18 @@ message FailTaskAttemptRequestProto { } message FailTaskAttemptResponseProto { } + +message RenewDelegationTokenRequestProto { + required DelegationTokenProto delegation_token = 1; +} + +message RenewDelegationTokenResponseProto { + required int64 next_expiry_ts = 1; +} + +message CancelDelegationTokenRequestProto { + required DelegationTokenProto delegation_token = 1; +} + +message CancelDelegationTokenResponseProto { +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer new file mode 100644 index 00000000000..76846fc1c5e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -0,0 +1 @@ +org.apache.hadoop.mapreduce.v2.security.MRDelegationTokenRenewer diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java index c76328d5056..2b0fceba513 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/TestRPCFactories.java @@ -26,6 +26,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -50,6 +52,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -198,6 +202,18 @@ public FailTaskAttemptResponse failTaskAttempt( public GetDelegationTokenResponse getDelegationToken( GetDelegationTokenRequest request) throws YarnRemoteException { return null; + } + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + return null; + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + return null; } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java index 7e49fa77cfe..803b584b7d5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java @@ -496,36 +496,6 @@ public void init(JobConf conf) throws IOException { clientUgi = UserGroupInformation.getCurrentUser(); } - @InterfaceAudience.Private - public static class Renewer extends TokenRenewer { - - @Override - public boolean handleKind(Text kind) { - return DelegationTokenIdentifier.MAPREDUCE_DELEGATION_KIND.equals(kind); - } - - @SuppressWarnings("unchecked") - @Override - public long renew(Token token, Configuration conf - ) throws IOException, InterruptedException { - return new Cluster(conf). - renewDelegationToken((Token) token); - } - - @SuppressWarnings("unchecked") - @Override - public void cancel(Token token, Configuration conf - ) throws IOException, InterruptedException { - new Cluster(conf). - cancelDelegationToken((Token) token); - } - - @Override - public boolean isManaged(Token token) throws IOException { - return true; - } - } - /** * Build a job client, connect to the indicated job tracker. * diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java index e456a7afa88..e93f2736044 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java @@ -33,14 +33,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.util.ConfigUtil; import org.apache.hadoop.mapreduce.v2.LogParams; -import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; @@ -401,12 +399,7 @@ public long getTaskTrackerExpiryInterval() throws IOException, public long renewDelegationToken(Token token ) throws InvalidToken, IOException, InterruptedException { - try { - return client.renewDelegationToken(token); - } catch (RemoteException re) { - throw re.unwrapRemoteException(InvalidToken.class, - AccessControlException.class); - } + return token.renew(getConf()); } /** @@ -418,12 +411,7 @@ public long renewDelegationToken(Token token public void cancelDelegationToken(Token token ) throws IOException, InterruptedException { - try { - client.cancelDelegationToken(token); - } catch (RemoteException re) { - throw re.unwrapRemoteException(InvalidToken.class, - AccessControlException.class); - } + token.cancel(getConf()); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index a8c02470552..7bb8265629b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -1,2 +1 @@ -org.apache.hadoop.mapred.JobClient$Renewer org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier$Renewer diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index 388356f01ab..2bae2fc95ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -24,6 +24,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; +import java.util.EnumSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,6 +38,8 @@ import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -61,6 +64,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; @@ -75,7 +80,6 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.DelegationToken; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -83,6 +87,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -314,13 +319,10 @@ public GetDelegationTokenResponse getDelegationToken( UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); // Verify that the connection is kerberos authenticated - AuthenticationMethod authMethod = UserGroupInformation - .getRealAuthenticationMethod(ugi); - if (UserGroupInformation.isSecurityEnabled() - && (authMethod != AuthenticationMethod.KERBEROS)) { - throw new IOException( - "Delegation Token can be issued only with kerberos authentication"); - } + if (!isAllowedDelegationTokenOp()) { + throw new IOException( + "Delegation Token can be issued only with kerberos authentication"); + } GetDelegationTokenResponse response = recordFactory.newRecordInstance( GetDelegationTokenResponse.class); @@ -347,6 +349,55 @@ public GetDelegationTokenResponse getDelegationToken( } } + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + try { + if (!isAllowedDelegationTokenOp()) { + throw new IOException( + "Delegation Token can be renewed only with kerberos authentication"); + } + + DelegationToken protoToken = request.getDelegationToken(); + Token token = new Token( + protoToken.getIdentifier().array(), protoToken.getPassword() + .array(), new Text(protoToken.getKind()), new Text( + protoToken.getService())); + + String user = UserGroupInformation.getCurrentUser().getShortUserName(); + long nextExpTime = jhsDTSecretManager.renewToken(token, user); + RenewDelegationTokenResponse renewResponse = Records + .newRecord(RenewDelegationTokenResponse.class); + renewResponse.setNextExpirationTime(nextExpTime); + return renewResponse; + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + try { + if (!isAllowedDelegationTokenOp()) { + throw new IOException( + "Delegation Token can be cancelled only with kerberos authentication"); + } + + DelegationToken protoToken = request.getDelegationToken(); + Token token = new Token( + protoToken.getIdentifier().array(), protoToken.getPassword() + .array(), new Text(protoToken.getKind()), new Text( + protoToken.getService())); + + String user = UserGroupInformation.getCurrentUser().getShortUserName(); + jhsDTSecretManager.cancelToken(token, user); + return Records.newRecord(CancelDelegationTokenResponse.class); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + } + private void checkAccess(Job job, JobACL jobOperation) throws YarnRemoteException { @@ -362,5 +413,18 @@ private void checkAccess(Job job, JobACL jobOperation) + jobOperation.name() + " on " + job.getID())); } } + + private boolean isAllowedDelegationTokenOp() throws IOException { + if (UserGroupInformation.isSecurityEnabled()) { + return EnumSet.of(AuthenticationMethod.KERBEROS, + AuthenticationMethod.KERBEROS_SSL, + AuthenticationMethod.CERTIFICATE) + .contains(UserGroupInformation.getCurrentUser() + .getRealAuthenticationMethod()); + } else { + return true; + } + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index f9b25b02131..7e46433319b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -24,6 +24,8 @@ import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -48,6 +50,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup; import org.apache.hadoop.mapreduce.v2.api.records.Counters; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; @@ -215,7 +219,21 @@ public GetDelegationTokenResponse getDelegationToken( /* Should not be invoked by anyone. */ throw new NotImplementedException(); } - + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + /* Should not be invoked by anyone. */ + throw new NotImplementedException(); + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + /* Should not be invoked by anyone. */ + throw new NotImplementedException(); + } + @Override public InetSocketAddress getConnectAddress() { /* Should not be invoked by anyone. Normally used to set token service */ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index c8cfe9c4db1..730809ce5d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -33,7 +33,6 @@ import org.apache.hadoop.mapreduce.QueueInfo; import org.apache.hadoop.mapreduce.TaskTrackerInfo; import org.apache.hadoop.mapreduce.TypeConverter; -import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -41,9 +40,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.client.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ProtoUtils; -import org.apache.hadoop.yarn.client.YarnClientImpl; public class ResourceMgrDelegate extends YarnClientImpl { private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); @@ -62,11 +61,6 @@ public ResourceMgrDelegate(YarnConfiguration conf) { init(conf); start(); } - - public void cancelDelegationToken(Token arg0) - throws IOException, InterruptedException { - return; - } public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException { @@ -168,13 +162,6 @@ public long getProtocolVersion(String arg0, long arg1) throws IOException { return 0; } - public long renewDelegationToken(Token arg0) - throws IOException, InterruptedException { - // TODO: Implement renewDelegationToken - LOG.warn("renewDelegationToken - Not implemented"); - return 0; - } - public ApplicationId getApplicationId() { return applicationId; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 834959967f8..1b684364f51 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -158,7 +158,7 @@ public void setResourceMgrDelegate(ResourceMgrDelegate resMgrDelegate) { @Override public void cancelDelegationToken(Token arg0) throws IOException, InterruptedException { - resMgrDelegate.cancelDelegationToken(arg0); + throw new UnsupportedOperationException("Use Token.renew instead"); } @Override @@ -195,7 +195,7 @@ private Token getDelegationTokenFromHS( return ProtoUtils.convertFromProtoFormat(mrDelegationToken, hsProxy.getConnectAddress()); } - + @Override public Token getDelegationToken(Text renewer) throws IOException, InterruptedException { @@ -466,7 +466,7 @@ public long getProtocolVersion(String arg0, long arg1) throws IOException { @Override public long renewDelegationToken(Token arg0) throws IOException, InterruptedException { - return resMgrDelegate.renewDelegationToken(arg0); + throw new UnsupportedOperationException("Use Token.renew instead"); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 095d3fd9301..503d188c97a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -25,10 +25,10 @@ import junit.framework.Assert; -import org.apache.hadoop.ipc.Server; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MRConfig; @@ -69,6 +69,8 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -87,6 +89,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -339,6 +343,18 @@ public GetDelegationTokenResponse getDelegationToken( GetDelegationTokenRequest request) throws YarnRemoteException { return null; } + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + return null; + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { @@ -504,6 +520,20 @@ public org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResp throws YarnRemoteException { return null; } + + @Override + public org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse renewDelegationToken( + org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest request) + throws YarnRemoteException { + return null; + } + + @Override + public org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse cancelDelegationToken( + org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest request) + throws YarnRemoteException { + return null; + } } static Counters getMyCounters() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java index 792806b624c..53ef152e890 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java @@ -18,19 +18,29 @@ package org.apache.hadoop.mapreduce.security; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.hs.JHSDelegationTokenSecretManager; import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; @@ -49,6 +59,8 @@ public class TestJHSSecurity { + private static final Log LOG = LogFactory.getLog(TestJHSSecurity.class); + @Test public void testDelegationToken() throws IOException, InterruptedException { @@ -63,55 +75,208 @@ public void testDelegationToken() throws IOException, InterruptedException { conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); + + final long initialInterval = 10000l; + final long maxLifetime= 20000l; + final long renewInterval = 10000l; - final JobHistoryServer jobHistoryServer = new JobHistoryServer() { - protected void doSecureLogin(Configuration conf) throws IOException { - // no keytab based login - }; - }; - jobHistoryServer.init(conf); - jobHistoryServer.start(); - - // Fake the authentication-method - UserGroupInformation loggedInUser = UserGroupInformation.getCurrentUser(); - loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); - - // Get the delegation token directly as it is a little difficult to setup - // the kerberos based rpc. - DelegationToken token = - loggedInUser.doAs(new PrivilegedExceptionAction() { - @Override - public DelegationToken run() throws YarnRemoteException { - GetDelegationTokenRequest request = - Records.newRecord(GetDelegationTokenRequest.class); - request.setRenewer("OneRenewerToRuleThemAll"); - return jobHistoryServer.getClientService().getClientHandler() - .getDelegationToken(request).getDelegationToken(); - } - }); - - // Now try talking to JHS using the delegation token - UserGroupInformation ugi = - UserGroupInformation.createRemoteUser("TheDarkLord"); - ugi.addToken(ProtoUtils.convertFromProtoFormat( - token, jobHistoryServer.getClientService().getBindAddress())); - final YarnRPC rpc = YarnRPC.create(conf); - MRClientProtocol userUsingDT = - ugi.doAs(new PrivilegedAction() { - @Override - public MRClientProtocol run() { - return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, - jobHistoryServer.getClientService().getBindAddress(), conf); - } - }); - GetJobReportRequest jobReportRequest = - Records.newRecord(GetJobReportRequest.class); - jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1)); + JobHistoryServer jobHistoryServer = null; + MRClientProtocol clientUsingDT = null; + long tokenFetchTime; try { - userUsingDT.getJobReport(jobReportRequest); - } catch (YarnRemoteException e) { - Assert.assertEquals("Unknown job job_123456_0001", e.getMessage()); + jobHistoryServer = new JobHistoryServer() { + protected void doSecureLogin(Configuration conf) throws IOException { + // no keytab based login + }; + + protected JHSDelegationTokenSecretManager createJHSSecretManager( + Configuration conf) { + return new JHSDelegationTokenSecretManager(initialInterval, + maxLifetime, renewInterval, 3600000); + } + }; +// final JobHistoryServer jobHistoryServer = jhServer; + jobHistoryServer.init(conf); + jobHistoryServer.start(); + final MRClientProtocol hsService = jobHistoryServer.getClientService() + .getClientHandler(); + + // Fake the authentication-method + UserGroupInformation loggedInUser = UserGroupInformation + .createRemoteUser("testrenewer@APACHE.ORG"); + Assert.assertEquals("testrenewer", loggedInUser.getShortUserName()); + // Default realm is APACHE.ORG + loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); + + + DelegationToken token = getDelegationToken(loggedInUser, hsService, + loggedInUser.getShortUserName()); + tokenFetchTime = System.currentTimeMillis(); + LOG.info("Got delegation token at: " + tokenFetchTime); + + // Now try talking to JHS using the delegation token + clientUsingDT = getMRClientProtocol(token, jobHistoryServer + .getClientService().getBindAddress(), "TheDarkLord", conf); + + GetJobReportRequest jobReportRequest = + Records.newRecord(GetJobReportRequest.class); + jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1)); + try { + clientUsingDT.getJobReport(jobReportRequest); + } catch (YarnRemoteException e) { + Assert.assertEquals("Unknown job job_123456_0001", e.getMessage()); + } + + // Renew after 50% of token age. + while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) { + Thread.sleep(500l); + } + long nextExpTime = renewDelegationToken(loggedInUser, hsService, token); + long renewalTime = System.currentTimeMillis(); + LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + + nextExpTime); + + // Wait for first expiry, but before renewed expiry. + while (System.currentTimeMillis() > tokenFetchTime + initialInterval + && System.currentTimeMillis() < nextExpTime) { + Thread.sleep(500l); + } + Thread.sleep(50l); + + // Valid token because of renewal. + try { + clientUsingDT.getJobReport(jobReportRequest); + } catch (UndeclaredThrowableException e) { + Assert.assertEquals("Unknown job job_123456_0001", e.getMessage()); + } + + // Wait for expiry. + while(System.currentTimeMillis() < renewalTime + renewInterval) { + Thread.sleep(500l); + } + Thread.sleep(50l); + LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid"); + // Token should have expired. + try { + clientUsingDT.getJobReport(jobReportRequest); + fail("Should not have succeeded with an expired token"); + } catch (UndeclaredThrowableException e) { + assertTrue(e.getCause().getMessage().contains("is expired")); + } + + // Test cancellation + // Stop the existing proxy, start another. + if (clientUsingDT != null) { +// RPC.stopProxy(clientUsingDT); + clientUsingDT = null; + } + token = getDelegationToken(loggedInUser, hsService, + loggedInUser.getShortUserName()); + tokenFetchTime = System.currentTimeMillis(); + LOG.info("Got delegation token at: " + tokenFetchTime); + + // Now try talking to HSService using the delegation token + clientUsingDT = getMRClientProtocol(token, jobHistoryServer + .getClientService().getBindAddress(), "loginuser2", conf); + + + try { + clientUsingDT.getJobReport(jobReportRequest); + } catch (UndeclaredThrowableException e) { + fail("Unexpected exception" + e); + } + cancelDelegationToken(loggedInUser, hsService, token); + if (clientUsingDT != null) { +// RPC.stopProxy(clientUsingDT); + clientUsingDT = null; + } + + // Creating a new connection. + clientUsingDT = getMRClientProtocol(token, jobHistoryServer + .getClientService().getBindAddress(), "loginuser2", conf); + LOG.info("Cancelled delegation token at: " + System.currentTimeMillis()); + // Verify cancellation worked. + try { + clientUsingDT.getJobReport(jobReportRequest); + fail("Should not have succeeded with a cancelled delegation token"); + } catch (UndeclaredThrowableException e) { + } + + + + } finally { + jobHistoryServer.stop(); } } + private DelegationToken getDelegationToken( + final UserGroupInformation loggedInUser, + final MRClientProtocol hsService, final String renewerString) + throws IOException, InterruptedException { + // Get the delegation token directly as it is a little difficult to setup + // the kerberos based rpc. + DelegationToken token = loggedInUser + .doAs(new PrivilegedExceptionAction() { + @Override + public DelegationToken run() throws YarnRemoteException { + GetDelegationTokenRequest request = Records + .newRecord(GetDelegationTokenRequest.class); + request.setRenewer(renewerString); + return hsService.getDelegationToken(request).getDelegationToken(); + } + + }); + return token; + } + + private long renewDelegationToken(final UserGroupInformation loggedInUser, + final MRClientProtocol hsService, final DelegationToken dToken) + throws IOException, InterruptedException { + long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction() { + + @Override + public Long run() throws YarnRemoteException { + RenewDelegationTokenRequest request = Records + .newRecord(RenewDelegationTokenRequest.class); + request.setDelegationToken(dToken); + return hsService.renewDelegationToken(request).getNextExpirationTime(); + } + }); + return nextExpTime; + } + + private void cancelDelegationToken(final UserGroupInformation loggedInUser, + final MRClientProtocol hsService, final DelegationToken dToken) + throws IOException, InterruptedException { + + loggedInUser.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws YarnRemoteException { + CancelDelegationTokenRequest request = Records + .newRecord(CancelDelegationTokenRequest.class); + request.setDelegationToken(dToken); + hsService.cancelDelegationToken(request); + return null; + } + }); + } + + private MRClientProtocol getMRClientProtocol(DelegationToken token, + final InetSocketAddress hsAddress, String user, final Configuration conf) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + ugi.addToken(ProtoUtils.convertFromProtoFormat(token, hsAddress)); + + final YarnRPC rpc = YarnRPC.create(conf); + MRClientProtocol hsWithDT = ugi + .doAs(new PrivilegedAction() { + + @Override + public MRClientProtocol run() { + return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, + hsAddress, conf); + } + }); + return hsWithDT; + } + }