From d7f9f9d1773429ccd7994b95b22666848db9a3f7 Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Mon, 14 Jan 2013 21:27:39 +0000 Subject: [PATCH 1/7] YARN-330. Fix flakey test: TestNodeManagerShutdown#testKillContainersOnShutdown. Contributed by Sandy Ryza git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433138 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/nodemanager/TestNodeManagerShutdown.java | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index be144837d0a..83835e83bb0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -185,6 +185,9 @@ Release 2.0.3-alpha - Unreleased YARN-253. Fixed container-launch to not fail when there are no local resources to localize. (Tom White via vinodkv) + YARN-330. Fix flakey test: TestNodeManagerShutdown#testKillContainersOnShutdown. + (Sandy Ryza via hitesh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index c5b48819641..f42261765fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -138,7 +138,14 @@ public void testKillContainersOnShutdown() throws IOException { containerManager.getContainerStatus(request).getStatus(); Assert.assertEquals(ContainerState.RUNNING, containerStatus.getState()); - try {Thread.sleep(5000);} catch (InterruptedException ex) {ex.printStackTrace();} + final int MAX_TRIES=20; + int numTries = 0; + while (!processStartFile.exists() && numTries < MAX_TRIES) { + try { + Thread.sleep(500); + } catch (InterruptedException ex) {ex.printStackTrace();} + numTries++; + } nm.stop(); @@ -202,7 +209,7 @@ private File createUnhaltingScriptFile() throws IOException { fileWriter.write("trap \"echo $hello >> " + processStartFile + "\" SIGTERM\n"); fileWriter.write("echo \"Writing pid to start file\"\n"); fileWriter.write("echo $$ >> " + processStartFile + "\n"); - fileWriter.write("while true; do\nsleep 1s;\ndone\n"); + fileWriter.write("while true; do\ndate >> /dev/null;\n done\n"); fileWriter.close(); return scriptFile; From 0671176111ce2f4d9a16a53232ef1b17e24e3349 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 14 Jan 2013 23:01:39 +0000 Subject: [PATCH 2/7] HDFS-4364. GetLinkTargetResponseProto does not handle null path. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433194 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../ClientNamenodeProtocolServerSideTranslatorPB.java | 8 ++++++-- .../protocolPB/ClientNamenodeProtocolTranslatorPB.java | 4 +++- .../src/main/proto/ClientNamenodeProtocol.proto | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c23876e9c18..a09ec7732ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -309,6 +309,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4367. GetDataEncryptionKeyResponseProto does not handle null response. (suresh) + HDFS-4364. GetLinkTargetResponseProto does not handle null path. (suresh) + + NEW FEATURES HDFS-2656. Add libwebhdfs, a pure C client based on WebHDFS. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 31da5e82219..ead8f9ea36c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -729,8 +729,12 @@ public GetLinkTargetResponseProto getLinkTarget(RpcController controller, GetLinkTargetRequestProto req) throws ServiceException { try { String result = server.getLinkTarget(req.getPath()); - return GetLinkTargetResponseProto.newBuilder().setTargetPath(result) - .build(); + GetLinkTargetResponseProto.Builder builder = GetLinkTargetResponseProto + .newBuilder(); + if (result != null) { + builder.setTargetPath(result); + } + return builder.build(); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 5ac29826b46..140ae4f23ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatusRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLinkTargetRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLinkTargetResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto; @@ -714,7 +715,8 @@ public String getLinkTarget(String path) throws AccessControlException, GetLinkTargetRequestProto req = GetLinkTargetRequestProto.newBuilder() .setPath(path).build(); try { - return rpcProxy.getLinkTarget(null, req).getTargetPath(); + GetLinkTargetResponseProto rsp = rpcProxy.getLinkTarget(null, req); + return rsp.hasTargetPath() ? rsp.getTargetPath() : null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 6bf5af34919..c10a938a71e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -387,7 +387,7 @@ message GetLinkTargetRequestProto { required string path = 1; } message GetLinkTargetResponseProto { - required string targetPath = 1; + optional string targetPath = 1; } message UpdateBlockForPipelineRequestProto { From 6dbc7e03e5a2f1db081f5146c94949b3f58e6c8c Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 14 Jan 2013 23:13:27 +0000 Subject: [PATCH 3/7] HDFS-4369. GetBlockKeysResponseProto does not handle null response. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433206 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../NamenodeProtocolServerSideTranslatorPB.java | 8 ++++++-- .../hdfs/protocolPB/NamenodeProtocolTranslatorPB.java | 6 ++++-- .../hadoop-hdfs/src/main/proto/NamenodeProtocol.proto | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a09ec7732ac..ec1d7a7d97d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -311,6 +311,8 @@ Release 2.0.3-alpha - Unreleased HDFS-4364. GetLinkTargetResponseProto does not handle null path. (suresh) + HDFS-4369. GetBlockKeysResponseProto does not handle null response. + (suresh) NEW FEATURES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java index c3466e15a5e..f9a7abbf78f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java @@ -91,8 +91,12 @@ public GetBlockKeysResponseProto getBlockKeys(RpcController unused, } catch (IOException e) { throw new ServiceException(e); } - return GetBlockKeysResponseProto.newBuilder() - .setKeys(PBHelper.convert(keys)).build(); + GetBlockKeysResponseProto.Builder builder = + GetBlockKeysResponseProto.newBuilder(); + if (keys != null) { + builder.setKeys(PBHelper.convert(keys)); + } + return builder.build(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java index 6c630d168ed..fcf95d6d39b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlockKeysRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlockKeysResponseProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlocksRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetEditLogManifestRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdRequestProto; @@ -104,8 +105,9 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size) @Override public ExportedBlockKeys getBlockKeys() throws IOException { try { - return PBHelper.convert(rpcProxy.getBlockKeys(NULL_CONTROLLER, - GET_BLOCKKEYS).getKeys()); + GetBlockKeysResponseProto rsp = rpcProxy.getBlockKeys(NULL_CONTROLLER, + GET_BLOCKKEYS); + return rsp.hasKeys() ? PBHelper.convert(rsp.getKeys()) : null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto index 62884c66815..24e72fa2f93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto @@ -56,7 +56,7 @@ message GetBlockKeysRequestProto { * keys - Information about block keys at the active namenode */ message GetBlockKeysResponseProto { - required ExportedBlockKeysProto keys = 1; + optional ExportedBlockKeysProto keys = 1; } /** From 6d807fcab144d8097cf559e5ac234db995819ba1 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 14 Jan 2013 23:48:50 +0000 Subject: [PATCH 4/7] HADOOP-9203. RPCCallBenchmark should find a random available port. Contributec by Andrew Purtell. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433220 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/net/NetUtils.java | 20 +++++++++++++++++++ .../apache/hadoop/ipc/RPCCallBenchmark.java | 20 ++++++++++++++----- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index dd4605504f2..cbfef30c7ff 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -540,6 +540,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9183. Potential deadlock in ActiveStandbyElector. (tomwhite) + HADOOP-9203. RPCCallBenchmark should find a random available port. + (Andrew Purtell via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index d61ac9b5023..29d8af9fd7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -25,6 +25,7 @@ import java.net.InetSocketAddress; import java.net.NetworkInterface; import java.net.NoRouteToHostException; +import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress; import java.net.SocketException; @@ -865,4 +866,23 @@ public static List getIPs(String subnet, } return addrs; } + + /** + * Return a free port number. There is no guarantee it will remain free, so + * it should be used immediately. + * + * @returns A free port for binding a local socket + */ + public static int getFreeSocketPort() { + int port = 0; + try { + ServerSocket s = new ServerSocket(0); + port = s.getLocalPort(); + s.close(); + return port; + } catch (IOException e) { + // Could not get a free port. Return default port 0. + } + return port; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java index 21aa44a7c99..d358913ef05 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java @@ -67,7 +67,7 @@ private static class MyOptions { private int serverReaderThreads = 1; private int clientThreads = 0; private String host = "0.0.0.0"; - private int port = 12345; + private int port = 0; public int secondsToRun = 15; private int msgSize = 1024; public Class rpcEngine = @@ -201,11 +201,21 @@ private void processOptions(CommandLine line, Options opts) } } + public int getPort() { + if (port == 0) { + port = NetUtils.getFreeSocketPort(); + if (port == 0) { + throw new RuntimeException("Could not find a free port"); + } + } + return port; + } + @Override public String toString() { return "rpcEngine=" + rpcEngine + "\nserverThreads=" + serverThreads + "\nserverReaderThreads=" + serverReaderThreads + "\nclientThreads=" - + clientThreads + "\nhost=" + host + "\nport=" + port + + clientThreads + "\nhost=" + host + "\nport=" + getPort() + "\nsecondsToRun=" + secondsToRun + "\nmsgSize=" + msgSize; } } @@ -228,12 +238,12 @@ private Server startServer(MyOptions opts) throws IOException { .newReflectiveBlockingService(serverImpl); server = new RPC.Builder(conf).setProtocol(TestRpcService.class) - .setInstance(service).setBindAddress(opts.host).setPort(opts.port) + .setInstance(service).setBindAddress(opts.host).setPort(opts.getPort()) .setNumHandlers(opts.serverThreads).setVerbose(false).build(); } else if (opts.rpcEngine == WritableRpcEngine.class) { server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host) - .setPort(opts.port).setNumHandlers(opts.serverThreads) + .setPort(opts.getPort()).setNumHandlers(opts.serverThreads) .setVerbose(false).build(); } else { throw new RuntimeException("Bad engine: " + opts.rpcEngine); @@ -378,7 +388,7 @@ private interface RpcServiceWrapper { * Create a client proxy for the specified engine. */ private RpcServiceWrapper createRpcClient(MyOptions opts) throws IOException { - InetSocketAddress addr = NetUtils.createSocketAddr(opts.host, opts.port); + InetSocketAddress addr = NetUtils.createSocketAddr(opts.host, opts.getPort()); if (opts.rpcEngine == ProtobufRpcEngine.class) { final TestRpcService proxy = RPC.getProxy(TestRpcService.class, 0, addr, conf); From 86825d839078d3fb01fcd1406ff99fa081f5dc49 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 15 Jan 2013 00:11:16 +0000 Subject: [PATCH 5/7] MAPREDUCE-4938. Use token request messages defined in hadoop common. Contributed by Suresh Srinvias. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433227 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../client/MRClientProtocolPBClientImpl.java | 9 ++++---- .../MRClientProtocolPBServiceImpl.java | 12 +++++----- .../CancelDelegationTokenRequestPBImpl.java | 13 +++++------ .../CancelDelegationTokenResponsePBImpl.java | 2 +- .../pb/GetDelegationTokenRequestPBImpl.java | 7 ++---- .../pb/GetDelegationTokenResponsePBImpl.java | 12 +++++----- .../pb/RenewDelegationTokenRequestPBImpl.java | 13 +++++------ .../RenewDelegationTokenResponsePBImpl.java | 8 +++---- .../src/main/proto/MRClientProtocol.proto | 7 +++--- .../src/main/proto/mr_service_protos.proto | 22 ------------------- 11 files changed, 40 insertions(+), 68 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index eeec45d2cdd..8e7f15c6aa0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -170,6 +170,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4123. Remove the 'mapred groups' command, which is no longer supported. (Devaraj K via sseth) + MAPREDUCE-4938. Use token request messages defined in hadoop common. + (suresh) + NEW FEATURES MAPREDUCE-4520. Added support for MapReduce applications to request for diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java index 930163a56ec..ad2ce63144f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java @@ -82,10 +82,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDiagnosticsRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetJobReportRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetTaskAttemptCompletionEventsRequestProto; @@ -95,7 +93,9 @@ import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillJobRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl; @@ -109,8 +109,7 @@ public class MRClientProtocolPBClientImpl implements MRClientProtocol { public MRClientProtocolPBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { RPC.setProtocolEngine(conf, MRClientProtocolPB.class, ProtobufRpcEngine.class); - proxy = (MRClientProtocolPB)RPC.getProxy( - MRClientProtocolPB.class, clientVersion, addr, conf); + proxy = RPC.getProxy(MRClientProtocolPB.class, clientVersion, addr, conf); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java index 492272ce8e6..0cfb5279e3a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java @@ -73,14 +73,10 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersResponseProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDiagnosticsRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDiagnosticsResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetJobReportRequestProto; @@ -99,8 +95,12 @@ import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptResponseProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto; import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskResponseProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import com.google.protobuf.RpcController; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java index 68ffcccf28f..8a4045552e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java @@ -18,8 +18,8 @@ package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProtoOrBuilder; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; @@ -52,10 +52,7 @@ public DelegationToken getDelegationToken() { if (this.token != null) { return this.token; } - if (!p.hasDelegationToken()) { - return null; - } - this.token = convertFromProtoFormat(p.getDelegationToken()); + this.token = convertFromProtoFormat(p.getToken()); return this.token; } @@ -63,7 +60,7 @@ public DelegationToken getDelegationToken() { public void setDelegationToken(DelegationToken token) { maybeInitBuilder(); if (token == null) - builder.clearDelegationToken(); + builder.clearToken(); this.token = token; } @@ -78,7 +75,7 @@ public CancelDelegationTokenRequestProto getProto() { private void mergeLocalToBuilder() { if (token != null) { - builder.setDelegationToken(convertToProtoFormat(this.token)); + builder.setToken(convertToProtoFormat(this.token)); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java index 4c7989aad17..59f0ae9f9cb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java @@ -18,7 +18,7 @@ package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.yarn.api.records.ProtoBase; public class CancelDelegationTokenResponsePBImpl extends diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java index 9cf26a2620b..b028c53b371 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java @@ -18,8 +18,8 @@ package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProtoOrBuilder; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.yarn.api.records.ProtoBase; @@ -50,9 +50,6 @@ public String getRenewer(){ if (this.renewer != null) { return this.renewer; } - if (!p.hasRenewer()) { - return null; - } this.renewer = p.getRenewer(); return this.renewer; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java index 59675225734..9a43a279581 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java @@ -18,8 +18,8 @@ package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProtoOrBuilder; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; @@ -53,10 +53,10 @@ public DelegationToken getDelegationToken() { if (this.mrToken != null) { return this.mrToken; } - if (!p.hasMRDelegationToken()) { + if (!p.hasToken()) { return null; } - this.mrToken = convertFromProtoFormat(p.getMRDelegationToken()); + this.mrToken = convertFromProtoFormat(p.getToken()); return this.mrToken; } @@ -64,7 +64,7 @@ public DelegationToken getDelegationToken() { public void setDelegationToken(DelegationToken mrToken) { maybeInitBuilder(); if (mrToken == null) - builder.clearMRDelegationToken(); + builder.getToken(); this.mrToken = mrToken; } @@ -79,7 +79,7 @@ public GetDelegationTokenResponseProto getProto() { private void mergeLocalToBuilder() { if (mrToken != null) { - builder.setMRDelegationToken(convertToProtoFormat(this.mrToken)); + builder.setToken(convertToProtoFormat(this.mrToken)); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java index 7d9017d9356..5b616b6b9a1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java @@ -18,8 +18,8 @@ package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProtoOrBuilder; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; @@ -52,10 +52,7 @@ public DelegationToken getDelegationToken() { if (this.token != null) { return this.token; } - if (!p.hasDelegationToken()) { - return null; - } - this.token = convertFromProtoFormat(p.getDelegationToken()); + this.token = convertFromProtoFormat(p.getToken()); return this.token; } @@ -63,7 +60,7 @@ public DelegationToken getDelegationToken() { public void setDelegationToken(DelegationToken token) { maybeInitBuilder(); if (token == null) - builder.clearDelegationToken(); + builder.clearToken(); this.token = token; } @@ -77,7 +74,7 @@ public RenewDelegationTokenRequestProto getProto() { private void mergeLocalToBuilder() { if (token != null) { - builder.setDelegationToken(convertToProtoFormat(this.token)); + builder.setToken(convertToProtoFormat(this.token)); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java index 0a9e1275e60..beb78de7c17 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java @@ -19,8 +19,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto; -import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProtoOrBuilder; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProtoOrBuilder; import org.apache.hadoop.yarn.api.records.ProtoBase; public class RenewDelegationTokenResponsePBImpl extends @@ -59,12 +59,12 @@ private void maybeInitBuilder() { @Override public long getNextExpirationTime() { RenewDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder; - return p.getNextExpiryTs(); + return p.getNewExpiryTime(); } @Override public void setNextExpirationTime(long expTime) { maybeInitBuilder(); - builder.setNextExpiryTs(expTime); + builder.setNewExpiryTime(expTime); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto index f9de094f43d..83a946f4cee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto @@ -20,6 +20,7 @@ option java_package = "org.apache.hadoop.yarn.proto"; option java_outer_classname = "MRClientProtocol"; option java_generic_services = true; +import "Security.proto"; import "mr_service_protos.proto"; /* If making changes to this, please edit HSClientProtocolService */ @@ -31,11 +32,11 @@ service MRClientProtocolService { rpc getTaskAttemptCompletionEvents (GetTaskAttemptCompletionEventsRequestProto) returns (GetTaskAttemptCompletionEventsResponseProto); rpc getTaskReports (GetTaskReportsRequestProto) returns (GetTaskReportsResponseProto); rpc getDiagnostics (GetDiagnosticsRequestProto) returns (GetDiagnosticsResponseProto); - rpc getDelegationToken (GetDelegationTokenRequestProto) returns (GetDelegationTokenResponseProto); + rpc getDelegationToken (hadoop.common.GetDelegationTokenRequestProto) returns (hadoop.common.GetDelegationTokenResponseProto); rpc killJob (KillJobRequestProto) returns (KillJobResponseProto); rpc killTask (KillTaskRequestProto) returns (KillTaskResponseProto); rpc killTaskAttempt (KillTaskAttemptRequestProto) returns (KillTaskAttemptResponseProto); rpc failTaskAttempt (FailTaskAttemptRequestProto) returns (FailTaskAttemptResponseProto); - rpc renewDelegationToken(RenewDelegationTokenRequestProto) returns (RenewDelegationTokenResponseProto); - rpc cancelDelegationToken(CancelDelegationTokenRequestProto) returns (CancelDelegationTokenResponseProto); + rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto) returns (hadoop.common.RenewDelegationTokenResponseProto); + rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto) returns (hadoop.common.CancelDelegationTokenResponseProto); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto index 12c05209f3d..ff965f30774 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto @@ -77,14 +77,6 @@ message GetDiagnosticsResponseProto { repeated string diagnostics = 1; } -message GetDelegationTokenRequestProto { - optional string renewer = 1; -} - -message GetDelegationTokenResponseProto { - optional hadoop.common.TokenProto m_r_delegation_token = 1; -} - message KillJobRequestProto { optional JobIdProto job_id = 1; } @@ -109,17 +101,3 @@ message FailTaskAttemptRequestProto { message FailTaskAttemptResponseProto { } -message RenewDelegationTokenRequestProto { - required hadoop.common.TokenProto delegation_token = 1; -} - -message RenewDelegationTokenResponseProto { - required int64 next_expiry_ts = 1; -} - -message CancelDelegationTokenRequestProto { - required hadoop.common.TokenProto delegation_token = 1; -} - -message CancelDelegationTokenResponseProto { -} From bbfcc709a22863df2e6b44382fbe6009154e13bb Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 15 Jan 2013 00:16:57 +0000 Subject: [PATCH 6/7] YARN-328. Use token request messages defined in hadoop common. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433231 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 ++ .../CancelDelegationTokenRequestPBImpl.java | 13 ++++------ .../CancelDelegationTokenResponsePBImpl.java | 2 +- .../pb/GetDelegationTokenRequestPBImpl.java | 7 ++---- .../pb/GetDelegationTokenResponsePBImpl.java | 12 +++++----- .../pb/RenewDelegationTokenRequestPBImpl.java | 13 ++++------ .../RenewDelegationTokenResponsePBImpl.java | 8 +++---- .../src/main/proto/client_RM_protocol.proto | 7 +++--- .../src/main/proto/yarn_service_protos.proto | 24 ------------------- .../client/ClientRMProtocolPBClientImpl.java | 10 ++++---- .../ClientRMProtocolPBServiceImpl.java | 12 +++++----- 11 files changed, 39 insertions(+), 71 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 83835e83bb0..a37b8f3edc0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -35,6 +35,8 @@ Release 2.0.3-alpha - Unreleased YARN-2. Enhanced CapacityScheduler to account for CPU alongwith memory for multi-dimensional resource scheduling. (acmurthy) + YARN-328. Use token request messages defined in hadoop common. (suresh) + IMPROVEMENTS YARN-223. Update process tree instead of getting new process trees. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java index 95dcd59d45b..a38f94a7ede 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java @@ -17,13 +17,13 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.CancelDelegationTokenRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.CancelDelegationTokenRequestProtoOrBuilder; public class CancelDelegationTokenRequestPBImpl extends ProtoBase implements @@ -52,10 +52,7 @@ public DelegationToken getDelegationToken() { if (this.token != null) { return this.token; } - if (!p.hasDelegationToken()) { - return null; - } - this.token = convertFromProtoFormat(p.getDelegationToken()); + this.token = convertFromProtoFormat(p.getToken()); return this.token; } @@ -63,7 +60,7 @@ public DelegationToken getDelegationToken() { public void setDelegationToken(DelegationToken token) { maybeInitBuilder(); if (token == null) - builder.clearDelegationToken(); + builder.clearToken(); this.token = token; } @@ -77,7 +74,7 @@ public CancelDelegationTokenRequestProto getProto() { private void mergeLocalToBuilder() { if (token != null) { - builder.setDelegationToken(convertToProtoFormat(this.token)); + builder.setToken(convertToProtoFormat(this.token)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java index 14908c7bbcc..5eb2b0713d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java @@ -17,9 +17,9 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.CancelDelegationTokenResponseProto; public class CancelDelegationTokenResponsePBImpl extends ProtoBase implements diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java index 57594446018..687d469b76c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenRequestProtoOrBuilder; public class GetDelegationTokenRequestPBImpl extends ProtoBase implements GetDelegationTokenRequest { @@ -48,9 +48,6 @@ public String getRenewer(){ if (this.renewer != null) { return this.renewer; } - if (!p.hasRenewer()) { - return null; - } this.renewer = p.getRenewer(); return this.renewer; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java index b459e2328d1..76c31b40c8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java @@ -18,13 +18,13 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenResponseProtoOrBuilder; public class GetDelegationTokenResponsePBImpl extends ProtoBase implements GetDelegationTokenResponse { @@ -53,10 +53,10 @@ public DelegationToken getRMDelegationToken() { if (this.appToken != null) { return this.appToken; } - if (!p.hasApplicationToken()) { + if (!p.hasToken()) { return null; } - this.appToken = convertFromProtoFormat(p.getApplicationToken()); + this.appToken = convertFromProtoFormat(p.getToken()); return this.appToken; } @@ -64,7 +64,7 @@ public DelegationToken getRMDelegationToken() { public void setRMDelegationToken(DelegationToken appToken) { maybeInitBuilder(); if (appToken == null) - builder.clearApplicationToken(); + builder.clearToken(); this.appToken = appToken; } @@ -79,7 +79,7 @@ public GetDelegationTokenResponseProto getProto() { private void mergeLocalToBuilder() { if (appToken != null) { - builder.setApplicationToken(convertToProtoFormat(this.appToken)); + builder.setToken(convertToProtoFormat(this.appToken)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java index ec927b15b6b..cf02e849085 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java @@ -17,13 +17,13 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProtoOrBuilder; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenRequestProtoOrBuilder; public class RenewDelegationTokenRequestPBImpl extends ProtoBase implements @@ -51,10 +51,7 @@ public DelegationToken getDelegationToken() { if (this.token != null) { return this.token; } - if (!p.hasDelegationToken()) { - return null; - } - this.token = convertFromProtoFormat(p.getDelegationToken()); + this.token = convertFromProtoFormat(p.getToken()); return this.token; } @@ -62,7 +59,7 @@ public DelegationToken getDelegationToken() { public void setDelegationToken(DelegationToken token) { maybeInitBuilder(); if (token == null) - builder.clearDelegationToken(); + builder.clearToken(); this.token = token; } @@ -77,7 +74,7 @@ public RenewDelegationTokenRequestProto getProto() { private void mergeLocalToBuilder() { if (token != null) { - builder.setDelegationToken(convertToProtoFormat(this.token)); + builder.setToken(convertToProtoFormat(this.token)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java index b5c80f199c9..ae1acc7ce1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java @@ -17,10 +17,10 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProtoOrBuilder; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenResponseProtoOrBuilder; public class RenewDelegationTokenResponsePBImpl extends ProtoBase implements @@ -58,12 +58,12 @@ private void maybeInitBuilder() { @Override public long getNextExpirationTime() { RenewDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder; - return p.getNextExpiryTs(); + return p.getNewExpiryTime(); } @Override public void setNextExpirationTime(long expTime) { maybeInitBuilder(); - builder.setNextExpiryTs(expTime); + builder.setNewExpiryTime(expTime); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto index 7495ce8784f..5aa2380ae5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto @@ -21,6 +21,7 @@ option java_outer_classname = "ClientRMProtocol"; option java_generic_services = true; option java_generate_equals_and_hash = true; +import "Security.proto"; import "yarn_service_protos.proto"; service ClientRMProtocolService { @@ -33,8 +34,8 @@ service ClientRMProtocolService { rpc getClusterNodes (GetClusterNodesRequestProto) returns (GetClusterNodesResponseProto); rpc getQueueInfo (GetQueueInfoRequestProto) returns (GetQueueInfoResponseProto); rpc getQueueUserAcls (GetQueueUserAclsInfoRequestProto) returns (GetQueueUserAclsInfoResponseProto); - rpc getDelegationToken(GetDelegationTokenRequestProto) returns (GetDelegationTokenResponseProto); - rpc renewDelegationToken(RenewDelegationTokenRequestProto) returns (RenewDelegationTokenResponseProto); - rpc cancelDelegationToken(CancelDelegationTokenRequestProto) returns (CancelDelegationTokenResponseProto); + rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto) returns (hadoop.common.GetDelegationTokenResponseProto); + rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto) returns (hadoop.common.RenewDelegationTokenResponseProto); + rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto) returns (hadoop.common.CancelDelegationTokenResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 637cb17c838..2c59e9fe5d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -140,30 +140,6 @@ message GetQueueUserAclsInfoResponseProto { } -message GetDelegationTokenRequestProto { - optional string renewer = 1; -} - -message GetDelegationTokenResponseProto { - optional hadoop.common.TokenProto application_token = 1; -} - -message RenewDelegationTokenRequestProto { - required hadoop.common.TokenProto delegation_token = 1; -} - -message RenewDelegationTokenResponseProto { - required int64 next_expiry_ts = 1; -} - -message CancelDelegationTokenRequestProto { - required hadoop.common.TokenProto delegation_token = 1; -} - -message CancelDelegationTokenResponseProto { -} - - ////////////////////////////////////////////////////// /////// client_NM_Protocol /////////////////////////// ////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java index 6302725250e..25212b83002 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java @@ -25,6 +25,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.ClientRMProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; @@ -77,17 +80,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import com.google.protobuf.ServiceException; @@ -101,9 +101,7 @@ public ClientRMProtocolPBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { RPC.setProtocolEngine(conf, ClientRMProtocolPB.class, ProtobufRpcEngine.class); - proxy = - (ClientRMProtocolPB) RPC.getProxy(ClientRMProtocolPB.class, - clientVersion, addr, conf); + proxy = RPC.getProxy(ClientRMProtocolPB.class, clientVersion, addr, conf); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java index 9eecdad68a6..1c2d5b05ebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java @@ -18,6 +18,12 @@ package org.apache.hadoop.yarn.api.impl.pb.service; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.ClientRMProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -57,8 +63,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.CancelDelegationTokenRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; @@ -67,8 +71,6 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; @@ -77,8 +79,6 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; From 8767a37b9bd589286871be1272ee72fb331a31ad Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 15 Jan 2013 00:26:23 +0000 Subject: [PATCH 7/7] HDFS-4375. Use token request messages defined in hadoop common. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433235 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ ...amenodeProtocolServerSideTranslatorPB.java | 14 ++++---- .../ClientNamenodeProtocolTranslatorPB.java | 10 +++--- .../main/proto/ClientNamenodeProtocol.proto | 35 ++++--------------- 4 files changed, 21 insertions(+), 41 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ec1d7a7d97d..4be0be767d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -485,6 +485,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4381. Document fsimage format details in FSImageFormat class javadoc. (Jing Zhao via suresh) + HDFS-4375. Use token request messages defined in hadoop common. + (suresh) + OPTIMIZATIONS HDFS-3429. DataNode reads checksums even if client does not need them (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index ead8f9ea36c..ef697de70a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -37,8 +37,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto; @@ -64,8 +62,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetContentSummaryResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto; @@ -94,8 +90,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2ResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewDelegationTokenRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto; @@ -130,6 +124,12 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.security.token.Token; import com.google.protobuf.RpcController; @@ -799,7 +799,7 @@ public RenewDelegationTokenResponseProto renewDelegationToken( long result = server.renewDelegationToken(PBHelper .convertDelegationToken(req.getToken())); return RenewDelegationTokenResponseProto.newBuilder() - .setNewExireTime(result).build(); + .setNewExpiryTime(result).build(); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 140ae4f23ae..5faa9a811d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateRequestProto; @@ -66,8 +65,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetContentSummaryRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto; @@ -86,7 +83,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RefreshNodesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto; @@ -113,6 +109,10 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto; +import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.security.token.Token; import com.google.protobuf.ByteString; @@ -777,7 +777,7 @@ public long renewDelegationToken(Token token) setToken(PBHelper.convert(token)). build(); try { - return rpcProxy.renewDelegationToken(null, req).getNewExireTime(); + return rpcProxy.renewDelegationToken(null, req).getNewExpiryTime(); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index c10a938a71e..0c23a7d6624 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -409,29 +409,6 @@ message UpdatePipelineRequestProto { message UpdatePipelineResponseProto { // void response } -message GetDelegationTokenRequestProto { - required string renewer = 1; -} - -message GetDelegationTokenResponseProto { - optional hadoop.common.TokenProto token = 1; -} - -message RenewDelegationTokenRequestProto { - required hadoop.common.TokenProto token = 1; -} - -message RenewDelegationTokenResponseProto { - required uint64 newExireTime = 1; -} - -message CancelDelegationTokenRequestProto { - required hadoop.common.TokenProto token = 1; -} - -message CancelDelegationTokenResponseProto { // void response -} - message SetBalancerBandwidthRequestProto { required int64 bandwidth = 1; } @@ -509,12 +486,12 @@ service ClientNamenodeProtocol { returns(UpdateBlockForPipelineResponseProto); rpc updatePipeline(UpdatePipelineRequestProto) returns(UpdatePipelineResponseProto); - rpc getDelegationToken(GetDelegationTokenRequestProto) - returns(GetDelegationTokenResponseProto); - rpc renewDelegationToken(RenewDelegationTokenRequestProto) - returns(RenewDelegationTokenResponseProto); - rpc cancelDelegationToken(CancelDelegationTokenRequestProto) - returns(CancelDelegationTokenResponseProto); + rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto) + returns(hadoop.common.GetDelegationTokenResponseProto); + rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto) + returns(hadoop.common.RenewDelegationTokenResponseProto); + rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto) + returns(hadoop.common.CancelDelegationTokenResponseProto); rpc setBalancerBandwidth(SetBalancerBandwidthRequestProto) returns(SetBalancerBandwidthResponseProto); rpc getDataEncryptionKey(GetDataEncryptionKeyRequestProto)