From 407a68f3ee06e897240a5ae46fd70e4db8da4245 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 5 Sep 2012 04:39:49 +0000 Subject: [PATCH] HDFS-2793. Add an admin command to trigger an edit log roll. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1380981 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/DFSClient.java | 14 ++++++++++ .../hadoop/hdfs/DistributedFileSystem.java | 10 +++++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 12 +++++++++ ...amenodeProtocolServerSideTranslatorPB.java | 16 +++++++++++ .../ClientNamenodeProtocolTranslatorPB.java | 13 +++++++++ .../hdfs/server/namenode/FSNamesystem.java | 1 + .../server/namenode/NameNodeRpcServer.java | 7 +++++ .../apache/hadoop/hdfs/tools/DFSAdmin.java | 27 +++++++++++++++++++ .../main/proto/ClientNamenodeProtocol.proto | 9 +++++++ .../src/test/resources/testHDFSConf.xml | 17 ++++++++++++ 11 files changed, 128 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b4fbba0dc3b..81709c7201c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -27,6 +27,8 @@ Release 2.0.1-alpha - UNRELEASED HDFS-3150. Add option for clients to contact DNs via hostname. (eli) + HDFS-2793. Add an admin command to trigger an edit log roll. (todd) + IMPROVEMENTS HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 8bff35e4287..121a70e6014 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1908,6 +1908,20 @@ public class DFSClient implements java.io.Closeable { throw re.unwrapRemoteException(AccessControlException.class); } } + + /** + * Rolls the edit log on the active NameNode. + * @return the txid of the new log segment + * + * @see ClientProtocol#rollEdits() + */ + long rollEdits() throws AccessControlException, IOException { + try { + return namenode.rollEdits(); + } catch(RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class); + } + } /** * enable/disable restore failed storage. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 2341c23eaaa..711ff595e50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -668,6 +668,16 @@ public class DistributedFileSystem extends FileSystem { public void saveNamespace() throws AccessControlException, IOException { dfs.saveNamespace(); } + + /** + * Rolls the edit log on the active NameNode. + * Requires super-user privileges. + * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#rollEdits() + * @return the transaction ID of the newly created segment + */ + public long rollEdits() throws AccessControlException, IOException { + return dfs.rollEdits(); + } /** * enable/disable/check restoreFaileStorage diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 86bbe10b981..3d2ea4c250d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -667,6 +667,18 @@ public interface ClientProtocol { */ public void saveNamespace() throws AccessControlException, IOException; + + /** + * Roll the edit log. + * Requires superuser privileges. + * + * @throws AccessControlException if the superuser privilege is violated + * @throws IOException if log roll fails + * @return the txid of the new segment + */ + @Idempotent + public long rollEdits() throws AccessControlException, IOException; + /** * Enable/Disable restore failed storage. *

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 37ab28505c3..1cbb1c08c96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -103,6 +103,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Report import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto; @@ -537,6 +539,20 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements } } + + @Override + public RollEditsResponseProto rollEdits(RpcController controller, + RollEditsRequestProto request) throws ServiceException { + try { + long txid = server.rollEdits(); + return RollEditsResponseProto.newBuilder() + .setNewSegmentTxId(txid) + .build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE = RefreshNodesResponseProto.newBuilder().build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 5626f038acd..02f4bb2b37b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -87,6 +87,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewD import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetOwnerRequestProto; @@ -525,6 +527,17 @@ public class ClientNamenodeProtocolTranslatorPB implements throw ProtobufHelper.getRemoteException(e); } } + + @Override + public long rollEdits() throws AccessControlException, IOException { + RollEditsRequestProto req = RollEditsRequestProto.getDefaultInstance(); + try { + RollEditsResponseProto resp = rpcProxy.rollEdits(null, req); + return resp.getNewSegmentTxId(); + } catch (ServiceException se) { + throw ProtobufHelper.getRemoteException(se); + } + } @Override public boolean restoreFailedStorage(String arg) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 38f01f6cffc..e56985d744f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4400,6 +4400,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, writeLock(); try { checkOperation(OperationCategory.JOURNAL); + checkSuperuserPrivilege(); if (isInSafeMode()) { throw new SafeModeException("Log not rolled", safeMode); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 2ff4e569fd4..9f856ca4b57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -702,6 +702,13 @@ class NameNodeRpcServer implements NamenodeProtocols { namesystem.checkOperation(OperationCategory.UNCHECKED); namesystem.saveNamespace(); } + + @Override // ClientProtocol + public long rollEdits() throws AccessControlException, IOException { + namesystem.checkOperation(OperationCategory.JOURNAL); + CheckpointSignature sig = namesystem.rollEditLog(); + return sig.getCurSegmentTxId(); + } @Override // ClientProtocol public void refreshNodes() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index d5487e41b7d..421f7bc3fd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -420,6 +420,14 @@ public class DFSAdmin extends FsShell { return exitCode; } + public int rollEdits() throws IOException { + DistributedFileSystem dfs = getDFS(); + long txid = dfs.rollEdits(); + System.out.println("Successfully rolled edit logs."); + System.out.println("New segment starts at txid " + txid); + return 0; + } + /** * Command to enable/disable/check restoring of failed storage replicas in the namenode. * Usage: java DFSAdmin -restoreFailedStorage true|false|check @@ -516,6 +524,7 @@ public class DFSAdmin extends FsShell { "The full syntax is: \n\n" + "hadoop dfsadmin [-report] [-safemode ]\n" + "\t[-saveNamespace]\n" + + "\t[-rollEdits]\n" + "\t[-restoreFailedStorage true|false|check]\n" + "\t[-refreshNodes]\n" + "\t[" + SetQuotaCommand.USAGE + "]\n" + @@ -548,6 +557,10 @@ public class DFSAdmin extends FsShell { "Save current namespace into storage directories and reset edits log.\n" + "\t\tRequires superuser permissions and safe mode.\n"; + String rollEdits = "-rollEdits:\t" + + "Rolls the edit log.\n" + + "\t\tRequires superuser permissions.\n"; + String restoreFailedStorage = "-restoreFailedStorage:\t" + "Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" + "\t\tRequires superuser permissions.\n"; @@ -625,6 +638,8 @@ public class DFSAdmin extends FsShell { System.out.println(safemode); } else if ("saveNamespace".equals(cmd)) { System.out.println(saveNamespace); + } else if ("rollEdits".equals(cmd)) { + System.out.println(rollEdits); } else if ("restoreFailedStorage".equals(cmd)) { System.out.println(restoreFailedStorage); } else if ("refreshNodes".equals(cmd)) { @@ -664,6 +679,7 @@ public class DFSAdmin extends FsShell { System.out.println(report); System.out.println(safemode); System.out.println(saveNamespace); + System.out.println(rollEdits); System.out.println(restoreFailedStorage); System.out.println(refreshNodes); System.out.println(finalizeUpgrade); @@ -859,6 +875,9 @@ public class DFSAdmin extends FsShell { } else if ("-saveNamespace".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-saveNamespace]"); + } else if ("-rollEdits".equals(cmd)) { + System.err.println("Usage: java DFSAdmin" + + " [-rollEdits]"); } else if ("-restoreFailedStorage".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-restoreFailedStorage true|false|check ]"); @@ -913,6 +932,7 @@ public class DFSAdmin extends FsShell { System.err.println(" [-report]"); System.err.println(" [-safemode enter | leave | get | wait]"); System.err.println(" [-saveNamespace]"); + System.err.println(" [-rollEdits]"); System.err.println(" [-restoreFailedStorage true|false|check]"); System.err.println(" [-refreshNodes]"); System.err.println(" [-finalizeUpgrade]"); @@ -970,6 +990,11 @@ public class DFSAdmin extends FsShell { printUsage(cmd); return exitCode; } + } else if ("-rollEdits".equals(cmd)) { + if (argv.length != 1) { + printUsage(cmd); + return exitCode; + } } else if ("-restoreFailedStorage".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); @@ -1048,6 +1073,8 @@ public class DFSAdmin extends FsShell { setSafeMode(argv, i); } else if ("-saveNamespace".equals(cmd)) { exitCode = saveNamespace(); + } else if ("-rollEdits".equals(cmd)) { + exitCode = rollEdits(); } else if ("-restoreFailedStorage".equals(cmd)) { exitCode = restoreFaileStorage(argv[i]); } else if ("-refreshNodes".equals(cmd)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 439b0cdb58b..8a4fd96d461 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -276,6 +276,13 @@ message SaveNamespaceRequestProto { // no parameters message SaveNamespaceResponseProto { // void response } +message RollEditsRequestProto { // no parameters +} + +message RollEditsResponseProto { // response + required uint64 newSegmentTxId = 1; +} + message RestoreFailedStorageRequestProto { required string arg = 1; } @@ -472,6 +479,8 @@ service ClientNamenodeProtocol { returns(SetSafeModeResponseProto); rpc saveNamespace(SaveNamespaceRequestProto) returns(SaveNamespaceResponseProto); + rpc rollEdits(RollEditsRequestProto) + returns(RollEditsResponseProto); rpc restoreFailedStorage(RestoreFailedStorageRequestProto) returns(RestoreFailedStorageResponseProto); rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 30ef443f71e..f18f2fd10b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -15885,6 +15885,23 @@ + + + rollEdits: test rollEdits admin command + + -fs NAMENODE -rollEdits + + + + + + + RegexpComparator + New segment starts at txid \d+ + + + + refreshNodes: to refresh the nodes