HDFS-2793. Add an admin command to trigger an edit log roll. Contributed by Todd Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1380981 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-09-05 04:39:49 +00:00
parent dad420c609
commit 407a68f3ee
11 changed files with 128 additions and 0 deletions

View File

@ -27,6 +27,8 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3150. Add option for clients to contact DNs via hostname. (eli) HDFS-3150. Add option for clients to contact DNs via hostname. (eli)
HDFS-2793. Add an admin command to trigger an edit log roll. (todd)
IMPROVEMENTS IMPROVEMENTS
HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG

View File

@ -1909,6 +1909,20 @@ public class DFSClient implements java.io.Closeable {
} }
} }
/**
* Rolls the edit log on the active NameNode.
* @return the txid of the new log segment
*
* @see ClientProtocol#rollEdits()
*/
long rollEdits() throws AccessControlException, IOException {
try {
return namenode.rollEdits();
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class);
}
}
/** /**
* enable/disable restore failed storage. * enable/disable restore failed storage.
* *

View File

@ -669,6 +669,16 @@ public class DistributedFileSystem extends FileSystem {
dfs.saveNamespace(); dfs.saveNamespace();
} }
/**
* Rolls the edit log on the active NameNode.
* Requires super-user privileges.
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#rollEdits()
* @return the transaction ID of the newly created segment
*/
public long rollEdits() throws AccessControlException, IOException {
return dfs.rollEdits();
}
/** /**
* enable/disable/check restoreFaileStorage * enable/disable/check restoreFaileStorage
* *

View File

@ -667,6 +667,18 @@ public interface ClientProtocol {
*/ */
public void saveNamespace() throws AccessControlException, IOException; public void saveNamespace() throws AccessControlException, IOException;
/**
* Roll the edit log.
* Requires superuser privileges.
*
* @throws AccessControlException if the superuser privilege is violated
* @throws IOException if log roll fails
* @return the txid of the new segment
*/
@Idempotent
public long rollEdits() throws AccessControlException, IOException;
/** /**
* Enable/Disable restore failed storage. * Enable/Disable restore failed storage.
* <p> * <p>

View File

@ -103,6 +103,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Report
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
@ -538,6 +540,20 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
} }
@Override
public RollEditsResponseProto rollEdits(RpcController controller,
RollEditsRequestProto request) throws ServiceException {
try {
long txid = server.rollEdits();
return RollEditsResponseProto.newBuilder()
.setNewSegmentTxId(txid)
.build();
} catch (IOException e) {
throw new ServiceException(e);
}
}
static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE = static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE =
RefreshNodesResponseProto.newBuilder().build(); RefreshNodesResponseProto.newBuilder().build();

View File

@ -87,6 +87,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewD
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetOwnerRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetOwnerRequestProto;
@ -526,6 +528,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
} }
} }
@Override
public long rollEdits() throws AccessControlException, IOException {
RollEditsRequestProto req = RollEditsRequestProto.getDefaultInstance();
try {
RollEditsResponseProto resp = rpcProxy.rollEdits(null, req);
return resp.getNewSegmentTxId();
} catch (ServiceException se) {
throw ProtobufHelper.getRemoteException(se);
}
}
@Override @Override
public boolean restoreFailedStorage(String arg) public boolean restoreFailedStorage(String arg)
throws AccessControlException, IOException{ throws AccessControlException, IOException{

View File

@ -4400,6 +4400,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
writeLock(); writeLock();
try { try {
checkOperation(OperationCategory.JOURNAL); checkOperation(OperationCategory.JOURNAL);
checkSuperuserPrivilege();
if (isInSafeMode()) { if (isInSafeMode()) {
throw new SafeModeException("Log not rolled", safeMode); throw new SafeModeException("Log not rolled", safeMode);
} }

View File

@ -703,6 +703,13 @@ class NameNodeRpcServer implements NamenodeProtocols {
namesystem.saveNamespace(); namesystem.saveNamespace();
} }
@Override // ClientProtocol
public long rollEdits() throws AccessControlException, IOException {
namesystem.checkOperation(OperationCategory.JOURNAL);
CheckpointSignature sig = namesystem.rollEditLog();
return sig.getCurSegmentTxId();
}
@Override // ClientProtocol @Override // ClientProtocol
public void refreshNodes() throws IOException { public void refreshNodes() throws IOException {
namesystem.refreshNodes(); namesystem.refreshNodes();

View File

@ -420,6 +420,14 @@ public class DFSAdmin extends FsShell {
return exitCode; return exitCode;
} }
public int rollEdits() throws IOException {
DistributedFileSystem dfs = getDFS();
long txid = dfs.rollEdits();
System.out.println("Successfully rolled edit logs.");
System.out.println("New segment starts at txid " + txid);
return 0;
}
/** /**
* Command to enable/disable/check restoring of failed storage replicas in the namenode. * Command to enable/disable/check restoring of failed storage replicas in the namenode.
* Usage: java DFSAdmin -restoreFailedStorage true|false|check * Usage: java DFSAdmin -restoreFailedStorage true|false|check
@ -516,6 +524,7 @@ public class DFSAdmin extends FsShell {
"The full syntax is: \n\n" + "The full syntax is: \n\n" +
"hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" + "hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" +
"\t[-saveNamespace]\n" + "\t[-saveNamespace]\n" +
"\t[-rollEdits]\n" +
"\t[-restoreFailedStorage true|false|check]\n" + "\t[-restoreFailedStorage true|false|check]\n" +
"\t[-refreshNodes]\n" + "\t[-refreshNodes]\n" +
"\t[" + SetQuotaCommand.USAGE + "]\n" + "\t[" + SetQuotaCommand.USAGE + "]\n" +
@ -548,6 +557,10 @@ public class DFSAdmin extends FsShell {
"Save current namespace into storage directories and reset edits log.\n" + "Save current namespace into storage directories and reset edits log.\n" +
"\t\tRequires superuser permissions and safe mode.\n"; "\t\tRequires superuser permissions and safe mode.\n";
String rollEdits = "-rollEdits:\t" +
"Rolls the edit log.\n" +
"\t\tRequires superuser permissions.\n";
String restoreFailedStorage = "-restoreFailedStorage:\t" + String restoreFailedStorage = "-restoreFailedStorage:\t" +
"Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" + "Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" +
"\t\tRequires superuser permissions.\n"; "\t\tRequires superuser permissions.\n";
@ -625,6 +638,8 @@ public class DFSAdmin extends FsShell {
System.out.println(safemode); System.out.println(safemode);
} else if ("saveNamespace".equals(cmd)) { } else if ("saveNamespace".equals(cmd)) {
System.out.println(saveNamespace); System.out.println(saveNamespace);
} else if ("rollEdits".equals(cmd)) {
System.out.println(rollEdits);
} else if ("restoreFailedStorage".equals(cmd)) { } else if ("restoreFailedStorage".equals(cmd)) {
System.out.println(restoreFailedStorage); System.out.println(restoreFailedStorage);
} else if ("refreshNodes".equals(cmd)) { } else if ("refreshNodes".equals(cmd)) {
@ -664,6 +679,7 @@ public class DFSAdmin extends FsShell {
System.out.println(report); System.out.println(report);
System.out.println(safemode); System.out.println(safemode);
System.out.println(saveNamespace); System.out.println(saveNamespace);
System.out.println(rollEdits);
System.out.println(restoreFailedStorage); System.out.println(restoreFailedStorage);
System.out.println(refreshNodes); System.out.println(refreshNodes);
System.out.println(finalizeUpgrade); System.out.println(finalizeUpgrade);
@ -859,6 +875,9 @@ public class DFSAdmin extends FsShell {
} else if ("-saveNamespace".equals(cmd)) { } else if ("-saveNamespace".equals(cmd)) {
System.err.println("Usage: java DFSAdmin" System.err.println("Usage: java DFSAdmin"
+ " [-saveNamespace]"); + " [-saveNamespace]");
} else if ("-rollEdits".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-rollEdits]");
} else if ("-restoreFailedStorage".equals(cmd)) { } else if ("-restoreFailedStorage".equals(cmd)) {
System.err.println("Usage: java DFSAdmin" System.err.println("Usage: java DFSAdmin"
+ " [-restoreFailedStorage true|false|check ]"); + " [-restoreFailedStorage true|false|check ]");
@ -913,6 +932,7 @@ public class DFSAdmin extends FsShell {
System.err.println(" [-report]"); System.err.println(" [-report]");
System.err.println(" [-safemode enter | leave | get | wait]"); System.err.println(" [-safemode enter | leave | get | wait]");
System.err.println(" [-saveNamespace]"); System.err.println(" [-saveNamespace]");
System.err.println(" [-rollEdits]");
System.err.println(" [-restoreFailedStorage true|false|check]"); System.err.println(" [-restoreFailedStorage true|false|check]");
System.err.println(" [-refreshNodes]"); System.err.println(" [-refreshNodes]");
System.err.println(" [-finalizeUpgrade]"); System.err.println(" [-finalizeUpgrade]");
@ -970,6 +990,11 @@ public class DFSAdmin extends FsShell {
printUsage(cmd); printUsage(cmd);
return exitCode; return exitCode;
} }
} else if ("-rollEdits".equals(cmd)) {
if (argv.length != 1) {
printUsage(cmd);
return exitCode;
}
} else if ("-restoreFailedStorage".equals(cmd)) { } else if ("-restoreFailedStorage".equals(cmd)) {
if (argv.length != 2) { if (argv.length != 2) {
printUsage(cmd); printUsage(cmd);
@ -1048,6 +1073,8 @@ public class DFSAdmin extends FsShell {
setSafeMode(argv, i); setSafeMode(argv, i);
} else if ("-saveNamespace".equals(cmd)) { } else if ("-saveNamespace".equals(cmd)) {
exitCode = saveNamespace(); exitCode = saveNamespace();
} else if ("-rollEdits".equals(cmd)) {
exitCode = rollEdits();
} else if ("-restoreFailedStorage".equals(cmd)) { } else if ("-restoreFailedStorage".equals(cmd)) {
exitCode = restoreFaileStorage(argv[i]); exitCode = restoreFaileStorage(argv[i]);
} else if ("-refreshNodes".equals(cmd)) { } else if ("-refreshNodes".equals(cmd)) {

View File

@ -276,6 +276,13 @@ message SaveNamespaceRequestProto { // no parameters
message SaveNamespaceResponseProto { // void response message SaveNamespaceResponseProto { // void response
} }
message RollEditsRequestProto { // no parameters
}
message RollEditsResponseProto { // response
required uint64 newSegmentTxId = 1;
}
message RestoreFailedStorageRequestProto { message RestoreFailedStorageRequestProto {
required string arg = 1; required string arg = 1;
} }
@ -472,6 +479,8 @@ service ClientNamenodeProtocol {
returns(SetSafeModeResponseProto); returns(SetSafeModeResponseProto);
rpc saveNamespace(SaveNamespaceRequestProto) rpc saveNamespace(SaveNamespaceRequestProto)
returns(SaveNamespaceResponseProto); returns(SaveNamespaceResponseProto);
rpc rollEdits(RollEditsRequestProto)
returns(RollEditsResponseProto);
rpc restoreFailedStorage(RestoreFailedStorageRequestProto) rpc restoreFailedStorage(RestoreFailedStorageRequestProto)
returns(RestoreFailedStorageResponseProto); returns(RestoreFailedStorageResponseProto);
rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto); rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto);

View File

@ -15885,6 +15885,23 @@
</comparators> </comparators>
</test> </test>
<!-- Test for rollEdits -->
<test> <!-- TESTED -->
<description>rollEdits: test rollEdits admin command</description>
<test-commands>
<dfs-admin-command>-fs NAMENODE -rollEdits</dfs-admin-command>
</test-commands>
<cleanup-commands>
<!-- no cleanup -->
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>New segment starts at txid \d+</expected-output>
</comparator>
</comparators>
</test>
<!-- Test for refreshNodes --> <!-- Test for refreshNodes -->
<test> <!-- TESTED --> <test> <!-- TESTED -->
<description>refreshNodes: to refresh the nodes</description> <description>refreshNodes: to refresh the nodes</description>