HDFS-2793. Add an admin command to trigger an edit log roll. Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1380981 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
dad420c609
commit
407a68f3ee
|
@ -27,6 +27,8 @@ Release 2.0.1-alpha - UNRELEASED
|
||||||
|
|
||||||
HDFS-3150. Add option for clients to contact DNs via hostname. (eli)
|
HDFS-3150. Add option for clients to contact DNs via hostname. (eli)
|
||||||
|
|
||||||
|
HDFS-2793. Add an admin command to trigger an edit log roll. (todd)
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG
|
HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG
|
||||||
|
|
|
@ -1909,6 +1909,20 @@ public class DFSClient implements java.io.Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rolls the edit log on the active NameNode.
|
||||||
|
* @return the txid of the new log segment
|
||||||
|
*
|
||||||
|
* @see ClientProtocol#rollEdits()
|
||||||
|
*/
|
||||||
|
long rollEdits() throws AccessControlException, IOException {
|
||||||
|
try {
|
||||||
|
return namenode.rollEdits();
|
||||||
|
} catch(RemoteException re) {
|
||||||
|
throw re.unwrapRemoteException(AccessControlException.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* enable/disable restore failed storage.
|
* enable/disable restore failed storage.
|
||||||
*
|
*
|
||||||
|
|
|
@ -669,6 +669,16 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
dfs.saveNamespace();
|
dfs.saveNamespace();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rolls the edit log on the active NameNode.
|
||||||
|
* Requires super-user privileges.
|
||||||
|
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#rollEdits()
|
||||||
|
* @return the transaction ID of the newly created segment
|
||||||
|
*/
|
||||||
|
public long rollEdits() throws AccessControlException, IOException {
|
||||||
|
return dfs.rollEdits();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* enable/disable/check restoreFaileStorage
|
* enable/disable/check restoreFaileStorage
|
||||||
*
|
*
|
||||||
|
|
|
@ -667,6 +667,18 @@ public interface ClientProtocol {
|
||||||
*/
|
*/
|
||||||
public void saveNamespace() throws AccessControlException, IOException;
|
public void saveNamespace() throws AccessControlException, IOException;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Roll the edit log.
|
||||||
|
* Requires superuser privileges.
|
||||||
|
*
|
||||||
|
* @throws AccessControlException if the superuser privilege is violated
|
||||||
|
* @throws IOException if log roll fails
|
||||||
|
* @return the txid of the new segment
|
||||||
|
*/
|
||||||
|
@Idempotent
|
||||||
|
public long rollEdits() throws AccessControlException, IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Enable/Disable restore failed storage.
|
* Enable/Disable restore failed storage.
|
||||||
* <p>
|
* <p>
|
||||||
|
|
|
@ -103,6 +103,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Report
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksResponseProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksResponseProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageResponseProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageResponseProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceResponseProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceResponseProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
|
||||||
|
@ -538,6 +540,20 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public RollEditsResponseProto rollEdits(RpcController controller,
|
||||||
|
RollEditsRequestProto request) throws ServiceException {
|
||||||
|
try {
|
||||||
|
long txid = server.rollEdits();
|
||||||
|
return RollEditsResponseProto.newBuilder()
|
||||||
|
.setNewSegmentTxId(txid)
|
||||||
|
.build();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE =
|
static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE =
|
||||||
RefreshNodesResponseProto.newBuilder().build();
|
RefreshNodesResponseProto.newBuilder().build();
|
||||||
|
|
||||||
|
|
|
@ -87,6 +87,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewD
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetOwnerRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetOwnerRequestProto;
|
||||||
|
@ -526,6 +528,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long rollEdits() throws AccessControlException, IOException {
|
||||||
|
RollEditsRequestProto req = RollEditsRequestProto.getDefaultInstance();
|
||||||
|
try {
|
||||||
|
RollEditsResponseProto resp = rpcProxy.rollEdits(null, req);
|
||||||
|
return resp.getNewSegmentTxId();
|
||||||
|
} catch (ServiceException se) {
|
||||||
|
throw ProtobufHelper.getRemoteException(se);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean restoreFailedStorage(String arg)
|
public boolean restoreFailedStorage(String arg)
|
||||||
throws AccessControlException, IOException{
|
throws AccessControlException, IOException{
|
||||||
|
|
|
@ -4400,6 +4400,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.JOURNAL);
|
checkOperation(OperationCategory.JOURNAL);
|
||||||
|
checkSuperuserPrivilege();
|
||||||
if (isInSafeMode()) {
|
if (isInSafeMode()) {
|
||||||
throw new SafeModeException("Log not rolled", safeMode);
|
throw new SafeModeException("Log not rolled", safeMode);
|
||||||
}
|
}
|
||||||
|
|
|
@ -703,6 +703,13 @@ class NameNodeRpcServer implements NamenodeProtocols {
|
||||||
namesystem.saveNamespace();
|
namesystem.saveNamespace();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override // ClientProtocol
|
||||||
|
public long rollEdits() throws AccessControlException, IOException {
|
||||||
|
namesystem.checkOperation(OperationCategory.JOURNAL);
|
||||||
|
CheckpointSignature sig = namesystem.rollEditLog();
|
||||||
|
return sig.getCurSegmentTxId();
|
||||||
|
}
|
||||||
|
|
||||||
@Override // ClientProtocol
|
@Override // ClientProtocol
|
||||||
public void refreshNodes() throws IOException {
|
public void refreshNodes() throws IOException {
|
||||||
namesystem.refreshNodes();
|
namesystem.refreshNodes();
|
||||||
|
|
|
@ -420,6 +420,14 @@ public class DFSAdmin extends FsShell {
|
||||||
return exitCode;
|
return exitCode;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public int rollEdits() throws IOException {
|
||||||
|
DistributedFileSystem dfs = getDFS();
|
||||||
|
long txid = dfs.rollEdits();
|
||||||
|
System.out.println("Successfully rolled edit logs.");
|
||||||
|
System.out.println("New segment starts at txid " + txid);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Command to enable/disable/check restoring of failed storage replicas in the namenode.
|
* Command to enable/disable/check restoring of failed storage replicas in the namenode.
|
||||||
* Usage: java DFSAdmin -restoreFailedStorage true|false|check
|
* Usage: java DFSAdmin -restoreFailedStorage true|false|check
|
||||||
|
@ -516,6 +524,7 @@ public class DFSAdmin extends FsShell {
|
||||||
"The full syntax is: \n\n" +
|
"The full syntax is: \n\n" +
|
||||||
"hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" +
|
"hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" +
|
||||||
"\t[-saveNamespace]\n" +
|
"\t[-saveNamespace]\n" +
|
||||||
|
"\t[-rollEdits]\n" +
|
||||||
"\t[-restoreFailedStorage true|false|check]\n" +
|
"\t[-restoreFailedStorage true|false|check]\n" +
|
||||||
"\t[-refreshNodes]\n" +
|
"\t[-refreshNodes]\n" +
|
||||||
"\t[" + SetQuotaCommand.USAGE + "]\n" +
|
"\t[" + SetQuotaCommand.USAGE + "]\n" +
|
||||||
|
@ -548,6 +557,10 @@ public class DFSAdmin extends FsShell {
|
||||||
"Save current namespace into storage directories and reset edits log.\n" +
|
"Save current namespace into storage directories and reset edits log.\n" +
|
||||||
"\t\tRequires superuser permissions and safe mode.\n";
|
"\t\tRequires superuser permissions and safe mode.\n";
|
||||||
|
|
||||||
|
String rollEdits = "-rollEdits:\t" +
|
||||||
|
"Rolls the edit log.\n" +
|
||||||
|
"\t\tRequires superuser permissions.\n";
|
||||||
|
|
||||||
String restoreFailedStorage = "-restoreFailedStorage:\t" +
|
String restoreFailedStorage = "-restoreFailedStorage:\t" +
|
||||||
"Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" +
|
"Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" +
|
||||||
"\t\tRequires superuser permissions.\n";
|
"\t\tRequires superuser permissions.\n";
|
||||||
|
@ -625,6 +638,8 @@ public class DFSAdmin extends FsShell {
|
||||||
System.out.println(safemode);
|
System.out.println(safemode);
|
||||||
} else if ("saveNamespace".equals(cmd)) {
|
} else if ("saveNamespace".equals(cmd)) {
|
||||||
System.out.println(saveNamespace);
|
System.out.println(saveNamespace);
|
||||||
|
} else if ("rollEdits".equals(cmd)) {
|
||||||
|
System.out.println(rollEdits);
|
||||||
} else if ("restoreFailedStorage".equals(cmd)) {
|
} else if ("restoreFailedStorage".equals(cmd)) {
|
||||||
System.out.println(restoreFailedStorage);
|
System.out.println(restoreFailedStorage);
|
||||||
} else if ("refreshNodes".equals(cmd)) {
|
} else if ("refreshNodes".equals(cmd)) {
|
||||||
|
@ -664,6 +679,7 @@ public class DFSAdmin extends FsShell {
|
||||||
System.out.println(report);
|
System.out.println(report);
|
||||||
System.out.println(safemode);
|
System.out.println(safemode);
|
||||||
System.out.println(saveNamespace);
|
System.out.println(saveNamespace);
|
||||||
|
System.out.println(rollEdits);
|
||||||
System.out.println(restoreFailedStorage);
|
System.out.println(restoreFailedStorage);
|
||||||
System.out.println(refreshNodes);
|
System.out.println(refreshNodes);
|
||||||
System.out.println(finalizeUpgrade);
|
System.out.println(finalizeUpgrade);
|
||||||
|
@ -859,6 +875,9 @@ public class DFSAdmin extends FsShell {
|
||||||
} else if ("-saveNamespace".equals(cmd)) {
|
} else if ("-saveNamespace".equals(cmd)) {
|
||||||
System.err.println("Usage: java DFSAdmin"
|
System.err.println("Usage: java DFSAdmin"
|
||||||
+ " [-saveNamespace]");
|
+ " [-saveNamespace]");
|
||||||
|
} else if ("-rollEdits".equals(cmd)) {
|
||||||
|
System.err.println("Usage: java DFSAdmin"
|
||||||
|
+ " [-rollEdits]");
|
||||||
} else if ("-restoreFailedStorage".equals(cmd)) {
|
} else if ("-restoreFailedStorage".equals(cmd)) {
|
||||||
System.err.println("Usage: java DFSAdmin"
|
System.err.println("Usage: java DFSAdmin"
|
||||||
+ " [-restoreFailedStorage true|false|check ]");
|
+ " [-restoreFailedStorage true|false|check ]");
|
||||||
|
@ -913,6 +932,7 @@ public class DFSAdmin extends FsShell {
|
||||||
System.err.println(" [-report]");
|
System.err.println(" [-report]");
|
||||||
System.err.println(" [-safemode enter | leave | get | wait]");
|
System.err.println(" [-safemode enter | leave | get | wait]");
|
||||||
System.err.println(" [-saveNamespace]");
|
System.err.println(" [-saveNamespace]");
|
||||||
|
System.err.println(" [-rollEdits]");
|
||||||
System.err.println(" [-restoreFailedStorage true|false|check]");
|
System.err.println(" [-restoreFailedStorage true|false|check]");
|
||||||
System.err.println(" [-refreshNodes]");
|
System.err.println(" [-refreshNodes]");
|
||||||
System.err.println(" [-finalizeUpgrade]");
|
System.err.println(" [-finalizeUpgrade]");
|
||||||
|
@ -970,6 +990,11 @@ public class DFSAdmin extends FsShell {
|
||||||
printUsage(cmd);
|
printUsage(cmd);
|
||||||
return exitCode;
|
return exitCode;
|
||||||
}
|
}
|
||||||
|
} else if ("-rollEdits".equals(cmd)) {
|
||||||
|
if (argv.length != 1) {
|
||||||
|
printUsage(cmd);
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
} else if ("-restoreFailedStorage".equals(cmd)) {
|
} else if ("-restoreFailedStorage".equals(cmd)) {
|
||||||
if (argv.length != 2) {
|
if (argv.length != 2) {
|
||||||
printUsage(cmd);
|
printUsage(cmd);
|
||||||
|
@ -1048,6 +1073,8 @@ public class DFSAdmin extends FsShell {
|
||||||
setSafeMode(argv, i);
|
setSafeMode(argv, i);
|
||||||
} else if ("-saveNamespace".equals(cmd)) {
|
} else if ("-saveNamespace".equals(cmd)) {
|
||||||
exitCode = saveNamespace();
|
exitCode = saveNamespace();
|
||||||
|
} else if ("-rollEdits".equals(cmd)) {
|
||||||
|
exitCode = rollEdits();
|
||||||
} else if ("-restoreFailedStorage".equals(cmd)) {
|
} else if ("-restoreFailedStorage".equals(cmd)) {
|
||||||
exitCode = restoreFaileStorage(argv[i]);
|
exitCode = restoreFaileStorage(argv[i]);
|
||||||
} else if ("-refreshNodes".equals(cmd)) {
|
} else if ("-refreshNodes".equals(cmd)) {
|
||||||
|
|
|
@ -276,6 +276,13 @@ message SaveNamespaceRequestProto { // no parameters
|
||||||
message SaveNamespaceResponseProto { // void response
|
message SaveNamespaceResponseProto { // void response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message RollEditsRequestProto { // no parameters
|
||||||
|
}
|
||||||
|
|
||||||
|
message RollEditsResponseProto { // response
|
||||||
|
required uint64 newSegmentTxId = 1;
|
||||||
|
}
|
||||||
|
|
||||||
message RestoreFailedStorageRequestProto {
|
message RestoreFailedStorageRequestProto {
|
||||||
required string arg = 1;
|
required string arg = 1;
|
||||||
}
|
}
|
||||||
|
@ -472,6 +479,8 @@ service ClientNamenodeProtocol {
|
||||||
returns(SetSafeModeResponseProto);
|
returns(SetSafeModeResponseProto);
|
||||||
rpc saveNamespace(SaveNamespaceRequestProto)
|
rpc saveNamespace(SaveNamespaceRequestProto)
|
||||||
returns(SaveNamespaceResponseProto);
|
returns(SaveNamespaceResponseProto);
|
||||||
|
rpc rollEdits(RollEditsRequestProto)
|
||||||
|
returns(RollEditsResponseProto);
|
||||||
rpc restoreFailedStorage(RestoreFailedStorageRequestProto)
|
rpc restoreFailedStorage(RestoreFailedStorageRequestProto)
|
||||||
returns(RestoreFailedStorageResponseProto);
|
returns(RestoreFailedStorageResponseProto);
|
||||||
rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto);
|
rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto);
|
||||||
|
|
|
@ -15885,6 +15885,23 @@
|
||||||
</comparators>
|
</comparators>
|
||||||
</test>
|
</test>
|
||||||
|
|
||||||
|
<!-- Test for rollEdits -->
|
||||||
|
<test> <!-- TESTED -->
|
||||||
|
<description>rollEdits: test rollEdits admin command</description>
|
||||||
|
<test-commands>
|
||||||
|
<dfs-admin-command>-fs NAMENODE -rollEdits</dfs-admin-command>
|
||||||
|
</test-commands>
|
||||||
|
<cleanup-commands>
|
||||||
|
<!-- no cleanup -->
|
||||||
|
</cleanup-commands>
|
||||||
|
<comparators>
|
||||||
|
<comparator>
|
||||||
|
<type>RegexpComparator</type>
|
||||||
|
<expected-output>New segment starts at txid \d+</expected-output>
|
||||||
|
</comparator>
|
||||||
|
</comparators>
|
||||||
|
</test>
|
||||||
|
|
||||||
<!-- Test for refreshNodes -->
|
<!-- Test for refreshNodes -->
|
||||||
<test> <!-- TESTED -->
|
<test> <!-- TESTED -->
|
||||||
<description>refreshNodes: to refresh the nodes</description>
|
<description>refreshNodes: to refresh the nodes</description>
|
||||||
|
|
Loading…
Reference in New Issue