Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13813. Exit NameNode if dangling child inode is detected when saving FsImage. Contributed by Siyao Meng.

Browse Source 
(cherry picked from commit 23854443efa62aa70a1c30c32c3816750e5d7a5b)
(cherry picked from commit b38649c59a70b3112811443464b3b3180f4b9873)
(cherry picked from commit 4b606eb9fd5dde5629e791d3338886a70b5aa57c)
This commit is contained in:
Wei-Chiu Chuang 2018-08-13 16:12:37 -07:00
parent 305b9a652a
commit ad5c3300ff
2 changed files with 32 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
View File

@ -407,6 +407,8 @@ private void loadRootINode(INodeSection.INode p) {
}
public final static class Saver {
private long numImageErrors;
private static long buildPermissionStatus(INodeAttributes n,
final SaverContext.DeduplicationMap<String> stringMap) {
long userId = stringMap.getId(n.getUserName());
@ -525,11 +527,13 @@ public static INodeSection.INodeDirectory.Builder buildINodeDirectory(
this.summary = summary;
this.context = parent.getContext();
this.fsn = context.getSourceNamesystem();
this.numImageErrors = 0;
}
void serializeINodeDirectorySection(OutputStream out) throws IOException {
Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory()
.getINodeMap().getMapIterator();
FSDirectory dir = fsn.getFSDirectory();
Iterator<INodeWithAdditionalFields> iter = dir.getINodeMap()
.getMapIterator();
final ArrayList<INodeReference> refList = parent.getSaverContext()
.getRefList();
int i = 0;
@ -545,6 +549,17 @@ void serializeINodeDirectorySection(OutputStream out) throws IOException {
INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection.
DirEntry.newBuilder().setParent(n.getId());
for (INode inode : children) {
// Error if the child inode doesn't exist in inodeMap
if (dir.getInode(inode.getId()) == null) {
FSImage.LOG.error(
"FSImageFormatPBINode#serializeINodeDirectorySection: " +
"Dangling child pointer found. Missing INode in " +
"inodeMap: id=" + inode.getId() +
"; path=" + inode.getFullPathName() +
"; parent=" + (inode.getParent() == null ? "null" :
inode.getParent().getFullPathName()));
++numImageErrors;
}
if (!inode.isReference()) {
b.addChildren(inode.getId());
} else {
@ -672,6 +687,15 @@ private final INodeSection.INode.Builder buildINodeCommon(INode n) {
.setId(n.getId())
.setName(ByteString.copyFrom(n.getLocalNameBytes()));
}
/**
* Number of non-fatal errors detected while writing the
* INodeSection and INodeDirectorySection sections.
* @return the number of non-fatal errors detected.
*/
public long getNumImageErrors() {
return numImageErrors;
}
}
private FSImageFormatPBINode() {

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
View File

@ -448,13 +448,15 @@ private static void saveFileSummary(OutputStream out, FileSummary summary)
out.write(lengthBytes);
}
private void saveInodes(FileSummary.Builder summary) throws IOException {
private long saveInodes(FileSummary.Builder summary) throws IOException {
FSImageFormatPBINode.Saver saver = new FSImageFormatPBINode.Saver(this,
summary);
saver.serializeINodeSection(sectionOutputStream);
saver.serializeINodeDirectorySection(sectionOutputStream);
saver.serializeFilesUCSection(sectionOutputStream);
return saver.getNumImageErrors();
}
/**
@ -511,8 +513,9 @@ private long saveInternal(FileOutputStream fout,
Step step = new Step(StepType.INODES, filePath);
prog.beginStep(Phase.SAVING_CHECKPOINT, step);
saveInodes(b);
long numErrors = saveSnapshots(b);
// Count number of non-fatal errors when saving inodes and snapshots.
long numErrors = saveInodes(b);
numErrors += saveSnapshots(b);
prog.endStep(Phase.SAVING_CHECKPOINT, step);
step = new Step(StepType.DELEGATION_TOKENS, filePath);