Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9110. Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for better efficiency. Contributed by Charlie Helin.

This commit is contained in:
Andrew Wang 2015-10-09 11:49:16 -07:00
parent de8efc65a4
commit 357b1fd082
2 changed files with 32 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1503,6 +1503,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9181. Better handling of exceptions thrown during upgrade shutdown.
(Wei-Chiu Chuang via Yongjun Zhang)
HDFS-9110. Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for
better efficiency. (Charlie Helin via wang)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

46
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
View File

@ -18,10 +18,14 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.util.List;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -31,7 +35,6 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import com.google.common.base.Preconditions;
import org.apache.hadoop.io.IOUtils;
public abstract class NNUpgradeUtil {
@ -116,21 +119,30 @@ public abstract class NNUpgradeUtil {
// rename current to tmp
renameCurToTmp(sd);
final File curDir = sd.getCurrentDir();
final File tmpDir = sd.getPreviousTmp();
List<String> fileNameList = IOUtils.listDirectory(tmpDir, new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return dir.equals(tmpDir)
&& name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
}
});
final Path curDir = sd.getCurrentDir().toPath();
final Path tmpDir = sd.getPreviousTmp().toPath();
for (String s : fileNameList) {
File prevFile = new File(tmpDir, s);
File newFile = new File(curDir, prevFile.getName());
Files.createLink(newFile.toPath(), prevFile.toPath());
}
Files.walkFileTree(tmpDir,
/* do not follow links */ Collections.<FileVisitOption>emptySet(),
1, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
String name = file.getFileName().toString();
if (Files.isRegularFile(file)
&& name.startsWith(NNStorage.NameNodeFile.EDITS.getName())) {
Path newFile = curDir.resolve(name);
Files.createLink(newFile, file);
}
return super.visitFile(file, attrs);
}
}
);
}
/**