Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-7118. Improve diagnostics on storage directory rename operations by using NativeIO#renameTo in Storage#rename. Contributed by Chris Nauroth.

This commit is contained in:
cnauroth 2014-09-25 10:03:59 -07:00
parent 6c3cebda3f
commit e0b1dc514f
3 changed files with 49 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -582,6 +582,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7138. Fix hftp to work with encryption. (clamb via wang)
HDFS-7118. Improve diagnostics on storage directory rename operations by
using NativeIO#renameTo in Storage#rename. (cnauroth)
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
View File

@ -36,6 +36,8 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIOException;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.util.VersionInfo;
@ -986,9 +988,13 @@ public abstract class Storage extends StorageInfo {
}
public static void rename(File from, File to) throws IOException {
if (!from.renameTo(to))
throw new IOException("Failed to rename "
+ from.getCanonicalPath() + " to " + to.getCanonicalPath());
try {
NativeIO.renameTo(from, to);
} catch (NativeIOException e) {
throw new IOException("Failed to rename " + from.getCanonicalPath()
+ " to " + to.getCanonicalPath() + " due to failure in native rename. "
+ e.toString());
}
}
/**

37
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
View File

@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.TXNS_PER_FAIL;
import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.TXNS_PER_ROLL;
import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.setupEdits;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@ -42,8 +43,11 @@ import org.apache.hadoop.hdfs.server.namenode.JournalManager.CorruptionException
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.TestEditLog.AbortSpec;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
@ -59,6 +63,9 @@ public class TestFileJournalManager {
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
}
@Rule
public ExpectedException exception = ExpectedException.none();
@Before
public void setUp() {
conf = new Configuration();
@ -472,6 +479,36 @@ public class TestFileJournalManager {
}
}
/**
* Tests that internal renames are done using native code on platforms that
* have it. The native rename includes more detailed information about the
* failure, which can be useful for troubleshooting.
*/
@Test
public void testDoPreUpgradeIOError() throws IOException {
File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
List<URI> editUris = Collections.singletonList(storageDir.toURI());
NNStorage storage = setupEdits(editUris, 5);
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
assertNotNull(sd);
// Change storage directory so that renaming current to previous.tmp fails.
FileUtil.setWritable(storageDir, false);
FileJournalManager jm = null;
try {
jm = new FileJournalManager(conf, sd, storage);
exception.expect(IOException.class);
if (NativeCodeLoader.isNativeCodeLoaded()) {
exception.expectMessage("failure in native rename");
}
jm.doPreUpgrade();
} finally {
IOUtils.cleanup(LOG, jm);
// Restore permissions on storage directory and make sure we can delete.
FileUtil.setWritable(storageDir, true);
FileUtil.fullyDelete(storageDir);
}
}
private static String getLogsAsString(
FileJournalManager fjm, long firstTxId) throws IOException {
return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId, false));