HADOOP-12678. Handle empty rename pending metadata file during atomic rename in redo path. Contributed by Madhumita Chakraborty.

This commit is contained in:
cnauroth 2016-01-08 20:18:54 -08:00
parent 109e528ef5
commit f0fa6d869b
3 changed files with 94 additions and 4 deletions

View File

@ -1571,6 +1571,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12675. Fix description about retention period in usage of expunge
command. (Masatake Iwasaki via stevel)
HADOOP-12678. Handle empty rename pending metadata file during atomic rename
in redo path. (Madhumita Chakraborty via cnauroth)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

View File

@ -143,9 +143,15 @@ public class NativeAzureFileSystem extends FileSystem {
FSDataInputStream input = fs.open(f);
byte[] bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
int l = input.read(bytes);
if (l < 0) {
throw new IOException(
"Error reading pending rename file contents -- no data available");
if (l <= 0) {
// Jira HADOOP-12678 -Handle empty rename pending metadata file during
// atomic rename in redo path. If during renamepending file is created
// but not written yet, then this means that rename operation
// has not started yet. So we should delete rename pending metadata file.
LOG.error("Deleting empty rename pending file "
+ redoFile + " -- no data available");
deleteRenamePendingFile(fs, redoFile);
return;
}
if (l == MAX_RENAME_PENDING_FILE_SIZE) {
throw new IOException(
@ -178,7 +184,7 @@ public class NativeAzureFileSystem extends FileSystem {
redoFile, contents);
// delete the -RenamePending.json file
fs.delete(redoFile, false);
deleteRenamePendingFile(fs, redoFile);
return;
}
@ -215,6 +221,30 @@ public class NativeAzureFileSystem extends FileSystem {
return folderLease;
}
/**
* Deletes rename pending metadata file
* @param fs -- the file system
* @param redoFile - rename pending metadata file path
* @throws IOException - If deletion fails
*/
@VisibleForTesting
void deleteRenamePendingFile(FileSystem fs, Path redoFile)
throws IOException {
try {
fs.delete(redoFile, false);
} catch (IOException e) {
// If the rename metadata was not found then somebody probably
// raced with us and finished the delete first
Throwable t = e.getCause();
if (t != null && t instanceof StorageException
&& "BlobNotFound".equals(((StorageException) t).getErrorCode())) {
LOG.warn("rename pending file " + redoFile + " is already deleted");
} else {
throw e;
}
}
}
/**
* Write to disk the information needed to redo folder rename,
* in JSON format. The file name will be

View File

@ -845,6 +845,63 @@ public abstract class NativeAzureFileSystemBaseTest {
assertTrue(fs.exists(new Path(inner2renamed, "file")));
}
/**
* Test the situation when the rename metadata file is empty
* i.e. it is created but not written yet. In that case in next rename
* this empty file should be deleted. As zero byte metadata file means
* rename has not started yet. This is to emulate the scenario where
* the process crashes just after creating rename metadata file.
* We had a bug (HADOOP-12678) that in that case listing used to fail and
* hbase master did not use to come up
*/
@Test
public void testRedoRenameFolderInFolderListingWithZeroByteRenameMetadata()
throws IOException {
// create original folder
String parent = "parent";
Path parentFolder = new Path(parent);
assertTrue(fs.mkdirs(parentFolder));
Path inner = new Path(parentFolder, "innerFolder");
assertTrue(fs.mkdirs(inner));
Path inner2 = new Path(parentFolder, "innerFolder2");
assertTrue(fs.mkdirs(inner2));
Path innerFile = new Path(inner2, "file");
assertTrue(fs.createNewFile(innerFile));
Path inner2renamed = new Path(parentFolder, "innerFolder2Renamed");
// Create an empty rename-pending file
final String renamePendingStr = inner2 + FolderRenamePending.SUFFIX;
Path renamePendingFile = new Path(renamePendingStr);
FSDataOutputStream out = fs.create(renamePendingFile, true);
assertTrue(out != null);
out.close();
// Redo the rename operation based on the contents of the
// -RenamePending.json file. Trigger the redo by listing
// the parent folder. It should not throw and it should
// delete empty rename pending file
FileStatus[] listed = fs.listStatus(parentFolder);
assertEquals(2, listed.length);
assertTrue(listed[0].isDirectory());
assertTrue(listed[1].isDirectory());
assertFalse(fs.exists(renamePendingFile));
// Verify that even if rename pending file is deleted,
// deletion should handle that
Path home = fs.getHomeDirectory();
String relativeHomeDir = getRelativePath(home.toString());
NativeAzureFileSystem.FolderRenamePending pending =
new NativeAzureFileSystem.FolderRenamePending(
relativeHomeDir + "/" + inner2,
relativeHomeDir + "/" + inner2renamed, null,
(NativeAzureFileSystem) fs);
pending.deleteRenamePendingFile(fs, renamePendingFile);
assertTrue(fs.exists(inner2)); // verify original folder is there
assertFalse(fs.exists(inner2renamed)); // verify the target is not there
}
/**
* Test the situation where a rename pending file exists but the rename
* is really done. This could happen if the rename process died just