diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 40b47cb3d71..28077b26211 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -947,6 +947,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12675. Fix description about retention period in usage of expunge command. (Masatake Iwasaki via stevel) + HADOOP-12678. Handle empty rename pending metadata file during atomic rename + in redo path. (Madhumita Chakraborty via cnauroth) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 34791e54ca0..96786aa0b70 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -143,9 +143,15 @@ public class NativeAzureFileSystem extends FileSystem { FSDataInputStream input = fs.open(f); byte[] bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE]; int l = input.read(bytes); - if (l < 0) { - throw new IOException( - "Error reading pending rename file contents -- no data available"); + if (l <= 0) { + // Jira HADOOP-12678 -Handle empty rename pending metadata file during + // atomic rename in redo path. If during renamepending file is created + // but not written yet, then this means that rename operation + // has not started yet. So we should delete rename pending metadata file. + LOG.error("Deleting empty rename pending file " + + redoFile + " -- no data available"); + deleteRenamePendingFile(fs, redoFile); + return; } if (l == MAX_RENAME_PENDING_FILE_SIZE) { throw new IOException( @@ -178,7 +184,7 @@ public class NativeAzureFileSystem extends FileSystem { redoFile, contents); // delete the -RenamePending.json file - fs.delete(redoFile, false); + deleteRenamePendingFile(fs, redoFile); return; } @@ -215,6 +221,30 @@ public class NativeAzureFileSystem extends FileSystem { return folderLease; } + /** + * Deletes rename pending metadata file + * @param fs -- the file system + * @param redoFile - rename pending metadata file path + * @throws IOException - If deletion fails + */ + @VisibleForTesting + void deleteRenamePendingFile(FileSystem fs, Path redoFile) + throws IOException { + try { + fs.delete(redoFile, false); + } catch (IOException e) { + // If the rename metadata was not found then somebody probably + // raced with us and finished the delete first + Throwable t = e.getCause(); + if (t != null && t instanceof StorageException + && "BlobNotFound".equals(((StorageException) t).getErrorCode())) { + LOG.warn("rename pending file " + redoFile + " is already deleted"); + } else { + throw e; + } + } + } + /** * Write to disk the information needed to redo folder rename, * in JSON format. The file name will be diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java index 1f07677fc23..78370986fd5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java @@ -845,6 +845,63 @@ public abstract class NativeAzureFileSystemBaseTest { assertTrue(fs.exists(new Path(inner2renamed, "file"))); } + /** + * Test the situation when the rename metadata file is empty + * i.e. it is created but not written yet. In that case in next rename + * this empty file should be deleted. As zero byte metadata file means + * rename has not started yet. This is to emulate the scenario where + * the process crashes just after creating rename metadata file. + * We had a bug (HADOOP-12678) that in that case listing used to fail and + * hbase master did not use to come up + */ + @Test + public void testRedoRenameFolderInFolderListingWithZeroByteRenameMetadata() + throws IOException { + // create original folder + String parent = "parent"; + Path parentFolder = new Path(parent); + assertTrue(fs.mkdirs(parentFolder)); + Path inner = new Path(parentFolder, "innerFolder"); + assertTrue(fs.mkdirs(inner)); + Path inner2 = new Path(parentFolder, "innerFolder2"); + assertTrue(fs.mkdirs(inner2)); + Path innerFile = new Path(inner2, "file"); + assertTrue(fs.createNewFile(innerFile)); + + Path inner2renamed = new Path(parentFolder, "innerFolder2Renamed"); + + // Create an empty rename-pending file + final String renamePendingStr = inner2 + FolderRenamePending.SUFFIX; + Path renamePendingFile = new Path(renamePendingStr); + FSDataOutputStream out = fs.create(renamePendingFile, true); + assertTrue(out != null); + out.close(); + + // Redo the rename operation based on the contents of the + // -RenamePending.json file. Trigger the redo by listing + // the parent folder. It should not throw and it should + // delete empty rename pending file + FileStatus[] listed = fs.listStatus(parentFolder); + assertEquals(2, listed.length); + assertTrue(listed[0].isDirectory()); + assertTrue(listed[1].isDirectory()); + assertFalse(fs.exists(renamePendingFile)); + + // Verify that even if rename pending file is deleted, + // deletion should handle that + Path home = fs.getHomeDirectory(); + String relativeHomeDir = getRelativePath(home.toString()); + NativeAzureFileSystem.FolderRenamePending pending = + new NativeAzureFileSystem.FolderRenamePending( + relativeHomeDir + "/" + inner2, + relativeHomeDir + "/" + inner2renamed, null, + (NativeAzureFileSystem) fs); + pending.deleteRenamePendingFile(fs, renamePendingFile); + + assertTrue(fs.exists(inner2)); // verify original folder is there + assertFalse(fs.exists(inner2renamed)); // verify the target is not there + } + /** * Test the situation where a rename pending file exists but the rename * is really done. This could happen if the rename process died just