HADOOP-12678. Handle empty rename pending metadata file during atomic rename in redo path. Contributed by Madhumita Chakraborty.

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -1571,6 +1571,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12675. Fix description about retention period in usage of expunge
     command. (Masatake Iwasaki via stevel)
 
+    HADOOP-12678. Handle empty rename pending metadata file during atomic rename
+    in redo path. (Madhumita Chakraborty via cnauroth)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java

@@ -143,9 +143,15 @@ public class NativeAzureFileSystem extends FileSystem {
       FSDataInputStream input = fs.open(f);
       byte[] bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
       int l = input.read(bytes);
-      if (l < 0) {
-        throw new IOException(
-            "Error reading pending rename file contents -- no data available");
+      if (l <= 0) {
+        // Jira HADOOP-12678 -Handle empty rename pending metadata file during
+        // atomic rename in redo path. If during renamepending file is created
+        // but not written yet, then this means that rename operation
+        // has not started yet. So we should delete rename pending metadata file.
+        LOG.error("Deleting empty rename pending file "
+            + redoFile + " -- no data available");
+        deleteRenamePendingFile(fs, redoFile);
+        return;
       }
       if (l == MAX_RENAME_PENDING_FILE_SIZE) {
         throw new IOException(
@@ -178,7 +184,7 @@ public class NativeAzureFileSystem extends FileSystem {
             redoFile, contents);
 
         // delete the -RenamePending.json file
-        fs.delete(redoFile, false);
+        deleteRenamePendingFile(fs, redoFile);
         return;
       }
 
@@ -215,6 +221,30 @@ public class NativeAzureFileSystem extends FileSystem {
       return folderLease;
     }
 
+    /**
+     * Deletes rename pending metadata file
+     * @param fs -- the file system
+     * @param redoFile - rename pending metadata file path
+     * @throws IOException - If deletion fails
+     */
+    @VisibleForTesting
+    void deleteRenamePendingFile(FileSystem fs, Path redoFile)
+        throws IOException {
+      try {
+        fs.delete(redoFile, false);
+      } catch (IOException e) {
+        // If the rename metadata was not found then somebody probably
+        // raced with us and finished the delete first
+        Throwable t = e.getCause();
+        if (t != null && t instanceof StorageException
+            && "BlobNotFound".equals(((StorageException) t).getErrorCode())) {
+          LOG.warn("rename pending file " + redoFile + " is already deleted");
+        } else {
+          throw e;
+        }
+      }
+    }
+
     /**
      * Write to disk the information needed to redo folder rename,
      * in JSON format. The file name will be

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java

@@ -845,6 +845,63 @@ public abstract class NativeAzureFileSystemBaseTest {
     assertTrue(fs.exists(new Path(inner2renamed, "file")));
   }
 
+  /**
+   * Test the situation when the rename metadata file is empty
+   * i.e. it is created but not written yet. In that case in next rename
+   * this empty file should be deleted. As zero byte metadata file means
+   * rename has not started yet. This is to emulate the scenario where
+   * the process crashes just after creating rename metadata file.
+   *  We had a bug (HADOOP-12678) that in that case listing used to fail and
+   * hbase master did not use to come up
+   */
+  @Test
+  public void testRedoRenameFolderInFolderListingWithZeroByteRenameMetadata()
+      throws IOException {
+    // create original folder
+    String parent = "parent";
+    Path parentFolder = new Path(parent);
+    assertTrue(fs.mkdirs(parentFolder));
+    Path inner = new Path(parentFolder, "innerFolder");
+    assertTrue(fs.mkdirs(inner));
+    Path inner2 = new Path(parentFolder, "innerFolder2");
+    assertTrue(fs.mkdirs(inner2));
+    Path innerFile = new Path(inner2, "file");
+    assertTrue(fs.createNewFile(innerFile));
+
+    Path inner2renamed = new Path(parentFolder, "innerFolder2Renamed");
+
+    // Create an empty rename-pending file
+    final String renamePendingStr = inner2 + FolderRenamePending.SUFFIX;
+    Path renamePendingFile = new Path(renamePendingStr);
+    FSDataOutputStream out = fs.create(renamePendingFile, true);
+    assertTrue(out != null);
+    out.close();
+
+    // Redo the rename operation based on the contents of the
+    // -RenamePending.json file. Trigger the redo by listing
+    // the parent folder. It should not throw and it should
+    // delete empty rename pending file
+    FileStatus[] listed = fs.listStatus(parentFolder);
+    assertEquals(2, listed.length);
+    assertTrue(listed[0].isDirectory());
+    assertTrue(listed[1].isDirectory());
+    assertFalse(fs.exists(renamePendingFile));
+
+    // Verify that even if rename pending file is deleted,
+    // deletion should handle that
+    Path home = fs.getHomeDirectory();
+    String relativeHomeDir = getRelativePath(home.toString());
+    NativeAzureFileSystem.FolderRenamePending pending =
+            new NativeAzureFileSystem.FolderRenamePending(
+                relativeHomeDir + "/" + inner2,
+                relativeHomeDir + "/" + inner2renamed, null,
+                (NativeAzureFileSystem) fs);
+    pending.deleteRenamePendingFile(fs, renamePendingFile);
+
+    assertTrue(fs.exists(inner2)); // verify original folder is there
+    assertFalse(fs.exists(inner2renamed)); // verify the target is not there
+  }
+
   /**
    * Test the situation where a rename pending file exists but the rename
    * is really done. This could happen if the rename process died just