HBASE-21300 Fix the wrong reference file path when restoring snapshots for tables with MOB columns

Signed-off-by: Guanghao Zhang <zghao@apache.org>

hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java

@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MetricsSnapshot;
 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
+import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
@@ -449,10 +450,38 @@ public class CloneSnapshotProcedure
 
     // 3. Move Table temp directory to the hbase root location
     CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir);
-
+    // Move Table temp mob directory to the hbase root location
+    Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName());
+    if (mfs.getFileSystem().exists(tempMobTableDir)) {
+      moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir);
+    }
     return newRegions;
   }
 
+  /**
+   * Move table temp mob directory to the hbase root location
+   * @param mfs The master file system
+   * @param tableDescriptor The table to operate on
+   * @param tempMobTableDir The temp mob directory of table
+   * @throws IOException If failed to move temp mob dir to hbase root dir
+   */
+  private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs,
+      final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException {
+    FileSystem fs = mfs.getFileSystem();
+    final Path tableMobDir =
+        MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName());
+    if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) {
+      throw new IOException("Couldn't delete mob table " + tableMobDir);
+    }
+    if (!fs.exists(tableMobDir.getParent())) {
+      fs.mkdirs(tableMobDir.getParent());
+    }
+    if (!fs.rename(tempMobTableDir, tableMobDir)) {
+      throw new IOException("Unable to move mob table from temp=" + tempMobTableDir
+          + " to hbase root=" + tableMobDir);
+    }
+  }
+
   /**
    * Add regions to hbase:meta table.
    * @param env MasterProcedureEnv

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java

@@ -367,7 +367,17 @@ public final class MobUtils {
    */
   public static Path getMobHome(Configuration conf) {
     Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
-    return new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
+    return getMobHome(hbaseDir);
+  }
+
+  /**
+   * Gets the root dir of the mob files under the qualified HBase root dir.
+   * It's {rootDir}/mobdir.
+   * @param rootDir The qualified path of HBase root directory.
+   * @return The root dir of the mob file.
+   */
+  public static Path getMobHome(Path rootDir) {
+    return new Path(rootDir, MobConstants.MOB_DIR_NAME);
   }
 
   /**
@@ -383,15 +393,37 @@ public final class MobUtils {
     return mobRootDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
   }
 
+  /**
+   * Gets the table dir of the mob files under the qualified HBase root dir.
+   * It's {rootDir}/mobdir/data/${namespace}/${tableName}
+   * @param rootDir The qualified path of HBase root directory.
+   * @param tableName The name of table.
+   * @return The table dir of the mob file.
+   */
+  public static Path getMobTableDir(Path rootDir, TableName tableName) {
+    return FSUtils.getTableDir(getMobHome(rootDir), tableName);
+  }
+
   /**
    * Gets the region dir of the mob files.
-   * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}.
+   * It's {HBASE_DIR}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
    * @param conf The current configuration.
    * @param tableName The current table name.
    * @return The region dir of the mob files.
    */
   public static Path getMobRegionPath(Configuration conf, TableName tableName) {
-    Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
+    return getMobRegionPath(new Path(conf.get(HConstants.HBASE_DIR)), tableName);
+  }
+
+  /**
+   * Gets the region dir of the mob files under the specified root dir.
+   * It's {rootDir}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
+   * @param rootDir The qualified path of HBase root directory.
+   * @param tableName The current table name.
+   * @return The region dir of the mob files.
+   */
+  public static Path getMobRegionPath(Path rootDir, TableName tableName) {
+    Path tablePath = FSUtils.getTableDir(getMobHome(rootDir), tableName);
     RegionInfo regionInfo = getMobRegionInfo(tableName);
     return new Path(tablePath, regionInfo.getEncodedName());
   }

hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java

@@ -620,7 +620,7 @@ public class RestoreSnapshotHelper {
   private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
       final RegionInfo region) throws IOException {
     // clone region info (change embedded tableName with the new one)
-    Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
+    Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName());
     cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java

@@ -20,9 +20,12 @@ package org.apache.hadoop.hbase.snapshot;
 import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.ClassRule;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
@@ -31,7 +34,7 @@ import org.slf4j.LoggerFactory;
 /**
  * Test the restore/clone operation from a file-system point of view.
  */
-@Category(SmallTests.class)
+@Category(MediumTests.class)
 public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper {
 
   @ClassRule
@@ -49,4 +52,13 @@ public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper {
   protected SnapshotMock createSnapshotMock() throws IOException {
     return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
   }
+
+  @Override
+  protected void createTableAndSnapshot(TableName tableName, String snapshotName)
+      throws IOException {
+    byte[] column = Bytes.toBytes("A");
+    Table table = MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, column);
+    TEST_UTIL.loadTable(table, column);
+    TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java

@@ -24,22 +24,32 @@ import java.io.IOException;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -52,7 +62,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.Snapshot
 /**
  * Test the restore/clone operation from a file-system point of view.
  */
-@Category({RegionServerTests.class, SmallTests.class})
+@Category({RegionServerTests.class, MediumTests.class})
 public class TestRestoreSnapshotHelper {
 
   @ClassRule
@@ -72,6 +82,16 @@ public class TestRestoreSnapshotHelper {
   protected void setupConf(Configuration conf) {
   }
 
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    TEST_UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownCluster() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
   @Before
   public void setup() throws Exception {
     rootDir = TEST_UTIL.getDataTestDir("testRestore");
@@ -101,6 +121,51 @@ public class TestRestoreSnapshotHelper {
     restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
   }
 
+  @Test
+  public void testNoHFileLinkInRootDir() throws IOException {
+    rootDir = TEST_UTIL.getDefaultRootDirPath();
+    FSUtils.setRootDir(conf, rootDir);
+    fs = rootDir.getFileSystem(conf);
+
+    TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
+    String snapshotName = tableName.getNameAsString() + "-snapshot";
+    createTableAndSnapshot(tableName, snapshotName);
+
+    Path restoreDir = new Path("/hbase/.tmp-restore");
+    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
+    checkNoHFileLinkInTableDir(tableName);
+  }
+
+  protected void createTableAndSnapshot(TableName tableName, String snapshotName)
+      throws IOException {
+    byte[] column = Bytes.toBytes("A");
+    Table table = TEST_UTIL.createTable(tableName, column, 2);
+    TEST_UTIL.loadTable(table, column);
+    TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
+  }
+
+  private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
+    Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
+        CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
+        CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
+    for (Path tableDir : tableDirs) {
+      Assert.assertFalse(hasHFileLink(tableDir));
+    }
+  }
+
+  private boolean hasHFileLink(Path tableDir) throws IOException {
+    if (fs.exists(tableDir)) {
+      RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
+      while (iterator.hasNext()) {
+        LocatedFileStatus fileStatus = iterator.next();
+        if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
   private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException {
     // Test Rolling-Upgrade like Snapshot.
     // half machines writing using v1 and the others using v2 format.