HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1167006 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -255,7 +255,7 @@ Release 0.91.0 - Unreleased
                TestMultiColumnScanner that caught it (Mikhail Bautin)
    HBASE-4007  distributed log splitting can get indefinitely stuck
                (Prakash Khemani)
-           
+   HBASE-4301  META migration from 0.90 to trunk fails (Subbu Iyer)
                
 
   IMPROVEMENTS

src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java

@@ -231,27 +231,23 @@ public class MetaEditor {
   }
 
   /**
-   * Updates the region information for the specified region in META.
+   * Update the metamigrated flag in -ROOT-.
    * @param catalogTracker
-   * @param regionInfo region to be updated in META
    * @throws IOException
    */
-  public static void updateRegionInfo(CatalogTracker catalogTracker,
+  public static void updateRootWithMetaMigrationStatus(
-      HRegionInfo regionInfo)
+      CatalogTracker catalogTracker) throws IOException {
-  throws IOException {
-    Put put = new Put(regionInfo.getRegionName());
-    addRegionInfo(put, regionInfo);
-    catalogTracker.waitForMetaServerConnectionDefault().put(
-        CatalogTracker.META_REGION, put);
-    LOG.info("Updated region " + regionInfo.getRegionNameAsString() + " in META");
-  }
-
-  public static void updateRootWithMetaMigrationStatus(CatalogTracker catalogTracker) throws IOException {
     updateRootWithMetaMigrationStatus(catalogTracker, true);
   }
 
-  public static void updateRootWithMetaMigrationStatus(CatalogTracker catalogTracker,
+  /**
-                                                       boolean metaUpdated)
+   * Update the metamigrated flag in -ROOT-.
+   * @param catalogTracker
+   * @param metaUpdated
+   * @throws IOException
+   */
+  public static void updateRootWithMetaMigrationStatus(
+      CatalogTracker catalogTracker, boolean metaUpdated)
       throws IOException {
     Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
     addMetaUpdateStatus(put, metaUpdated);
@@ -260,6 +256,12 @@ public class MetaEditor {
     LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
   }
 
+  /**
+   * Update legacy META rows, removing HTD from HRI.
+   * @param masterServices
+   * @return
+   * @throws IOException
+   */
   public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
       final MasterServices masterServices)
   throws IOException {
@@ -271,16 +273,11 @@ public class MetaEditor {
         HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
         if (hrfm == null) return true;
         htds.add(hrfm.getTableDesc());
-        masterServices.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc());
+        masterServices.getMasterFileSystem()
-        HRegionInfo regionInfo = new HRegionInfo(hrfm);
+          .createTableDescriptor(hrfm.getTableDesc());
-        LOG.debug(" MetaEditor.updatemeta RegionInfo = " + regionInfo.toString()
+        updateHRI(masterServices.getCatalogTracker()
-        + " old HRI = " + hrfm.toString());
+            .waitForMetaServerConnectionDefault(),
-        Put put = new Put(regionInfo.getRegionName());
+            hrfm, CatalogTracker.META_REGION);
-        put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-            Writables.getBytes(regionInfo));
-        masterServices.getCatalogTracker().waitForMetaServerConnectionDefault().put(
-            CatalogTracker.META_REGION, put);
-        LOG.info("Updated region " + regionInfo + " to META");
         return true;
       }
     };
@@ -289,6 +286,60 @@ public class MetaEditor {
     return htds;
   }
 
+  /**
+   * Migrate root and meta to newer version. This updates the META and ROOT
+   * and removes the HTD from HRI.
+   * @param masterServices
+   * @throws IOException
+   */
+  public static void migrateRootAndMeta(final MasterServices masterServices)
+      throws IOException {
+    updateRootWithNewRegionInfo(masterServices);
+    updateMetaWithNewRegionInfo(masterServices);
+  }
+
+  /**
+   * Update the ROOT with new HRI. (HRI with no HTD)
+   * @param masterServices
+   * @return
+   * @throws IOException
+   */
+  public static List<HTableDescriptor> updateRootWithNewRegionInfo(
+      final MasterServices masterServices)
+  throws IOException {
+    final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+    Visitor v = new Visitor() {
+      @Override
+      public boolean visit(Result r) throws IOException {
+        if (r ==  null || r.isEmpty()) return true;
+        HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
+        if (hrfm == null) return true;
+        htds.add(hrfm.getTableDesc());
+        masterServices.getMasterFileSystem().createTableDescriptor(
+            hrfm.getTableDesc());
+        updateHRI(masterServices.getCatalogTracker()
+            .waitForRootServerConnectionDefault(),
+            hrfm, CatalogTracker.ROOT_REGION);
+        return true;
+      }
+    };
+    MetaReader.fullScan(
+        masterServices.getCatalogTracker().waitForRootServerConnectionDefault(),
+        v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null);
+    return htds;
+  }
+
+  private static void updateHRI(HRegionInterface hRegionInterface,
+                                HRegionInfo090x hRegionInfo090x, byte[] regionName)
+    throws IOException {
+    HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
+    Put put = new Put(regionInfo.getRegionName());
+    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(regionInfo));
+    hRegionInterface.put(regionName, put);
+    LOG.info("Updated region " + regionInfo + " to " + Bytes.toString(regionName));
+  }
+
   public static HRegionInfo090x getHRegionInfoForMigration(
       Result data) throws IOException {
     HRegionInfo090x info = null;

src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java

@@ -252,19 +252,8 @@ public class MetaReader {
   throws IOException {
     HRegionInterface metaServer =
       catalogTracker.waitForMetaServerConnectionDefault();
-    Scan scan = new Scan();
+    fullScan(metaServer, visitor,
-    if (startrow != null) scan.setStartRow(startrow);
+        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow);
-    scan.addFamily(HConstants.CATALOG_FAMILY);
-    long scannerid = metaServer.openScanner(
-        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
-    try {
-      Result data;
-      while((data = metaServer.next(scannerid)) != null) {
-        if (!data.isEmpty()) visitor.visit(data);
-      }
-    } finally {
-      metaServer.close(scannerid);
-    }
     return;
   }
 
@@ -622,7 +611,32 @@ public class MetaReader {
     return regions;
   }
 
-
+  /**
+   * Fully scan a given region, on a given server starting with given row.
+   * @param hRegionInterface region server
+   * @param visitor visitor
+   * @param regionName name of region
+   * @param startrow start row
+   * @throws IOException
+   */
+  public static void fullScan(HRegionInterface hRegionInterface,
+                              Visitor visitor, final byte[] regionName,
+                              byte[] startrow) throws IOException {
+    if (hRegionInterface == null) return;
+    Scan scan = new Scan();
+    if (startrow != null) scan.setStartRow(startrow);
+    scan.addFamily(HConstants.CATALOG_FAMILY);
+    long scannerid = hRegionInterface.openScanner(regionName, scan);
+    try {
+      Result data;
+      while((data = hRegionInterface.next(scannerid)) != null) {
+        if (!data.isEmpty()) visitor.visit(data);
+      }
+    } finally {
+      hRegionInterface.close(scannerid);
+    }
+    return;
+  }
 
 
   /**

src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java

@@ -914,7 +914,8 @@ public class HConnectionManager {
           // possible we got a region of a different table...
           if (!Bytes.equals(regionInfo.getTableName(), tableName)) {
             throw new TableNotFoundException(
-              "Table '" + Bytes.toString(tableName) + "' was not found.");
+                  "Table '" + Bytes.toString(tableName) + "' was not found, got: " +
+                  Bytes.toString(regionInfo.getTableName()) + ".");
           }
           if (regionInfo.isSplit()) {
             throw new RegionOfflineException("the only available region for" +

src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -526,14 +526,15 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
     if (!isMetaHRIUpdated()) {
       LOG.info("Meta has HRI with HTDs. Updating meta now.");
       try {
-        MetaEditor.updateMetaWithNewRegionInfo(this);
+        MetaEditor.migrateRootAndMeta(this);
-        LOG.info("Meta updated with new HRI.");
+        LOG.info("ROOT and Meta updated with new HRI.");
         return true;
       } catch (IOException e) {
-        throw new RuntimeException("Update Meta with nw HRI failed. Master startup aborted.");
+        throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
+            "Master startup aborted.");
       }
     }
-    LOG.info("Meta already up-to date with new HRI.");
+    LOG.info("ROOT/Meta already up-to date with new HRI.");
     return true;
   }
 
@@ -977,7 +978,6 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
     return hRegionInfos;
   }
 
-
   private static boolean isCatalogTable(final byte [] tableName) {
     return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
            Bytes.equals(tableName, HConstants.META_TABLE_NAME);

src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java

@@ -327,9 +327,18 @@ public class MasterFileSystem {
     if (!FSUtils.rootRegionExists(fs, rd)) {
       bootstrap(rd, c);
     }
+    createRootTableInfo(rd);
     return rd;
   }
 
+  private void createRootTableInfo(Path rd) throws IOException {
+    // Create ROOT tableInfo if required.
+    if (!FSUtils.tableInfoExists(fs, rd,
+        Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
+      FSUtils.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf);
+    }
+  }
+
   private static void bootstrap(final Path rd, final Configuration c)
   throws IOException {
     LOG.info("BOOTSTRAP: creating ROOT and first META regions");

src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java

@@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 
 /**
  * Implementation of {@link TableDescriptors} that reads descriptors from the
@@ -39,6 +42,8 @@ import org.apache.hadoop.hbase.TableExistsException;
  * the filesystem or can be read and write.
  */
 public class FSTableDescriptors implements TableDescriptors {
+
+  private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
   private final FileSystem fs;
   private final Path rootdir;
   private final boolean fsreadonly;
@@ -151,7 +156,14 @@ public class FSTableDescriptors implements TableDescriptors {
     Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
     List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
     for (Path d: tableDirs) {
-      HTableDescriptor htd = get(d.getName());
+      HTableDescriptor htd = null;
+      try {
+
+        htd = get(d.getName());
+      } catch (FileNotFoundException fnfe) {
+        // inability of retrieving one HTD shouldn't stop getting the remaining
+        LOG.warn("Trouble retrieving htd", fnfe);
+      }
       if (htd == null) continue;
       htds.put(d.getName(), htd);
     }

src/main/java/org/apache/hadoop/hbase/util/FSUtils.java

@@ -490,6 +490,20 @@ public abstract class FSUtils {
     return fs.exists(rootRegionDir);
   }
 
+  /**
+   * Checks if .tableinfo exists for given table
+   * 
+   * @param fs file system
+   * @param rootdir root directory of HBase installation
+   * @param tableName name of table
+   * @return true if exists
+   * @throws IOException
+   */
+  public static boolean tableInfoExists(FileSystem fs, Path rootdir,
+      String tableName) throws IOException {
+    Path tablePath = getTableInfoPath(rootdir, tableName);
+    return fs.exists(tablePath);
+  }
 
   /**
    * Compute HDFS blocks distribution of a given file, or a portion of the file