HBASE-480 Tool to manually merge two regions

Commit to TRUNK and branch.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@632663 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -71,6 +71,7 @@ Hbase Change Log
    HBASE-466   Move HMasterInterface, HRegionInterface, and 
                HMasterRegionInterface into o.a.h.h.ipc
    HBASE-479   Speed up TestLogRolling
+   HBASE-480   Tool to manually merge two regions
    
 Branch 0.1
 

src/java/org/apache/hadoop/hbase/HMerge.java

@@ -22,35 +22,42 @@ package org.apache.hadoop.hbase;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Random;
+import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
-
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.regionserver.HLog;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /** 
  * A non-instantiable class that has a static method capable of compacting
  * a table by merging adjacent regions that have grown too small.
  */
-class HMerge implements HConstants {
+class HMerge implements HConstants, Tool {
   static final Log LOG = LogFactory.getLog(HMerge.class);
   static final Random rand = new Random();
+  private Configuration conf;
   
+  /*
+   * Not instantiable
+   */
   private HMerge() {
-    // Not instantiable
+    super();
   }
   
   /**
@@ -67,8 +74,8 @@ class HMerge implements HConstants {
    * @throws IOException
    */
   public static void merge(HBaseConfiguration conf, FileSystem fs,
-      Text tableName) throws IOException {
+    Text tableName)
-    
+  throws IOException {
     HConnection connection = HConnectionManager.getConnection(conf);
     boolean masterIsRunning = connection.isMasterRunning();
     HConnectionManager.deleteConnection(conf);
@@ -78,7 +85,6 @@ class HMerge implements HConstants {
             "Can not compact META table if instance is on-line");
       }
       new OfflineMerger(conf, fs).process();
-
     } else {
       if(!masterIsRunning) {
         throw new IllegalStateException(
@@ -117,9 +123,8 @@ class HMerge implements HConstants {
     void process() throws IOException {
       try {
         for(HRegionInfo[] regionsToMerge = next();
-        regionsToMerge != null;
+            regionsToMerge != null;
-        regionsToMerge = next()) {
+            regionsToMerge = next()) {
-
           if (!merge(regionsToMerge)) {
             return;
           }
@@ -134,7 +139,12 @@ class HMerge implements HConstants {
       }
     }
     
-    private boolean merge(HRegionInfo[] info) throws IOException {
+    protected boolean merge(final HRegionInfo[] info) throws IOException {
+      return merge(info, false);
+    }
+    
+    protected boolean merge(final HRegionInfo[] info, final boolean force)
+    throws IOException {
       if(info.length < 2) {
         LOG.info("only one region - nothing to merge");
         return false;
@@ -156,23 +166,19 @@ class HMerge implements HConstants {
 
         nextSize = nextRegion.largestHStore(midKey).getAggregate();
 
-        if ((currentSize + nextSize) <= (maxFilesize / 2)) {
+        if (force || (currentSize + nextSize) <= (maxFilesize / 2)) {
           // We merge two adjacent regions if their total size is less than
           // one half of the desired maximum size
-
           LOG.info("merging regions " + currentRegion.getRegionName()
               + " and " + nextRegion.getRegionName());
-
+          HRegion mergedRegion =
-          HRegion mergedRegion = HRegion.closeAndMerge(currentRegion, nextRegion);
+            HRegion.closeAndMerge(currentRegion, nextRegion);
-
           updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
               mergedRegion);
-
           break;
         }
         LOG.info("not merging regions " + currentRegion.getRegionName()
             + " and " + nextRegion.getRegionName());
-
         currentRegion.close();
         currentRegion = nextRegion;
         currentSize = nextSize;
@@ -199,7 +205,6 @@ class HMerge implements HConstants {
     
     OnlineMerger(HBaseConfiguration conf, FileSystem fs, Text tableName)
     throws IOException {
-      
       super(conf, fs, tableName);
       this.tableName = tableName;
       this.table = new HTable(conf, META_TABLE_NAME);
@@ -209,35 +214,26 @@ class HMerge implements HConstants {
     
     private HRegionInfo nextRegion() throws IOException {
       try {
-        HStoreKey key = new HStoreKey();
+        Map<Text, byte[]> results = getMetaRow();
-        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
+        if (results == null) {
-        if (! metaScanner.next(key, results)) {
           return null;
         }
         byte[] bytes = results.get(COL_REGIONINFO);
         if (bytes == null || bytes.length == 0) {
-          throw new NoSuchElementException("meta region entry missing "
+          throw new NoSuchElementException("meta region entry missing " +
-              + COL_REGIONINFO);
+            COL_REGIONINFO);
         }
-        HRegionInfo region =
+        HRegionInfo region = Writables.getHRegionInfo(bytes);
-          (HRegionInfo) Writables.getWritable(bytes, new HRegionInfo());
+        if (!region.getTableDesc().getName().equals(this.tableName)) {
-
-        if (!region.getTableDesc().getName().equals(tableName)) {
           return null;
         }
-        
+        checkOfflined(region);
-        if (!region.isOffline()) {
-          throw new TableNotDisabledException("region " + region.getRegionName()
-              + " is not disabled");
-        }
         return region;
-        
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
         LOG.error("meta scanner error", e);
         try {
           metaScanner.close();
-          
         } catch (IOException ex) {
           ex = RemoteExceptionHandler.checkIOException(ex);
           LOG.error("error closing scanner", ex);
@@ -245,6 +241,35 @@ class HMerge implements HConstants {
         throw e;
       }
     }
+    
+    protected void checkOfflined(final HRegionInfo hri)
+    throws TableNotDisabledException {
+      if (!hri.isOffline()) {
+        throw new TableNotDisabledException("region " +
+          hri.getRegionName() + " is not disabled");
+      }
+    }
+    
+    /*
+     * Check current row has a HRegionInfo.  Skip to next row if HRI is empty.
+     * @return A Map of the row content else null if we are off the end.
+     * @throws IOException
+     */
+    private Map<Text, byte[]> getMetaRow() throws IOException {
+      HStoreKey key = new HStoreKey();
+      SortedMap<Text, byte[]> value = new TreeMap<Text, byte[]>();
+      boolean foundResult = false;
+      while (metaScanner.next(key, value)) {
+        LOG.info("Row: <" + key.getRow() + ">");
+        byte[] bytes = value.get(COL_REGIONINFO);
+        if (bytes == null || bytes.length == 0) {
+          continue;
+        }
+        foundResult = true;
+        break;
+      }
+      return foundResult? value: null;
+    }
 
     @Override
     protected HRegionInfo[] next() throws IOException {
@@ -376,4 +401,94 @@ class HMerge implements HConstants {
       }
     }
   }
-}
+
+  public int run(String[] args) throws Exception {
+    if (args.length == 0 || args.length > 4) {
+      printUsage();
+      return 1;
+    }
+    final String masterPrefix = "--master=";
+    String tableName = null;
+    String loRegion = null;
+    String hiRegion = null;
+    for (int i = 0; i < args.length; i++) {
+      String arg = args[i];
+      if (arg.startsWith(masterPrefix)) {
+        this.conf.set("hbase.master", arg.substring(masterPrefix.length()));
+      } else if (tableName == null) {
+        tableName = arg;
+        continue;
+      } else if (loRegion == null) {
+        loRegion = arg;
+        continue;
+      } else if (hiRegion == null) {
+        hiRegion = arg;
+        continue;
+      } else {
+        throw new IllegalArgumentException("Unsupported argument: " + arg);
+      }
+    }
+    // Make finals of the region names so can be refererred to by anonymous
+    // class.
+    final Text lo = new Text(loRegion);
+    final Text hi = new Text(hiRegion);
+    // Make a version of OnlineMerger that does two regions only.
+    Merger m = new OnlineMerger((HBaseConfiguration)this.conf,
+        FileSystem.get(this.conf), new Text(tableName)) {
+      @Override
+      void process() throws IOException {
+        try {
+          for (HRegionInfo[] regionsToMerge = next(); regionsToMerge != null;
+                regionsToMerge = next()) {
+            if (regionsToMerge[0].getRegionName().equals(lo) &&
+                  regionsToMerge[1].getRegionName().equals(hi)) {
+              merge(regionsToMerge, true);
+              // Done
+              break;
+            }
+          }
+        } finally {
+          try {
+            this.hlog.closeAndDelete();
+          } catch(IOException e) {
+            LOG.error(e);
+          }
+        }
+      }
+      
+      @Override
+      protected void checkOfflined(@SuppressWarnings("unused") HRegionInfo hri)
+      throws TableNotDisabledException {
+        // Disabling does not work reliably.  Just go ahead and merge.
+        return;
+      }
+    };
+    m.process();
+    return 0;
+  }
+
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  public void setConf(final Configuration c) {
+    this.conf = c;
+  }
+  
+  static int printUsage() {
+    System.out.println("merge [--master=MASTER:PORT] <tableName> " +
+      "<lo-region> <hi-region>");
+    System.out.println("Presumes quiescent/offlined table -- does not check");
+    return -1;
+  }
+
+  /**
+   * Run merging of two regions.
+   * @param args
+   * @throws Exception 
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(new HBaseConfiguration(), new HMerge(), args);
+    System.exit(errCode);
+  }
+}