diff --git a/CHANGES.txt b/CHANGES.txt index f2838038062..b53793a0efd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -443,6 +443,7 @@ Release 0.91.0 - Unreleased HBASE-1730 Online Schema Changes HBASE-4206 jenkins hash implementation uses longs unnecessarily (Ron Yang) + HBASE-3842 Refactor Coprocessor Compaction API TASKS HBASE-3559 Move report of split to master OFF the heartbeat channel diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java index d473ba71a1f..54106f08f3a 100644 --- a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.coprocessor; import java.util.List; import java.util.Map; +import com.google.common.collect.ImmutableList; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; @@ -33,6 +34,8 @@ import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -78,12 +81,22 @@ public abstract class BaseRegionObserver implements RegionObserver { HRegion l, HRegion r) { } @Override - public void preCompact(ObserverContext e, - boolean willSplit) { } + public void preCompactSelection(final ObserverContext c, + final Store store, final List candidates) { } + + @Override + public void postCompactSelection(final ObserverContext c, + final Store store, final ImmutableList selected) { } + + @Override + public InternalScanner preCompact(ObserverContext e, + final Store store, final InternalScanner scanner) { + return scanner; + } @Override public void postCompact(ObserverContext e, - boolean willSplit) { } + final Store store, final StoreFile resultFile) { } @Override public void preGetClosestRowBefore(final ObserverContext e, diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 008d027f903..bdf888ce7d7 100644 --- a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.coprocessor; import java.util.List; import java.util.Map; +import com.google.common.collect.ImmutableList; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; @@ -33,6 +34,8 @@ import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -69,22 +72,61 @@ public interface RegionObserver extends Coprocessor { void postFlush(final ObserverContext c); /** - * Called before compaction. + * Called prior to selecting the {@link StoreFile}s to compact from the list + * of available candidates. To alter the files used for compaction, you may + * mutate the passed in list of candidates. * @param c the environment provided by the region server - * @param willSplit true if compaction will result in a split, false - * otherwise + * @param store the store where compaction is being requested + * @param candidates the store files currently available for compaction */ - void preCompact(final ObserverContext c, - final boolean willSplit); + void preCompactSelection(final ObserverContext c, + final Store store, final List candidates); /** - * Called after compaction. + * Called after the {@link StoreFile}s to compact have been selected from the + * available candidates. * @param c the environment provided by the region server - * @param willSplit true if compaction will result in a split, false - * otherwise + * @param store the store being compacted + * @param selected the store files selected to compact + */ + void postCompactSelection(final ObserverContext c, + final Store store, final ImmutableList selected); + + /** + * Called prior to writing the {@link StoreFile}s selected for compaction into + * a new {@code StoreFile}. To override or modify the compaction process, + * implementing classes have two options: + *
    + *
  • Wrap the provided {@link InternalScanner} with a custom + * implementation that is returned from this method. The custom scanner + * can then inspect {@link KeyValue}s from the wrapped scanner, applying + * its own policy to what gets written.
    • + *
  • Call {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} + * and provide a custom implementation for writing of new + * {@link StoreFile}s. Note: any implementations bypassing + * core compaction using this approach must write out new store files + * themselves or the existing data will no longer be available after + * compaction.
    • + *
+ * @param c the environment provided by the region server + * @param store the store being compacted + * @param scanner the scanner over existing data used in the store file + * rewriting + * @return the scanner to use during compaction. Should not be {@code null} + * unless the implementation is writing new store files on its own. + */ + InternalScanner preCompact(final ObserverContext c, + final Store store, final InternalScanner scanner); + + /** + * Called after compaction has completed and the new store file has been + * moved in to place. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param resultFile the new store file written out during compaction */ void postCompact(final ObserverContext c, - final boolean willSplit); + final Store store, StoreFile resultFile); /** * Called before the region is split. diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 30c9d69a040..a6c7147dd7c 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -937,10 +937,6 @@ public class HRegion implements HeapSize { // , Writable{ LOG.debug("Skipping compaction on " + this + " because closed"); return false; } - if (coprocessorHost != null) { - status.setStatus("Running coprocessor preCompact hooks"); - coprocessorHost.preCompact(false); - } boolean decr = true; try { synchronized (writestate) { @@ -976,10 +972,6 @@ public class HRegion implements HeapSize { // , Writable{ } } } - if (coprocessorHost != null) { - status.setStatus("Running coprocessor post-compact hooks"); - coprocessorHost.postCompact(false); - } status.markComplete("Compaction complete"); return true; } finally { diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 5796413cba9..a6cf6a8068e 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver; +import com.google.common.collect.ImmutableList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -257,15 +258,43 @@ public class RegionCoprocessorHost } /** - * Invoked before a region is compacted. - * @param willSplit true if the compaction is about to trigger a split + * Called prior to selecting the {@link StoreFile}s for compaction from + * the list of currently available candidates. + * @param store The store where compaction is being requested + * @param candidates The currently available store files + * @return If {@code true}, skip the normal selection process and use the current list */ - public void preCompact(boolean willSplit) { + public boolean preCompactSelection(Store store, List candidates) { + ObserverContext ctx = null; + boolean bypass = false; + for (RegionEnvironment env: coprocessors) { + if (env.getInstance() instanceof RegionObserver) { + ctx = ObserverContext.createAndPrepare(env, ctx); + ((RegionObserver)env.getInstance()).preCompactSelection( + ctx, store, candidates); + bypass |= ctx.shouldBypass(); + if (ctx.shouldComplete()) { + break; + } + } + } + return bypass; + } + + /** + * Called after the {@link StoreFile}s to be compacted have been selected + * from the available candidates. + * @param store The store where compaction is being requested + * @param selected The store files selected to compact + */ + public void postCompactSelection(Store store, + ImmutableList selected) { ObserverContext ctx = null; for (RegionEnvironment env: coprocessors) { if (env.getInstance() instanceof RegionObserver) { ctx = ObserverContext.createAndPrepare(env, ctx); - ((RegionObserver)env.getInstance()).preCompact(ctx, willSplit); + ((RegionObserver)env.getInstance()).postCompactSelection( + ctx, store, selected); if (ctx.shouldComplete()) { break; } @@ -274,15 +303,38 @@ public class RegionCoprocessorHost } /** - * Invoked after a region is compacted. - * @param willSplit true if the compaction is about to trigger a split + * Called prior to rewriting the store files selected for compaction + * @param store the store being compacted + * @param scanner the scanner used to read store data during compaction */ - public void postCompact(boolean willSplit) { + public InternalScanner preCompact(Store store, InternalScanner scanner) { + ObserverContext ctx = null; + boolean bypass = false; + for (RegionEnvironment env: coprocessors) { + if (env.getInstance() instanceof RegionObserver) { + ctx = ObserverContext.createAndPrepare(env, ctx); + scanner = ((RegionObserver)env.getInstance()).preCompact( + ctx, store, scanner); + bypass |= ctx.shouldBypass(); + if (ctx.shouldComplete()) { + break; + } + } + } + return bypass ? null : scanner; + } + + /** + * Called after the store compaction has completed. + * @param store the store being compacted + * @param resultFile the new store file written during compaction + */ + public void postCompact(Store store, StoreFile resultFile) { ObserverContext ctx = null; for (RegionEnvironment env: coprocessors) { if (env.getInstance() instanceof RegionObserver) { ctx = ObserverContext.createAndPrepare(env, ctx); - ((RegionObserver)env.getInstance()).postCompact(ctx, willSplit); + ((RegionObserver)env.getInstance()).postCompact(ctx, store, resultFile); if (ctx.shouldComplete()) { break; } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 655db7d7482..f1b13732f80 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -684,6 +684,9 @@ public class Store implements HeapSize { maxId); // Move the compaction into place. sf = completeCompaction(filesToCompact, writer); + if (region.getCoprocessorHost() != null) { + region.getCoprocessorHost().postCompact(this, sf); + } } finally { synchronized (filesCompacting) { filesCompacting.removeAll(filesToCompact); @@ -739,7 +742,10 @@ public class Store implements HeapSize { // Ready to go. Have list of files to compact. StoreFile.Writer writer = compactStore(filesToCompact, isMajor, maxId); // Move the compaction into place. - completeCompaction(filesToCompact, writer); + StoreFile sf = completeCompaction(filesToCompact, writer); + if (region.getCoprocessorHost() != null) { + region.getCoprocessorHost().postCompact(this, sf); + } } finally { synchronized (filesCompacting) { filesCompacting.removeAll(filesToCompact); @@ -900,7 +906,24 @@ public class Store implements HeapSize { Preconditions.checkArgument(idx != -1); candidates.subList(0, idx + 1).clear(); } - List filesToCompact = compactSelection(candidates); + + boolean override = false; + if (region.getCoprocessorHost() != null) { + override = region.getCoprocessorHost().preCompactSelection( + this, candidates); + } + List filesToCompact; + if (override) { + // coprocessor is overriding normal file selection + filesToCompact = candidates; + } else { + filesToCompact = compactSelection(candidates); + } + + if (region.getCoprocessorHost() != null) { + region.getCoprocessorHost().postCompactSelection(this, + ImmutableList.copyOf(filesToCompact)); + } // no files to compact if (filesToCompact.isEmpty()) { @@ -1121,6 +1144,17 @@ public class Store implements HeapSize { scan.setMaxVersions(family.getMaxVersions()); /* include deletes, unless we are doing a major compaction */ scanner = new StoreScanner(this, scan, scanners, !majorCompaction); + if (region.getCoprocessorHost() != null) { + InternalScanner cpScanner = region.getCoprocessorHost().preCompact( + this, scanner); + // NULL scanner returned from coprocessor hooks means skip normal processing + if (cpScanner == null) { + return null; + } + + scanner = cpScanner; + } + int bytesWritten = 0; // since scanner.next() can return 'false' but still be delivering data, // we have to use a do/while loop. diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index c0b726771ea..7a2937a8a68 100644 --- a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Arrays; +import com.google.common.collect.ImmutableList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.KeyValue; @@ -38,6 +39,8 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; /** @@ -57,6 +60,8 @@ public class SimpleRegionObserver extends BaseRegionObserver { boolean hadPostFlush; boolean hadPreSplit; boolean hadPostSplit; + boolean hadPreCompactSelect; + boolean hadPostCompactSelect; boolean hadPreCompact; boolean hadPostCompact; boolean hadPreGet = false; @@ -135,15 +140,34 @@ public class SimpleRegionObserver extends BaseRegionObserver { } @Override - public void preCompact(ObserverContext c, boolean willSplit) { - hadPreCompact = true; + public void preCompactSelection(ObserverContext c, + Store store, List candidates) { + hadPreCompactSelect = true; } @Override - public void postCompact(ObserverContext c, boolean willSplit) { + public void postCompactSelection(ObserverContext c, + Store store, ImmutableList selected) { + hadPostCompactSelect = true; + } + + @Override + public InternalScanner preCompact(ObserverContext e, + Store store, InternalScanner scanner) { + hadPreCompact = true; + return scanner; + } + + @Override + public void postCompact(ObserverContext e, + Store store, StoreFile resultFile) { hadPostCompact = true; } + public boolean wasCompacted() { + return hadPreCompact && hadPostCompact; + } + @Override public RegionScanner preScannerOpen(final ObserverContext c, final Scan scan, @@ -188,10 +212,6 @@ public class SimpleRegionObserver extends BaseRegionObserver { hadPostScannerClose = true; } - public boolean wasCompacted() { - return hadPreCompact && hadPostCompact; - } - @Override public void preGet(final ObserverContext c, final Get get, final List results) throws IOException { diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index c2af6a12f51..eda5a9b882a 100644 --- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -140,8 +140,13 @@ public class TestClassLoading { // only add hbase classes to classpath. This is a little bit tricky: assume // the classpath is {hbaseSrc}/target/classes. String currentDir = new File(".").getAbsolutePath(); - options.add(currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + - "classes"); + String classpath = + currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" + + System.getProperty("path.separator") + + System.getProperty("surefire.test.class.path"); + options.add(classpath); + LOG.debug("Setting classpath to: "+classpath); + JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null, options, null, cu); assertTrue("Compile file " + sourceCodeFile + " failed.", task.call()); diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java index a8edb425794..fbfdd839cef 100644 --- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java @@ -36,10 +36,13 @@ import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.regionserver.SplitTransaction; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.Server; @@ -130,11 +133,14 @@ public class TestCoprocessorInterface extends HBaseTestCase { postCloseCalled = true; } @Override - public void preCompact(ObserverContext e, boolean willSplit) { + public InternalScanner preCompact(ObserverContext e, + Store store, InternalScanner scanner) { preCompactCalled = true; + return scanner; } @Override - public void postCompact(ObserverContext e, boolean willSplit) { + public void postCompact(ObserverContext e, + Store store, StoreFile resultFile) { postCompactCalled = true; } @Override diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java index b370ff0d0b7..1f4e02d61ea 100644 --- a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java @@ -22,18 +22,28 @@ package org.apache.hadoop.hbase.coprocessor; import java.io.IOException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.junit.AfterClass; @@ -241,6 +251,134 @@ public class TestRegionObserverInterface { util.deleteTable(tableName); } + /* Overrides compaction to only output rows with keys that are even numbers */ + public static class EvenOnlyCompactor extends BaseRegionObserver { + long lastCompaction; + long lastFlush; + + @Override + public InternalScanner preCompact(ObserverContext e, + Store store, final InternalScanner scanner) { + return new InternalScanner() { + @Override + public boolean next(List results) throws IOException { + return next(results, -1); + } + + @Override + public boolean next(List results, int limit) throws IOException { + List internalResults = new ArrayList(); + boolean hasMore; + do { + hasMore = scanner.next(internalResults, limit); + if (!internalResults.isEmpty()) { + long row = Bytes.toLong(internalResults.get(0).getRow()); + if (row % 2 == 0) { + // return this row + break; + } + // clear and continue + internalResults.clear(); + } + } while (hasMore); + + if (!internalResults.isEmpty()) { + results.addAll(internalResults); + } + return hasMore; + } + + @Override + public void close() throws IOException { + scanner.close(); + } + }; + } + + @Override + public void postCompact(ObserverContext e, + Store store, StoreFile resultFile) { + lastCompaction = EnvironmentEdgeManager.currentTimeMillis(); + } + + @Override + public void postFlush(ObserverContext e) { + lastFlush = EnvironmentEdgeManager.currentTimeMillis(); + } + } + /** + * Tests overriding compaction handling via coprocessor hooks + * @throws Exception + */ + @Test + public void testCompactionOverride() throws Exception { + byte[] compactTable = Bytes.toBytes("TestCompactionOverride"); + HBaseAdmin admin = util.getHBaseAdmin(); + if (admin.tableExists(compactTable)) { + admin.disableTable(compactTable); + admin.deleteTable(compactTable); + } + + HTableDescriptor htd = new HTableDescriptor(compactTable); + htd.addFamily(new HColumnDescriptor(A)); + htd.addCoprocessor(EvenOnlyCompactor.class.getName()); + admin.createTable(htd); + + HTable table = new HTable(util.getConfiguration(), compactTable); + for (long i=1; i<=10; i++) { + byte[] iBytes = Bytes.toBytes(i); + Put p = new Put(iBytes); + p.add(A, A, iBytes); + table.put(p); + } + + HRegion firstRegion = cluster.getRegions(compactTable).get(0); + Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor( + EvenOnlyCompactor.class.getName()); + assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp); + EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp; + + // force a compaction + long ts = System.currentTimeMillis(); + admin.flush(compactTable); + // wait for flush + for (int i=0; i<10; i++) { + if (compactor.lastFlush >= ts) { + break; + } + Thread.sleep(1000); + } + assertTrue("Flush didn't complete", compactor.lastFlush >= ts); + LOG.debug("Flush complete"); + + ts = compactor.lastFlush; + admin.majorCompact(compactTable); + // wait for compaction + for (int i=0; i<30; i++) { + if (compactor.lastCompaction >= ts) { + break; + } + Thread.sleep(1000); + } + LOG.debug("Last compaction was at "+compactor.lastCompaction); + assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts); + + // only even rows should remain + ResultScanner scanner = table.getScanner(new Scan()); + try { + for (long i=2; i<=10; i+=2) { + Result r = scanner.next(); + assertNotNull(r); + assertFalse(r.isEmpty()); + byte[] iBytes = Bytes.toBytes(i); + assertArrayEquals("Row should be "+i, r.getRow(), iBytes); + assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes); + } + } finally { + scanner.close(); + } + } + // check each region whether the coprocessor upcalls are called or not. private void verifyMethodResult(Class c, String methodName[], byte[] tableName, Object value[]) throws IOException {