From e6c687d1593377a8b66b0a2ba4284ebdc8658978 Mon Sep 17 00:00:00 2001 From: Jim Kellerman Date: Wed, 13 Feb 2008 00:09:52 +0000 Subject: [PATCH] HBASE-43[45] TestTableIndex and TestTableMapReduce failed in Hudson builds git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@627195 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../apache/hadoop/hbase/MultiRegionTable.java | 34 ++++++++++++------- .../hadoop/hbase/mapred/TestTableIndex.java | 12 ++++--- .../hbase/mapred/TestTableMapReduce.java | 19 ++++------- 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 26d02975eef..741855630e3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,7 @@ Hbase Change Log HBASE-421 TestRegionServerExit broken HBASE-426 hbase can't find remote filesystem HBASE-437 Clear Command should use system.out (Edward Yoon via Stack) + HBASE-43[45] TestTableIndex and TestTableMapReduce failed in Hudson builds IMPROVEMENTS HBASE-415 Rewrite leases to use DelayedBlockingQueue instead of polling diff --git a/src/test/org/apache/hadoop/hbase/MultiRegionTable.java b/src/test/org/apache/hadoop/hbase/MultiRegionTable.java index c5c85a958c6..6206b9bb0dd 100644 --- a/src/test/org/apache/hadoop/hbase/MultiRegionTable.java +++ b/src/test/org/apache/hadoop/hbase/MultiRegionTable.java @@ -88,11 +88,17 @@ public class MultiRegionTable extends HBaseTestCase { // with EMPTY_START_ROW will be one of the unsplittable daughters. HRegionInfo hri = null; HRegion r = null; + HRegionServer server = cluster.getRegionThreads().get(0).getRegionServer(); for (int i = 0; i < 30; i++) { - hri = t.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); + try { + hri = t.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); + } catch (IOException e) { + e = RemoteExceptionHandler.checkIOException(e); + e.printStackTrace(); + continue; + } LOG.info("Region location: " + hri); - r = cluster.getRegionThreads().get(0).getRegionServer(). - onlineRegions.get(hri.getRegionName()); + r = server.onlineRegions.get(hri.getRegionName()); if (r != null) { break; } @@ -102,10 +108,10 @@ public class MultiRegionTable extends HBaseTestCase { LOG.warn("Waiting on region to come online", e); } } + assertNotNull(r); // Flush the cache - cluster.getRegionThreads().get(0).getRegionServer().getCacheFlushListener(). - flushRequested(r); + server.getCacheFlushListener().flushRequested(r); // Now, wait until split makes it into the meta table. int oldCount = count; @@ -158,7 +164,8 @@ public class MultiRegionTable extends HBaseTestCase { // still has references. while (true) { data = getSplitParentInfo(meta, parent); - if (data == null || data.size() == 3) { + if (data != null && data.size() == 3) { + LOG.info("Waiting for splitA to release reference to parent"); try { Thread.sleep(waitTime); } catch (InterruptedException e) { @@ -168,7 +175,9 @@ public class MultiRegionTable extends HBaseTestCase { } break; } - LOG.info("Parent split info returned " + data.keySet().toString()); + if (data != null) { + LOG.info("Parent split info returned " + data.keySet().toString()); + } } if (splitB == null) { @@ -199,8 +208,10 @@ public class MultiRegionTable extends HBaseTestCase { for (int i = 0; i < retries; i++) { if (!fs.exists(parentDir)) { + LOG.info("Parent directory was deleted. tries=" + i); break; } + LOG.info("Waiting for parent directory to be deleted. tries=" + i); try { Thread.sleep(waitTime); } catch (InterruptedException e) { @@ -260,8 +271,7 @@ public class MultiRegionTable extends HBaseTestCase { continue; } // Make sure I get the parent. - if (hri.getRegionName().toString(). - equals(parent.getRegionName().toString()) && + if (hri.getRegionName().equals(parent.getRegionName()) && hri.getRegionId() == parent.getRegionId()) { return curVals; } @@ -316,8 +326,7 @@ public class MultiRegionTable extends HBaseTestCase { * @throws IOException */ protected static void compact(final MiniHBaseCluster cluster, - final HRegionInfo r) - throws IOException { + final HRegionInfo r) throws IOException { if (r == null) { LOG.debug("Passed region is null"); return; @@ -332,8 +341,7 @@ public class MultiRegionTable extends HBaseTestCase { for (int i = 0; i < 10; i++) { try { for (HRegion online: regions.values()) { - if (online.getRegionName().toString(). - equals(r.getRegionName().toString())) { + if (online.getRegionName().equals(r.getRegionName())) { online.compactStores(); } } diff --git a/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java b/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java index 716c6012dc8..e9fe29698c5 100644 --- a/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java +++ b/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java @@ -31,6 +31,7 @@ import junit.textui.TestRunner; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.dfs.MiniDFSCluster; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseAdmin; @@ -96,6 +97,9 @@ public class TestTableIndex extends MultiRegionTable { // below. After adding all data, the first region is 1.3M conf.setLong("hbase.hregion.max.filesize", 1024 * 1024); + // Always compact if there is more than one store file. + conf.setInt("hbase.hstore.compactionThreshold", 2); + desc = new HTableDescriptor(TABLE_NAME); desc.addFamily(new HColumnDescriptor(INPUT_COLUMN)); desc.addFamily(new HColumnDescriptor(OUTPUT_COLUMN)); @@ -160,7 +164,6 @@ public class TestTableIndex extends MultiRegionTable { } scanTable(printResults); - @SuppressWarnings("deprecation") MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1); // set configuration parameter for index build @@ -267,17 +270,17 @@ public class TestTableIndex extends MultiRegionTable { Integer.toString(new Random().nextInt())); this.fs.copyToLocalFile(new Path(INDEX_DIR), localDir); FileSystem localfs = FileSystem.getLocal(conf); - Path [] indexDirs = localfs.listPaths(new Path [] {localDir}); + FileStatus [] indexDirs = localfs.listStatus(localDir); Searcher searcher = null; HScannerInterface scanner = null; try { if (indexDirs.length == 1) { - searcher = new IndexSearcher((new File(indexDirs[0]. + searcher = new IndexSearcher((new File(indexDirs[0].getPath(). toUri())).getAbsolutePath()); } else if (indexDirs.length > 1) { Searchable[] searchers = new Searchable[indexDirs.length]; for (int i = 0; i < indexDirs.length; i++) { - searchers[i] = new IndexSearcher((new File(indexDirs[i]. + searchers[i] = new IndexSearcher((new File(indexDirs[i].getPath(). toUri()).getAbsolutePath())); } searcher = new MultiSearcher(searchers); @@ -301,7 +304,6 @@ public class TestTableIndex extends MultiRegionTable { int count = 0; while (scanner.next(key, results)) { String value = key.getRow().toString(); - LOG.debug("Scanned over " + key.getRow()); Term term = new Term(rowkeyName, value); int hitCount = searcher.search(new TermQuery(term)).length(); assertEquals("check row " + value, 1, hitCount); diff --git a/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java b/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java index 161d3f10509..9bfd3f29ca4 100644 --- a/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java +++ b/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.MultiRegionTable; import org.apache.hadoop.hbase.StaticTestEnvironment; +import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; @@ -99,7 +100,7 @@ public class TestTableMapReduce extends MultiRegionTable { // This size should make it so we always split using the addContent // below. After adding all data, the first region is 1.3M - conf.setLong("hbase.hregion.max.filesize", 256 * 1024); + conf.setLong("hbase.hregion.max.filesize", 1024 * 1024); // Make lease timeout longer, lease checks less frequent conf.setInt("hbase.master.lease.period", 10 * 1000); @@ -156,7 +157,7 @@ public class TestTableMapReduce extends MultiRegionTable { /** * Pass the key, and reversed value to reduce * - * @see org.apache.hadoop.hbase.mapred.TableMap#map(org.apache.hadoop.hbase.HStoreKey, org.apache.hadoop.io.MapWritable, org.apache.hadoop.hbase.mapred.TableOutputCollector, org.apache.hadoop.mapred.Reporter) + * @see org.apache.hadoop.hbase.mapred.TableMap#map(org.apache.hadoop.hbase.HStoreKey, org.apache.hadoop.io.MapWritable, org.apache.hadoop.mapred.OutputCollector, org.apache.hadoop.mapred.Reporter) */ @SuppressWarnings("unchecked") @Override @@ -222,17 +223,11 @@ public class TestTableMapReduce extends MultiRegionTable { HTable table = new HTable(conf, new Text(SINGLE_REGION_TABLE_NAME)); for(int i = 0; i < values.length; i++) { - long lockid = table.startUpdate(new Text("row_" - + String.format("%1$05d", i))); + BatchUpdate b = new BatchUpdate(new Text("row_" + + String.format("%1$05d", i))); - try { - table.put(lockid, TEXT_INPUT_COLUMN, values[i]); - table.commit(lockid, System.currentTimeMillis()); - lockid = -1; - } finally { - if (lockid != -1) - table.abort(lockid); - } + b.put(TEXT_INPUT_COLUMN, values[i]); + table.commit(b); } LOG.info("Print table contents before map/reduce for " +