From 438b82450a5f0714a3c581852274c94f6b9f8704 Mon Sep 17 00:00:00 2001 From: Jim Kellerman Date: Thu, 13 Dec 2007 21:28:03 +0000 Subject: [PATCH] HADOOP-2418 Fix assertion failures in TestTableMapReduce, TestTableIndex, and TestTableJoinMapReduce git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@604034 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + .../hadoop/hbase/mapred/TestTableIndex.java | 33 ++++++++++------ .../hbase/mapred/TestTableMapReduce.java | 39 ++++++++++--------- .../shell/algebra/TestTableJoinMapReduce.java | 32 ++++++++++----- 4 files changed, 68 insertions(+), 38 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3f18c2aadb2..2eda2387bc6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -78,6 +78,8 @@ Trunk (unreleased changes) HADOOP-2397 The only time that a meta scanner should try to recover a log is when the master is starting HADOOP-2417 Fix critical shutdown problem introduced by HADOOP-2338 + HADOOP-2418 Fix assertion failures in TestTableMapReduce, TestTableIndex, + and TestTableJoinMapReduce IMPROVEMENTS HADOOP-2401 Add convenience put method that takes writable diff --git a/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java b/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java index 483602ac2f7..d6b67202c5e 100644 --- a/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java +++ b/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java @@ -34,7 +34,6 @@ import org.apache.hadoop.dfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseAdmin; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HScannerInterface; @@ -68,6 +67,11 @@ public class TestTableIndex extends MultiRegionTable { static final Text TEXT_OUTPUT_COLUMN = new Text(OUTPUT_COLUMN); static final String ROWKEY_NAME = "key"; static final String INDEX_DIR = "testindex"; + private static final Text[] columns = { + TEXT_INPUT_COLUMN, + TEXT_OUTPUT_COLUMN + }; + private HTableDescriptor desc; @@ -146,7 +150,7 @@ public class TestTableIndex extends MultiRegionTable { if (printResults) { LOG.info("Print table contents before map/reduce"); } - scanTable(conf, printResults); + scanTable(printResults); @SuppressWarnings("deprecation") MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1); @@ -179,10 +183,10 @@ public class TestTableIndex extends MultiRegionTable { if (printResults) { LOG.info("Print table contents after map/reduce"); } - scanTable(conf, printResults); + scanTable(printResults); // verify index results - verify(conf); + verify(); } private String createIndexConfContent() { @@ -218,10 +222,9 @@ public class TestTableIndex extends MultiRegionTable { return c.toString(); } - private void scanTable(HBaseConfiguration c, boolean printResults) + private void scanTable(boolean printResults) throws IOException { - HTable table = new HTable(c, new Text(TABLE_NAME)); - Text[] columns = { TEXT_INPUT_COLUMN, TEXT_OUTPUT_COLUMN }; + HTable table = new HTable(conf, new Text(TABLE_NAME)); HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW); try { @@ -243,7 +246,16 @@ public class TestTableIndex extends MultiRegionTable { } } - private void verify(HBaseConfiguration c) throws IOException { + private void verify() throws IOException { + // Sleep before we start the verify to ensure that when the scanner takes + // its snapshot, all the updates have made it into the cache. + try { + Thread.sleep(conf.getLong("hbase.regionserver.optionalcacheflushinterval", + 60L * 1000L)); + } catch (InterruptedException e) { + // ignore + } + Path localDir = new Path(this.testDir, "index_" + Integer.toString(new Random().nextInt())); this.fs.copyToLocalFile(new Path(INDEX_DIR), localDir); @@ -265,15 +277,14 @@ public class TestTableIndex extends MultiRegionTable { throw new IOException("no index directory found"); } - HTable table = new HTable(c, new Text(TABLE_NAME)); - Text[] columns = { TEXT_INPUT_COLUMN, TEXT_OUTPUT_COLUMN }; + HTable table = new HTable(conf, new Text(TABLE_NAME)); scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW); HStoreKey key = new HStoreKey(); TreeMap results = new TreeMap(); IndexConfiguration indexConf = new IndexConfiguration(); - String content = c.get("hbase.index.conf"); + String content = conf.get("hbase.index.conf"); if (content != null) { indexConf.addFromXML(content); } diff --git a/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java b/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java index 7c5ae2d0d36..9912f915b3e 100644 --- a/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java +++ b/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java @@ -30,7 +30,6 @@ import org.apache.hadoop.dfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseAdmin; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HScannerInterface; @@ -55,7 +54,7 @@ public class TestTableMapReduce extends MultiRegionTable { @SuppressWarnings("hiding") private static final Log LOG = LogFactory.getLog(TestTableMapReduce.class.getName()); - + static final String SINGLE_REGION_TABLE_NAME = "srtest"; static final String MULTI_REGION_TABLE_NAME = "mrtest"; static final String INPUT_COLUMN = "contents:"; @@ -63,6 +62,11 @@ public class TestTableMapReduce extends MultiRegionTable { static final String OUTPUT_COLUMN = "text:"; static final Text TEXT_OUTPUT_COLUMN = new Text(OUTPUT_COLUMN); + private static final Text[] columns = { + TEXT_INPUT_COLUMN, + TEXT_OUTPUT_COLUMN + }; + private MiniDFSCluster dfsCluster = null; private FileSystem fs; private Path dir; @@ -232,7 +236,7 @@ public class TestTableMapReduce extends MultiRegionTable { } LOG.info("Print table contents before map/reduce"); - scanTable(conf, SINGLE_REGION_TABLE_NAME, true); + scanTable(SINGLE_REGION_TABLE_NAME, true); @SuppressWarnings("deprecation") MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1); @@ -256,10 +260,10 @@ public class TestTableMapReduce extends MultiRegionTable { } LOG.info("Print table contents after map/reduce"); - scanTable(conf, SINGLE_REGION_TABLE_NAME, true); + scanTable(SINGLE_REGION_TABLE_NAME, true); // verify map-reduce results - verify(conf, SINGLE_REGION_TABLE_NAME); + verify(SINGLE_REGION_TABLE_NAME); } finally { table.close(); @@ -311,21 +315,17 @@ public class TestTableMapReduce extends MultiRegionTable { } // verify map-reduce results - verify(conf, MULTI_REGION_TABLE_NAME); + verify(MULTI_REGION_TABLE_NAME); } finally { table.close(); } } - private void scanTable(HBaseConfiguration conf, String tableName, - boolean printValues) throws IOException { + private void scanTable(String tableName, boolean printValues) + throws IOException { HTable table = new HTable(conf, new Text(tableName)); - Text[] columns = { - TEXT_INPUT_COLUMN, - TEXT_OUTPUT_COLUMN - }; HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW); @@ -350,14 +350,17 @@ public class TestTableMapReduce extends MultiRegionTable { } @SuppressWarnings("null") - private void verify(HBaseConfiguration conf, String tableName) - throws IOException { + private void verify(String tableName) throws IOException { + // Sleep before we start the verify to ensure that when the scanner takes + // its snapshot, all the updates have made it into the cache. + try { + Thread.sleep(conf.getLong("hbase.regionserver.optionalcacheflushinterval", + 60L * 1000L)); + } catch (InterruptedException e) { + // ignore + } HTable table = new HTable(conf, new Text(tableName)); - Text[] columns = { - TEXT_INPUT_COLUMN, - TEXT_OUTPUT_COLUMN - }; HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW); diff --git a/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java b/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java index 518054a56b8..52b596a7f31 100644 --- a/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java +++ b/src/test/org/apache/hadoop/hbase/shell/algebra/TestTableJoinMapReduce.java @@ -28,7 +28,6 @@ import org.apache.hadoop.dfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseAdmin; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HScannerInterface; @@ -94,7 +93,10 @@ public class TestTableJoinMapReduce extends MultiRegionTable { StaticTestEnvironment.shutdownDfs(dfsCluster); } - public void testTableJoinMapReduce() { + /** + * @throws Exception + */ + public void testTableJoinMapReduce() throws Exception { HTable table = null; try { HTableDescriptor desc = new HTableDescriptor(FIRST_RELATION); @@ -188,16 +190,20 @@ public class TestTableJoinMapReduce extends MultiRegionTable { JobClient.runJob(jobConf); - } catch (IOException e) { + } catch (Exception e) { e.printStackTrace(); + throw e; } finally { - mrCluster.shutdown(); + if (mrCluster != null) { + mrCluster.shutdown(); + } } try { - verify(conf, OUTPUT_TABLE); - } catch (IOException e) { + verify(OUTPUT_TABLE); + } catch (Exception e) { e.printStackTrace(); + throw e; } } @@ -208,8 +214,16 @@ public class TestTableJoinMapReduce extends MultiRegionTable { * @param outputTable * @throws IOException */ - private void verify(HBaseConfiguration conf, String outputTable) - throws IOException { + private void verify(String outputTable) throws IOException { + // Sleep before we start the verify to ensure that when the scanner takes + // its snapshot, all the updates have made it into the cache. + try { + Thread.sleep(conf.getLong("hbase.regionserver.optionalcacheflushinterval", + 60L * 1000L)); + } catch (InterruptedException e) { + // ignore + } + HTable table = new HTable(conf, new Text(outputTable)); Text[] columns = { new Text("a:"), new Text("b:"), new Text("c:"), new Text("d:"), new Text("e:") }; @@ -222,8 +236,8 @@ public class TestTableJoinMapReduce extends MultiRegionTable { int i = 0; while (scanner.next(key, results)) { - assertTrue(results.keySet().size() == 5); LOG.info("result_table.column.size: " + results.keySet().size()); + assertEquals(5, results.keySet().size()); i++; } assertTrue(i == 3);