Revert "more"
Remove overcommit..... made mistakenly.
This reverts commit 1b3557649c
.
This commit is contained in:
parent
3f7994b5ab
commit
34fe1f5fd7
|
@ -19,29 +19,35 @@
|
||||||
package org.apache.hadoop.hbase;
|
package org.apache.hadoop.hbase;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
import java.util.NavigableMap;
|
import java.util.NavigableMap;
|
||||||
|
|
||||||
|
import junit.framework.AssertionFailedError;
|
||||||
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Durability;
|
import org.apache.hadoop.hbase.client.Durability;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
|
import org.apache.hadoop.hbase.client.HTable;
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
|
import org.apache.hadoop.hbase.client.ResultScanner;
|
||||||
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
import org.apache.hadoop.hbase.client.Table;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
|
import org.apache.hadoop.hbase.regionserver.InternalScanner;
|
||||||
import org.apache.hadoop.hbase.regionserver.Region;
|
import org.apache.hadoop.hbase.regionserver.Region;
|
||||||
import org.apache.hadoop.hbase.regionserver.RegionTable;
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.FSTableDescriptors;
|
import org.apache.hadoop.hbase.util.FSTableDescriptors;
|
||||||
import org.apache.hadoop.hbase.util.FSUtils;
|
import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
|
|
||||||
import junit.framework.AssertionFailedError;
|
|
||||||
import junit.framework.TestCase;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abstract HBase test class. Initializes a few things that can come in handly
|
* Abstract HBase test class. Initializes a few things that can come in handly
|
||||||
* like an HBaseConfiguration and filesystem.
|
* like an HBaseConfiguration and filesystem.
|
||||||
|
@ -233,7 +239,7 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
if (startKeyBytes == null || startKeyBytes.length == 0) {
|
if (startKeyBytes == null || startKeyBytes.length == 0) {
|
||||||
startKeyBytes = START_KEY_BYTES;
|
startKeyBytes = START_KEY_BYTES;
|
||||||
}
|
}
|
||||||
return addContent(new RegionTable(r), Bytes.toString(columnFamily), Bytes.toString(column),
|
return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
|
||||||
startKeyBytes, endKey, -1);
|
startKeyBytes, endKey, -1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -245,15 +251,18 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
* Add content to region <code>r</code> on the passed column
|
* Add content to region <code>r</code> on the passed column
|
||||||
* <code>column</code>.
|
* <code>column</code>.
|
||||||
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
|
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
|
||||||
|
* @param updater An instance of {@link Incommon}.
|
||||||
|
* @param columnFamily
|
||||||
|
* @param writeToWAL
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @return count of what we added.
|
* @return count of what we added.
|
||||||
*/
|
*/
|
||||||
public static long addContent(final Table updater,
|
public static long addContent(final Incommon updater,
|
||||||
final String columnFamily) throws IOException {
|
final String columnFamily) throws IOException {
|
||||||
return addContent(updater, columnFamily, START_KEY_BYTES, null);
|
return addContent(updater, columnFamily, START_KEY_BYTES, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static long addContent(final Table updater, final String family,
|
public static long addContent(final Incommon updater, final String family,
|
||||||
final String column) throws IOException {
|
final String column) throws IOException {
|
||||||
return addContent(updater, family, column, START_KEY_BYTES, null);
|
return addContent(updater, family, column, START_KEY_BYTES, null);
|
||||||
}
|
}
|
||||||
|
@ -262,16 +271,21 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
* Add content to region <code>r</code> on the passed column
|
* Add content to region <code>r</code> on the passed column
|
||||||
* <code>column</code>.
|
* <code>column</code>.
|
||||||
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
|
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
|
||||||
|
* @param updater An instance of {@link Incommon}.
|
||||||
|
* @param columnFamily
|
||||||
|
* @param startKeyBytes Where to start the rows inserted
|
||||||
|
* @param endKey Where to stop inserting rows.
|
||||||
|
* @param writeToWAL
|
||||||
* @return count of what we added.
|
* @return count of what we added.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static long addContent(final Table updater, final String columnFamily,
|
public static long addContent(final Incommon updater, final String columnFamily,
|
||||||
final byte [] startKeyBytes, final byte [] endKey)
|
final byte [] startKeyBytes, final byte [] endKey)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
|
return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static long addContent(final Table updater, final String family, String column,
|
public static long addContent(final Incommon updater, final String family, String column,
|
||||||
final byte [] startKeyBytes, final byte [] endKey) throws IOException {
|
final byte [] startKeyBytes, final byte [] endKey) throws IOException {
|
||||||
return addContent(updater, family, column, startKeyBytes, endKey, -1);
|
return addContent(updater, family, column, startKeyBytes, endKey, -1);
|
||||||
}
|
}
|
||||||
|
@ -280,10 +294,16 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
* Add content to region <code>r</code> on the passed column
|
* Add content to region <code>r</code> on the passed column
|
||||||
* <code>column</code>.
|
* <code>column</code>.
|
||||||
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
|
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
|
||||||
|
* @param updater An instance of {@link Incommon}.
|
||||||
|
* @param column
|
||||||
|
* @param startKeyBytes Where to start the rows inserted
|
||||||
|
* @param endKey Where to stop inserting rows.
|
||||||
|
* @param ts Timestamp to write the content with.
|
||||||
|
* @param writeToWAL
|
||||||
* @return count of what we added.
|
* @return count of what we added.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static long addContent(final Table updater,
|
public static long addContent(final Incommon updater,
|
||||||
final String columnFamily,
|
final String columnFamily,
|
||||||
final String column,
|
final String column,
|
||||||
final byte [] startKeyBytes, final byte [] endKey, final long ts)
|
final byte [] startKeyBytes, final byte [] endKey, final long ts)
|
||||||
|
@ -358,6 +378,209 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implementors can flushcache.
|
||||||
|
*/
|
||||||
|
public interface FlushCache {
|
||||||
|
/**
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
void flushcache() throws IOException;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interface used by tests so can do common operations against an HTable
|
||||||
|
* or an HRegion.
|
||||||
|
*
|
||||||
|
* TOOD: Come up w/ a better name for this interface.
|
||||||
|
*/
|
||||||
|
public interface Incommon {
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param delete
|
||||||
|
* @param writeToWAL
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
void delete(Delete delete, boolean writeToWAL)
|
||||||
|
throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param put
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
void put(Put put) throws IOException;
|
||||||
|
|
||||||
|
Result get(Get get) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param family
|
||||||
|
* @param qualifiers
|
||||||
|
* @param firstRow
|
||||||
|
* @param ts
|
||||||
|
* @return scanner for specified columns, first row and timestamp
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
ScannerIncommon getScanner(
|
||||||
|
byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
|
||||||
|
)
|
||||||
|
throws IOException;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A class that makes a {@link Incommon} out of a {@link HRegion}
|
||||||
|
*/
|
||||||
|
public static class HRegionIncommon implements Incommon, FlushCache {
|
||||||
|
final HRegion region;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param HRegion
|
||||||
|
*/
|
||||||
|
public HRegionIncommon(final HRegion HRegion) {
|
||||||
|
this.region = HRegion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public HRegionIncommon(final Region region) {
|
||||||
|
this.region = (HRegion)region;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void put(Put put) throws IOException {
|
||||||
|
region.put(put);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void delete(Delete delete, boolean writeToWAL)
|
||||||
|
throws IOException {
|
||||||
|
this.region.delete(delete);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Result get(Get get) throws IOException {
|
||||||
|
return region.get(get);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
|
||||||
|
byte [] firstRow, long ts)
|
||||||
|
throws IOException {
|
||||||
|
Scan scan = new Scan(firstRow);
|
||||||
|
if(qualifiers == null || qualifiers.length == 0) {
|
||||||
|
scan.addFamily(family);
|
||||||
|
} else {
|
||||||
|
for(int i=0; i<qualifiers.length; i++){
|
||||||
|
scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
scan.setTimeRange(0, ts);
|
||||||
|
return new
|
||||||
|
InternalScannerIncommon(region.getScanner(scan));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void flushcache() throws IOException {
|
||||||
|
this.region.flush(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A class that makes a {@link Incommon} out of a {@link HTable}
|
||||||
|
*/
|
||||||
|
public static class HTableIncommon implements Incommon {
|
||||||
|
final Table table;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param table
|
||||||
|
*/
|
||||||
|
public HTableIncommon(final Table table) {
|
||||||
|
super();
|
||||||
|
this.table = table;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void put(Put put) throws IOException {
|
||||||
|
table.put(put);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void delete(Delete delete, boolean writeToWAL)
|
||||||
|
throws IOException {
|
||||||
|
this.table.delete(delete);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Result get(Get get) throws IOException {
|
||||||
|
return table.get(get);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
|
||||||
|
byte [] firstRow, long ts)
|
||||||
|
throws IOException {
|
||||||
|
Scan scan = new Scan(firstRow);
|
||||||
|
if(qualifiers == null || qualifiers.length == 0) {
|
||||||
|
scan.addFamily(family);
|
||||||
|
} else {
|
||||||
|
for(int i=0; i<qualifiers.length; i++){
|
||||||
|
scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
scan.setTimeRange(0, ts);
|
||||||
|
return new
|
||||||
|
ClientScannerIncommon(table.getScanner(scan));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public interface ScannerIncommon
|
||||||
|
extends Iterable<Result> {
|
||||||
|
boolean next(List<Cell> values)
|
||||||
|
throws IOException;
|
||||||
|
|
||||||
|
void close() throws IOException;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class ClientScannerIncommon implements ScannerIncommon {
|
||||||
|
ResultScanner scanner;
|
||||||
|
public ClientScannerIncommon(ResultScanner scanner) {
|
||||||
|
this.scanner = scanner;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean next(List<Cell> values)
|
||||||
|
throws IOException {
|
||||||
|
Result results = scanner.next();
|
||||||
|
if (results == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
values.clear();
|
||||||
|
values.addAll(results.listCells());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close() throws IOException {
|
||||||
|
scanner.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Iterator<Result> iterator() {
|
||||||
|
return scanner.iterator();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class InternalScannerIncommon implements ScannerIncommon {
|
||||||
|
InternalScanner scanner;
|
||||||
|
|
||||||
|
public InternalScannerIncommon(InternalScanner scanner) {
|
||||||
|
this.scanner = scanner;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean next(List<Cell> results)
|
||||||
|
throws IOException {
|
||||||
|
return scanner.next(results);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
scanner.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Iterator<Result> iterator() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
protected void assertResultEquals(final HRegion region, final byte [] row,
|
protected void assertResultEquals(final HRegion region, final byte [] row,
|
||||||
final byte [] family, final byte [] qualifier, final long timestamp,
|
final byte [] family, final byte [] qualifier, final long timestamp,
|
||||||
final byte [] value)
|
final byte [] value)
|
||||||
|
@ -446,4 +669,5 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
Bytes.toStringBinary(actual) + ">");
|
Bytes.toStringBinary(actual) + ">");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -29,6 +29,9 @@ import java.util.NavigableMap;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.FlushCache;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.HTableIncommon;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.Incommon;
|
||||||
import org.apache.hadoop.hbase.client.Admin;
|
import org.apache.hadoop.hbase.client.Admin;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||||
|
@ -99,7 +102,8 @@ public class TestMultiVersions {
|
||||||
Table table = new HTable(UTIL.getConfiguration(), desc.getTableName());
|
Table table = new HTable(UTIL.getConfiguration(), desc.getTableName());
|
||||||
// TODO: Remove these deprecated classes or pull them in here if this is
|
// TODO: Remove these deprecated classes or pull them in here if this is
|
||||||
// only test using them.
|
// only test using them.
|
||||||
TimestampTestBase.doTestDelete(table, new FlushCache() {
|
Incommon incommon = new HTableIncommon(table);
|
||||||
|
TimestampTestBase.doTestDelete(incommon, new FlushCache() {
|
||||||
public void flushcache() throws IOException {
|
public void flushcache() throws IOException {
|
||||||
UTIL.getHBaseCluster().flushcache();
|
UTIL.getHBaseCluster().flushcache();
|
||||||
}
|
}
|
||||||
|
@ -107,7 +111,7 @@ public class TestMultiVersions {
|
||||||
|
|
||||||
// Perhaps drop and readd the table between tests so the former does
|
// Perhaps drop and readd the table between tests so the former does
|
||||||
// not pollute this latter? Or put into separate tests.
|
// not pollute this latter? Or put into separate tests.
|
||||||
TimestampTestBase.doTestTimestampScanning(table, new FlushCache() {
|
TimestampTestBase.doTestTimestampScanning(incommon, new FlushCache() {
|
||||||
public void flushcache() throws IOException {
|
public void flushcache() throws IOException {
|
||||||
UTIL.getMiniHBaseCluster().flushcache();
|
UTIL.getMiniHBaseCluster().flushcache();
|
||||||
}
|
}
|
||||||
|
@ -137,7 +141,7 @@ public class TestMultiVersions {
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
this.admin.createTable(desc);
|
this.admin.createTable(desc);
|
||||||
Put put = new Put(row, timestamp1);
|
Put put = new Put(row, timestamp1);
|
||||||
put.addColumn(contents, contents, value1);
|
put.add(contents, contents, value1);
|
||||||
Table table = new HTable(UTIL.getConfiguration(), desc.getTableName());
|
Table table = new HTable(UTIL.getConfiguration(), desc.getTableName());
|
||||||
table.put(put);
|
table.put(put);
|
||||||
// Shut down and restart the HBase cluster
|
// Shut down and restart the HBase cluster
|
||||||
|
@ -150,7 +154,7 @@ public class TestMultiVersions {
|
||||||
table = new HTable(new Configuration(UTIL.getConfiguration()), desc.getTableName());
|
table = new HTable(new Configuration(UTIL.getConfiguration()), desc.getTableName());
|
||||||
// Overwrite previous value
|
// Overwrite previous value
|
||||||
put = new Put(row, timestamp2);
|
put = new Put(row, timestamp2);
|
||||||
put.addColumn(contents, contents, value2);
|
put.add(contents, contents, value2);
|
||||||
table.put(put);
|
table.put(put);
|
||||||
// Now verify that getRow(row, column, latest) works
|
// Now verify that getRow(row, column, latest) works
|
||||||
Get get = new Get(row);
|
Get get = new Get(row);
|
||||||
|
@ -220,7 +224,7 @@ public class TestMultiVersions {
|
||||||
for (int i = 0; i < locations.size(); i++) {
|
for (int i = 0; i < locations.size(); i++) {
|
||||||
for (int j = 0; j < timestamp.length; j++) {
|
for (int j = 0; j < timestamp.length; j++) {
|
||||||
Put put = new Put(rows[i], timestamp[j]);
|
Put put = new Put(rows[i], timestamp[j]);
|
||||||
put.addColumn(HConstants.CATALOG_FAMILY, null, timestamp[j],
|
put.add(HConstants.CATALOG_FAMILY, null, timestamp[j],
|
||||||
Bytes.toBytes(timestamp[j]));
|
Bytes.toBytes(timestamp[j]));
|
||||||
table.put(put);
|
table.put(put);
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,9 +24,6 @@ import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.client.Durability;
|
import org.apache.hadoop.hbase.client.Durability;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
|
||||||
|
@ -52,7 +49,7 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
* @param flusher
|
* @param flusher
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void doTestDelete(final Table incommon, FlushCache flusher)
|
public static void doTestDelete(final Incommon incommon, FlushCache flusher)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// Add values at various timestamps (Values are timestampes as bytes).
|
// Add values at various timestamps (Values are timestampes as bytes).
|
||||||
put(incommon, T0);
|
put(incommon, T0);
|
||||||
|
@ -90,9 +87,8 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
put(incommon, T1);
|
put(incommon, T1);
|
||||||
|
|
||||||
Delete delete = new Delete(ROW);
|
Delete delete = new Delete(ROW);
|
||||||
delete.setWriteToWAL(true);
|
delete.deleteColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
|
||||||
delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, T2);
|
incommon.delete(delete, true);
|
||||||
incommon.delete(delete);
|
|
||||||
|
|
||||||
// Should only be current value in set. Assert this is so
|
// Should only be current value in set. Assert this is so
|
||||||
assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
|
assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
|
||||||
|
@ -102,7 +98,7 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
|
assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void assertOnlyLatest(final Table incommon,
|
private static void assertOnlyLatest(final Incommon incommon,
|
||||||
final long currentTime)
|
final long currentTime)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Get get = null;
|
Get get = null;
|
||||||
|
@ -123,7 +119,7 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
* @param tss
|
* @param tss
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void assertVersions(final Table incommon, final long [] tss)
|
public static void assertVersions(final Incommon incommon, final long [] tss)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// Assert that 'latest' is what we expect.
|
// Assert that 'latest' is what we expect.
|
||||||
Get get = null;
|
Get get = null;
|
||||||
|
@ -173,7 +169,7 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
* @param flusher
|
* @param flusher
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void doTestTimestampScanning(final Table incommon,
|
public static void doTestTimestampScanning(final Incommon incommon,
|
||||||
final FlushCache flusher)
|
final FlushCache flusher)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// Add a couple of values for three different timestamps.
|
// Add a couple of values for three different timestamps.
|
||||||
|
@ -199,12 +195,10 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
* @return Count of items scanned.
|
* @return Count of items scanned.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static int assertScanContentTimestamp(final Table in, final long ts)
|
public static int assertScanContentTimestamp(final Incommon in, final long ts)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Scan scan = new Scan(HConstants.EMPTY_START_ROW);
|
ScannerIncommon scanner =
|
||||||
scan.addFamily(COLUMNS[0]);
|
in.getScanner(COLUMNS[0], null, HConstants.EMPTY_START_ROW, ts);
|
||||||
scan.setTimeRange(0, ts);
|
|
||||||
ResultScanner scanner = in.getScanner(scan);
|
|
||||||
int count = 0;
|
int count = 0;
|
||||||
try {
|
try {
|
||||||
// TODO FIX
|
// TODO FIX
|
||||||
|
@ -227,12 +221,12 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void put(final Table loader, final long ts)
|
public static void put(final Incommon loader, final long ts)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
put(loader, Bytes.toBytes(ts), ts);
|
put(loader, Bytes.toBytes(ts), ts);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void put(final Table loader)
|
public static void put(final Incommon loader)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
long ts = HConstants.LATEST_TIMESTAMP;
|
long ts = HConstants.LATEST_TIMESTAMP;
|
||||||
put(loader, Bytes.toBytes(ts), ts);
|
put(loader, Bytes.toBytes(ts), ts);
|
||||||
|
@ -245,40 +239,39 @@ public class TimestampTestBase extends HBaseTestCase {
|
||||||
* @param ts
|
* @param ts
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void put(final Table loader, final byte [] bytes,
|
public static void put(final Incommon loader, final byte [] bytes,
|
||||||
final long ts)
|
final long ts)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Put put = new Put(ROW, ts);
|
Put put = new Put(ROW, ts);
|
||||||
put.setDurability(Durability.SKIP_WAL);
|
put.setDurability(Durability.SKIP_WAL);
|
||||||
put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
|
put.add(FAMILY_NAME, QUALIFIER_NAME, bytes);
|
||||||
loader.put(put);
|
loader.put(put);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void delete(final Table loader) throws IOException {
|
public static void delete(final Incommon loader) throws IOException {
|
||||||
delete(loader, null);
|
delete(loader, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void delete(final Table loader, final byte [] column)
|
public static void delete(final Incommon loader, final byte [] column)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
delete(loader, column, HConstants.LATEST_TIMESTAMP);
|
delete(loader, column, HConstants.LATEST_TIMESTAMP);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void delete(final Table loader, final long ts)
|
public static void delete(final Incommon loader, final long ts)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
delete(loader, null, ts);
|
delete(loader, null, ts);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void delete(final Table loader, final byte [] column,
|
public static void delete(final Incommon loader, final byte [] column,
|
||||||
final long ts)
|
final long ts)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Delete delete = ts == HConstants.LATEST_TIMESTAMP?
|
Delete delete = ts == HConstants.LATEST_TIMESTAMP?
|
||||||
new Delete(ROW): new Delete(ROW, ts);
|
new Delete(ROW): new Delete(ROW, ts);
|
||||||
delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
|
delete.deleteColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
|
||||||
delete.setWriteToWAL(true);
|
loader.delete(delete, true);
|
||||||
loader.delete(delete);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Result get(final Table loader) throws IOException {
|
public static Result get(final Incommon loader) throws IOException {
|
||||||
return loader.get(new Get(ROW));
|
return loader.get(new Get(ROW));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,329 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hbase.regionserver;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.hbase.Cell;
|
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.TableName;
|
|
||||||
import org.apache.hadoop.hbase.client.Append;
|
|
||||||
import org.apache.hadoop.hbase.client.Delete;
|
|
||||||
import org.apache.hadoop.hbase.client.Durability;
|
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
|
||||||
import org.apache.hadoop.hbase.client.Increment;
|
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
|
||||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
|
||||||
import org.apache.hadoop.hbase.client.Row;
|
|
||||||
import org.apache.hadoop.hbase.client.RowMutations;
|
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
|
|
||||||
import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
|
|
||||||
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
|
|
||||||
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
|
|
||||||
|
|
||||||
import com.google.protobuf.Descriptors.MethodDescriptor;
|
|
||||||
import com.google.protobuf.Message;
|
|
||||||
import com.google.protobuf.Service;
|
|
||||||
import com.google.protobuf.ServiceException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An implementation of {@link Table} that sits directly on a Region; it decorates the passed in
|
|
||||||
* Region instance with the Table API. Some API is not implemented yet (throws
|
|
||||||
* {@link UnsupportedOperationException}).
|
|
||||||
*
|
|
||||||
* <p>Use as an instance of a {@link Table} in-the-small -- no networking or servers
|
|
||||||
* necessary -- or to write a test that can run directly against the datastore and then
|
|
||||||
* over the network.
|
|
||||||
*/
|
|
||||||
public class RegionTable implements Table {
|
|
||||||
private final Region region;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param region Region to decorate with Table API.
|
|
||||||
*/
|
|
||||||
public RegionTable(final Region region) {
|
|
||||||
this.region = region;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public TableName getName() {
|
|
||||||
return this.region.getTableDesc().getTableName();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Configuration getConfiguration() {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public HTableDescriptor getTableDescriptor() throws IOException {
|
|
||||||
return this.region.getTableDesc();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean exists(Get get) throws IOException {
|
|
||||||
if (!get.isCheckExistenceOnly()) throw new IllegalArgumentException();
|
|
||||||
return get(get) != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean[] existsAll(List<Get> gets) throws IOException {
|
|
||||||
boolean [] results = new boolean[gets.size()];
|
|
||||||
int index = 0;
|
|
||||||
for (Get get: gets) {
|
|
||||||
results[index++] = exists(get);
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void batch(List<? extends Row> actions, Object[] results)
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public <R> void batchCallback(List<? extends Row> actions, Object[] results,
|
|
||||||
Callback<R> callback)
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public <R> Object[] batchCallback(List<? extends Row> actions, Callback<R> callback)
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result get(Get get) throws IOException {
|
|
||||||
return this.region.get(get);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result[] get(List<Get> gets) throws IOException {
|
|
||||||
Result [] results = new Result[gets.size()];
|
|
||||||
int index = 0;
|
|
||||||
for (Get get: gets) {
|
|
||||||
results[index++] = get(get);
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
static class RegionScannerToResultScannerAdaptor implements ResultScanner {
|
|
||||||
private static final Result [] EMPTY_RESULT_ARRAY = new Result[0];
|
|
||||||
private final RegionScanner regionScanner;
|
|
||||||
|
|
||||||
RegionScannerToResultScannerAdaptor(final RegionScanner regionScanner) {
|
|
||||||
this.regionScanner = regionScanner;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Iterator<Result> iterator() {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result next() throws IOException {
|
|
||||||
List<Cell> cells = new ArrayList<Cell>();
|
|
||||||
return regionScanner.next(cells)? Result.create(cells): null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result[] next(int nbRows) throws IOException {
|
|
||||||
List<Result> results = new ArrayList<Result>(nbRows);
|
|
||||||
for (int i = 0; i < nbRows; i++) {
|
|
||||||
Result result = next();
|
|
||||||
if (result == null) break;
|
|
||||||
results.add(result);
|
|
||||||
}
|
|
||||||
return results.toArray(EMPTY_RESULT_ARRAY);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void close() {
|
|
||||||
try {
|
|
||||||
regionScanner.close();
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ResultScanner getScanner(Scan scan) throws IOException {
|
|
||||||
return new RegionScannerToResultScannerAdaptor(this.region.getScanner(scan));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ResultScanner getScanner(byte[] family) throws IOException {
|
|
||||||
return getScanner(new Scan().addFamily(family));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
|
|
||||||
return getScanner(new Scan().addColumn(family, qualifier));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void put(Put put) throws IOException {
|
|
||||||
this.region.put(put);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void put(List<Put> puts) throws IOException {
|
|
||||||
for (Put put: puts) put(put);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
|
|
||||||
byte[] value, Put put)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void delete(Delete delete) throws IOException {
|
|
||||||
this.region.delete(delete);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void delete(List<Delete> deletes) throws IOException {
|
|
||||||
for(Delete delete: deletes) delete(delete);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value,
|
|
||||||
Delete delete)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
|
|
||||||
CompareOp compareOp, byte[] value, Delete delete)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void mutateRow(RowMutations rm) throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result append(Append append) throws IOException {
|
|
||||||
return this.region.append(append, HConstants.NO_NONCE, HConstants.NO_NONCE);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result increment(Increment increment) throws IOException {
|
|
||||||
return this.region.increment(increment, HConstants.NO_NONCE, HConstants.NO_NONCE);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount,
|
|
||||||
Durability durability)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This call will NOT close the underlying region.
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public void close() throws IOException {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public CoprocessorRpcChannel coprocessorService(byte[] row) {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey,
|
|
||||||
byte[] endKey, Call<T, R> callable)
|
|
||||||
throws ServiceException, Throwable {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey,
|
|
||||||
byte[] endKey, Call<T, R> callable, Callback<R> callback)
|
|
||||||
throws ServiceException, Throwable {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public long getWriteBufferSize() {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void setWriteBufferSize(long writeBufferSize) throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public <R extends Message> Map<byte[], R> batchCoprocessorService(MethodDescriptor
|
|
||||||
methodDescriptor, Message request,
|
|
||||||
byte[] startKey, byte[] endKey, R responsePrototype)
|
|
||||||
throws ServiceException, Throwable {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
|
|
||||||
Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
|
|
||||||
throws ServiceException, Throwable {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
|
|
||||||
byte[] value, RowMutations mutation)
|
|
||||||
throws IOException {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -37,6 +37,8 @@ import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
|
@ -45,25 +47,25 @@ import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.ChoreService;
|
import org.apache.hadoop.hbase.ChoreService;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||||
import org.apache.hadoop.hbase.client.Delete;
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Durability;
|
import org.apache.hadoop.hbase.client.Durability;
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
|
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
|
||||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
|
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
|
||||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
|
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
|
||||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
|
|
||||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputControllerFactory;
|
|
||||||
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
|
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
|
||||||
import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
|
import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
|
||||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
|
||||||
|
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputControllerFactory;
|
||||||
|
import org.apache.hadoop.hbase.wal.WAL;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
import org.apache.hadoop.hbase.util.Threads;
|
import org.apache.hadoop.hbase.util.Threads;
|
||||||
import org.apache.hadoop.hbase.wal.WAL;
|
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assume;
|
import org.junit.Assume;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -82,6 +84,7 @@ import org.mockito.stubbing.Answer;
|
||||||
@Category(MediumTests.class)
|
@Category(MediumTests.class)
|
||||||
public class TestCompaction {
|
public class TestCompaction {
|
||||||
@Rule public TestName name = new TestName();
|
@Rule public TestName name = new TestName();
|
||||||
|
private static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
|
||||||
private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
|
private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
|
||||||
protected Configuration conf = UTIL.getConfiguration();
|
protected Configuration conf = UTIL.getConfiguration();
|
||||||
|
|
||||||
|
@ -143,15 +146,15 @@ public class TestCompaction {
|
||||||
int jmax = (int) Math.ceil(15.0/compactionThreshold);
|
int jmax = (int) Math.ceil(15.0/compactionThreshold);
|
||||||
byte [] pad = new byte[1000]; // 1 KB chunk
|
byte [] pad = new byte[1000]; // 1 KB chunk
|
||||||
for (int i = 0; i < compactionThreshold; i++) {
|
for (int i = 0; i < compactionThreshold; i++) {
|
||||||
Table loader = new RegionTable(r);
|
HRegionIncommon loader = new HRegionIncommon(r);
|
||||||
Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
|
Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
|
||||||
p.setDurability(Durability.SKIP_WAL);
|
p.setDurability(Durability.SKIP_WAL);
|
||||||
for (int j = 0; j < jmax; j++) {
|
for (int j = 0; j < jmax; j++) {
|
||||||
p.addColumn(COLUMN_FAMILY, Bytes.toBytes(j), pad);
|
p.add(COLUMN_FAMILY, Bytes.toBytes(j), pad);
|
||||||
}
|
}
|
||||||
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY));
|
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY));
|
||||||
loader.put(p);
|
loader.put(p);
|
||||||
r.flush(true);
|
loader.flushcache();
|
||||||
}
|
}
|
||||||
|
|
||||||
HRegion spyR = spy(r);
|
HRegion spyR = spy(r);
|
||||||
|
@ -225,9 +228,9 @@ public class TestCompaction {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createStoreFile(final HRegion region, String family) throws IOException {
|
private void createStoreFile(final HRegion region, String family) throws IOException {
|
||||||
Table loader = new RegionTable(region);
|
HRegionIncommon loader = new HRegionIncommon(region);
|
||||||
HBaseTestCase.addContent(loader, family);
|
HBaseTestCase.addContent(loader, family);
|
||||||
region.flush(true);
|
loader.flushcache();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -40,6 +40,7 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.Cell;
|
import org.apache.hadoop.hbase.Cell;
|
||||||
import org.apache.hadoop.hbase.CellUtil;
|
import org.apache.hadoop.hbase.CellUtil;
|
||||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
@ -47,7 +48,6 @@ import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
||||||
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
|
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
|
||||||
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
|
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
|
||||||
|
@ -197,7 +197,7 @@ public class TestMajorCompaction {
|
||||||
createStoreFile(r);
|
createStoreFile(r);
|
||||||
}
|
}
|
||||||
// Add more content.
|
// Add more content.
|
||||||
HBaseTestCase.addContent(new RegionTable(r), Bytes.toString(COLUMN_FAMILY));
|
HBaseTestCase.addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY));
|
||||||
|
|
||||||
// Now there are about 5 versions of each column.
|
// Now there are about 5 versions of each column.
|
||||||
// Default is that there only 3 (MAXVERSIONS) versions allowed per column.
|
// Default is that there only 3 (MAXVERSIONS) versions allowed per column.
|
||||||
|
@ -386,13 +386,13 @@ public class TestMajorCompaction {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createStoreFile(final Region region, String family) throws IOException {
|
private void createStoreFile(final Region region, String family) throws IOException {
|
||||||
Table loader = new RegionTable(region);
|
HRegionIncommon loader = new HRegionIncommon(region);
|
||||||
HBaseTestCase.addContent(loader, family);
|
HBaseTestCase.addContent(loader, family);
|
||||||
loader.flushcache();
|
loader.flushcache();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createSmallerStoreFile(final Region region) throws IOException {
|
private void createSmallerStoreFile(final Region region) throws IOException {
|
||||||
Table loader = new RegionTable(region);
|
HRegionIncommon loader = new HRegionIncommon(region);
|
||||||
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
|
HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
|
||||||
"bbb").getBytes(), null);
|
"bbb").getBytes(), null);
|
||||||
loader.flushcache();
|
loader.flushcache();
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
@ -35,7 +36,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||||
import org.apache.hadoop.hbase.client.Delete;
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.wal.WAL;
|
import org.apache.hadoop.hbase.wal.WAL;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
|
@ -161,7 +161,7 @@ public class TestMinorCompaction {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception {
|
private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception {
|
||||||
Table loader = new RegionTable(r);
|
HRegionIncommon loader = new HRegionIncommon(r);
|
||||||
for (int i = 0; i < compactionThreshold + 1; i++) {
|
for (int i = 0; i < compactionThreshold + 1; i++) {
|
||||||
HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
|
HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
|
||||||
thirdRowBytes, i);
|
thirdRowBytes, i);
|
||||||
|
|
|
@ -36,6 +36,8 @@ import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.hbase.Cell;
|
import org.apache.hadoop.hbase.Cell;
|
||||||
import org.apache.hadoop.hbase.CellUtil;
|
import org.apache.hadoop.hbase.CellUtil;
|
||||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase.ScannerIncommon;
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
@ -47,9 +49,7 @@ import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.filter.Filter;
|
import org.apache.hadoop.hbase.filter.Filter;
|
||||||
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
|
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
|
||||||
import org.apache.hadoop.hbase.filter.PrefixFilter;
|
import org.apache.hadoop.hbase.filter.PrefixFilter;
|
||||||
|
@ -99,10 +99,11 @@ public class TestScanner {
|
||||||
|
|
||||||
private static final long START_CODE = Long.MAX_VALUE;
|
private static final long START_CODE = Long.MAX_VALUE;
|
||||||
|
|
||||||
private HRegion region;
|
private HRegion r;
|
||||||
|
private HRegionIncommon region;
|
||||||
|
|
||||||
private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
|
private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
|
||||||
final private byte[] col1;
|
final private byte[] col1, col2;
|
||||||
|
|
||||||
public TestScanner() {
|
public TestScanner() {
|
||||||
super();
|
super();
|
||||||
|
@ -114,6 +115,7 @@ public class TestScanner {
|
||||||
thirdRowBytes = START_KEY_BYTES.clone();
|
thirdRowBytes = START_KEY_BYTES.clone();
|
||||||
thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
|
thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
|
||||||
col1 = Bytes.toBytes("column1");
|
col1 = Bytes.toBytes("column1");
|
||||||
|
col2 = Bytes.toBytes("column2");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -125,14 +127,14 @@ public class TestScanner {
|
||||||
byte [] startrow = Bytes.toBytes("bbb");
|
byte [] startrow = Bytes.toBytes("bbb");
|
||||||
byte [] stoprow = Bytes.toBytes("ccc");
|
byte [] stoprow = Bytes.toBytes("ccc");
|
||||||
try {
|
try {
|
||||||
this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
||||||
HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY);
|
HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY);
|
||||||
List<Cell> results = new ArrayList<Cell>();
|
List<Cell> results = new ArrayList<Cell>();
|
||||||
// Do simple test of getting one row only first.
|
// Do simple test of getting one row only first.
|
||||||
Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
|
Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
|
||||||
scan.addFamily(HConstants.CATALOG_FAMILY);
|
scan.addFamily(HConstants.CATALOG_FAMILY);
|
||||||
|
|
||||||
InternalScanner s = region.getScanner(scan);
|
InternalScanner s = r.getScanner(scan);
|
||||||
int count = 0;
|
int count = 0;
|
||||||
while (s.next(results)) {
|
while (s.next(results)) {
|
||||||
count++;
|
count++;
|
||||||
|
@ -143,7 +145,7 @@ public class TestScanner {
|
||||||
scan = new Scan(startrow, stoprow);
|
scan = new Scan(startrow, stoprow);
|
||||||
scan.addFamily(HConstants.CATALOG_FAMILY);
|
scan.addFamily(HConstants.CATALOG_FAMILY);
|
||||||
|
|
||||||
s = region.getScanner(scan);
|
s = r.getScanner(scan);
|
||||||
count = 0;
|
count = 0;
|
||||||
Cell kv = null;
|
Cell kv = null;
|
||||||
results = new ArrayList<Cell>();
|
results = new ArrayList<Cell>();
|
||||||
|
@ -160,14 +162,14 @@ public class TestScanner {
|
||||||
assertTrue(count > 10);
|
assertTrue(count > 10);
|
||||||
s.close();
|
s.close();
|
||||||
} finally {
|
} finally {
|
||||||
HRegion.closeHRegion(this.region);
|
HRegion.closeHRegion(this.r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void rowPrefixFilter(Scan scan) throws IOException {
|
void rowPrefixFilter(Scan scan) throws IOException {
|
||||||
List<Cell> results = new ArrayList<Cell>();
|
List<Cell> results = new ArrayList<Cell>();
|
||||||
scan.addFamily(HConstants.CATALOG_FAMILY);
|
scan.addFamily(HConstants.CATALOG_FAMILY);
|
||||||
InternalScanner s = region.getScanner(scan);
|
InternalScanner s = r.getScanner(scan);
|
||||||
boolean hasMore = true;
|
boolean hasMore = true;
|
||||||
while (hasMore) {
|
while (hasMore) {
|
||||||
hasMore = s.next(results);
|
hasMore = s.next(results);
|
||||||
|
@ -183,7 +185,7 @@ public class TestScanner {
|
||||||
void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException {
|
void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException {
|
||||||
List<Cell> results = new ArrayList<Cell>();
|
List<Cell> results = new ArrayList<Cell>();
|
||||||
scan.addFamily(HConstants.CATALOG_FAMILY);
|
scan.addFamily(HConstants.CATALOG_FAMILY);
|
||||||
InternalScanner s = region.getScanner(scan);
|
InternalScanner s = r.getScanner(scan);
|
||||||
boolean hasMore = true;
|
boolean hasMore = true;
|
||||||
while (hasMore) {
|
while (hasMore) {
|
||||||
hasMore = s.next(results);
|
hasMore = s.next(results);
|
||||||
|
@ -198,8 +200,8 @@ public class TestScanner {
|
||||||
@Test
|
@Test
|
||||||
public void testFilters() throws IOException {
|
public void testFilters() throws IOException {
|
||||||
try {
|
try {
|
||||||
this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
||||||
HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY);
|
HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY);
|
||||||
byte [] prefix = Bytes.toBytes("ab");
|
byte [] prefix = Bytes.toBytes("ab");
|
||||||
Filter newFilter = new PrefixFilter(prefix);
|
Filter newFilter = new PrefixFilter(prefix);
|
||||||
Scan scan = new Scan();
|
Scan scan = new Scan();
|
||||||
|
@ -213,7 +215,7 @@ public class TestScanner {
|
||||||
rowInclusiveStopFilter(scan, stopRow);
|
rowInclusiveStopFilter(scan, stopRow);
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
HRegion.closeHRegion(this.region);
|
HRegion.closeHRegion(this.r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -225,10 +227,10 @@ public class TestScanner {
|
||||||
@Test
|
@Test
|
||||||
public void testRaceBetweenClientAndTimeout() throws Exception {
|
public void testRaceBetweenClientAndTimeout() throws Exception {
|
||||||
try {
|
try {
|
||||||
this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
||||||
HBaseTestCase.addContent(this.region, HConstants.CATALOG_FAMILY);
|
HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY);
|
||||||
Scan scan = new Scan();
|
Scan scan = new Scan();
|
||||||
InternalScanner s = region.getScanner(scan);
|
InternalScanner s = r.getScanner(scan);
|
||||||
List<Cell> results = new ArrayList<Cell>();
|
List<Cell> results = new ArrayList<Cell>();
|
||||||
try {
|
try {
|
||||||
s.next(results);
|
s.next(results);
|
||||||
|
@ -240,7 +242,7 @@ public class TestScanner {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
HRegion.closeHRegion(this.region);
|
HRegion.closeHRegion(this.r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -250,70 +252,71 @@ public class TestScanner {
|
||||||
@Test
|
@Test
|
||||||
public void testScanner() throws IOException {
|
public void testScanner() throws IOException {
|
||||||
try {
|
try {
|
||||||
region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
||||||
Table table = new RegionTable(region);
|
region = new HRegionIncommon(r);
|
||||||
|
|
||||||
// Write information to the meta table
|
// Write information to the meta table
|
||||||
|
|
||||||
Put put = new Put(ROW_KEY, System.currentTimeMillis());
|
Put put = new Put(ROW_KEY, System.currentTimeMillis());
|
||||||
|
|
||||||
put.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
|
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
|
||||||
REGION_INFO.toByteArray());
|
REGION_INFO.toByteArray());
|
||||||
table.put(put);
|
region.put(put);
|
||||||
|
|
||||||
// What we just committed is in the memstore. Verify that we can get
|
// What we just committed is in the memstore. Verify that we can get
|
||||||
// it back both with scanning and get
|
// it back both with scanning and get
|
||||||
|
|
||||||
scan(false, null);
|
scan(false, null);
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// Close and re-open
|
// Close and re-open
|
||||||
|
|
||||||
((HRegion)region).close();
|
((HRegion)r).close();
|
||||||
region = HRegion.openHRegion(region, null);
|
r = HRegion.openHRegion(r, null);
|
||||||
table = new RegionTable(region);
|
region = new HRegionIncommon(r);
|
||||||
|
|
||||||
// Verify we can get the data back now that it is on disk.
|
// Verify we can get the data back now that it is on disk.
|
||||||
|
|
||||||
scan(false, null);
|
scan(false, null);
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// Store some new information
|
// Store some new information
|
||||||
|
|
||||||
String address = HConstants.LOCALHOST_IP + ":" + HBaseTestingUtility.randomFreePort();
|
String address = HConstants.LOCALHOST_IP + ":" + HBaseTestingUtility.randomFreePort();
|
||||||
|
|
||||||
put = new Put(ROW_KEY, System.currentTimeMillis());
|
put = new Put(ROW_KEY, System.currentTimeMillis());
|
||||||
put.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
|
put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
|
||||||
Bytes.toBytes(address));
|
Bytes.toBytes(address));
|
||||||
|
|
||||||
// put.add(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE));
|
// put.add(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE));
|
||||||
|
|
||||||
table.put(put);
|
region.put(put);
|
||||||
|
|
||||||
// Validate that we can still get the HRegionInfo, even though it is in
|
// Validate that we can still get the HRegionInfo, even though it is in
|
||||||
// an older row on disk and there is a newer row in the memstore
|
// an older row on disk and there is a newer row in the memstore
|
||||||
|
|
||||||
scan(true, address.toString());
|
scan(true, address.toString());
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// flush cache
|
// flush cache
|
||||||
this.region.flush(true);
|
|
||||||
|
region.flushcache();
|
||||||
|
|
||||||
// Validate again
|
// Validate again
|
||||||
|
|
||||||
scan(true, address.toString());
|
scan(true, address.toString());
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// Close and reopen
|
// Close and reopen
|
||||||
|
|
||||||
((HRegion)region).close();
|
((HRegion)r).close();
|
||||||
region = HRegion.openHRegion(region,null);
|
r = HRegion.openHRegion(r,null);
|
||||||
table = new RegionTable(region);
|
region = new HRegionIncommon(r);
|
||||||
|
|
||||||
// Validate again
|
// Validate again
|
||||||
|
|
||||||
scan(true, address.toString());
|
scan(true, address.toString());
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// Now update the information again
|
// Now update the information again
|
||||||
|
|
||||||
|
@ -321,36 +324,38 @@ public class TestScanner {
|
||||||
|
|
||||||
put = new Put(ROW_KEY, System.currentTimeMillis());
|
put = new Put(ROW_KEY, System.currentTimeMillis());
|
||||||
|
|
||||||
put.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(address));
|
put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
|
||||||
table.put(put);
|
Bytes.toBytes(address));
|
||||||
|
region.put(put);
|
||||||
|
|
||||||
// Validate again
|
// Validate again
|
||||||
|
|
||||||
scan(true, address.toString());
|
scan(true, address.toString());
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// flush cache
|
// flush cache
|
||||||
region.flush(true);
|
|
||||||
|
region.flushcache();
|
||||||
|
|
||||||
// Validate again
|
// Validate again
|
||||||
|
|
||||||
scan(true, address.toString());
|
scan(true, address.toString());
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
// Close and reopen
|
// Close and reopen
|
||||||
|
|
||||||
((HRegion)region).close();
|
((HRegion)r).close();
|
||||||
region = HRegion.openHRegion(region,null);
|
r = HRegion.openHRegion(r,null);
|
||||||
table = new RegionTable(region);
|
region = new HRegionIncommon(r);
|
||||||
|
|
||||||
// Validate again
|
// Validate again
|
||||||
|
|
||||||
scan(true, address.toString());
|
scan(true, address.toString());
|
||||||
getRegionInfo(table);
|
getRegionInfo();
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
// clean up
|
// clean up
|
||||||
HRegion.closeHRegion(region);
|
HRegion.closeHRegion(r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -382,7 +387,7 @@ public class TestScanner {
|
||||||
for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) {
|
for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) {
|
||||||
scan.addColumn(COLS[0], EXPLICIT_COLS[ii]);
|
scan.addColumn(COLS[0], EXPLICIT_COLS[ii]);
|
||||||
}
|
}
|
||||||
scanner = region.getScanner(scan);
|
scanner = r.getScanner(scan);
|
||||||
while (scanner.next(results)) {
|
while (scanner.next(results)) {
|
||||||
assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
|
assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
|
||||||
HConstants.REGIONINFO_QUALIFIER));
|
HConstants.REGIONINFO_QUALIFIER));
|
||||||
|
@ -443,10 +448,10 @@ public class TestScanner {
|
||||||
|
|
||||||
|
|
||||||
/** Use get to retrieve the HRegionInfo and validate it */
|
/** Use get to retrieve the HRegionInfo and validate it */
|
||||||
private void getRegionInfo(Table table) throws IOException {
|
private void getRegionInfo() throws IOException {
|
||||||
Get get = new Get(ROW_KEY);
|
Get get = new Get(ROW_KEY);
|
||||||
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
|
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
|
||||||
Result result = table.get(get);
|
Result result = region.get(get);
|
||||||
byte [] bytes = result.value();
|
byte [] bytes = result.value();
|
||||||
validateRegionInfo(bytes);
|
validateRegionInfo(bytes);
|
||||||
}
|
}
|
||||||
|
@ -459,11 +464,10 @@ public class TestScanner {
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testScanAndSyncFlush() throws Exception {
|
public void testScanAndSyncFlush() throws Exception {
|
||||||
this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
||||||
Table hri = new RegionTable(region);
|
HRegionIncommon hri = new HRegionIncommon(r);
|
||||||
try {
|
try {
|
||||||
LOG.info("Added: " +
|
LOG.info("Added: " + HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
|
||||||
HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
|
|
||||||
Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
|
Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
|
||||||
int count = count(hri, -1, false);
|
int count = count(hri, -1, false);
|
||||||
assertEquals(count, count(hri, 100, false)); // do a sync flush.
|
assertEquals(count, count(hri, 100, false)); // do a sync flush.
|
||||||
|
@ -471,7 +475,7 @@ public class TestScanner {
|
||||||
LOG.error("Failed", e);
|
LOG.error("Failed", e);
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
HRegion.closeHRegion(this.region);
|
HRegion.closeHRegion(this.r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -483,11 +487,10 @@ public class TestScanner {
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testScanAndRealConcurrentFlush() throws Exception {
|
public void testScanAndRealConcurrentFlush() throws Exception {
|
||||||
this.region = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null);
|
||||||
Table hri = new RegionTable(region);
|
HRegionIncommon hri = new HRegionIncommon(r);
|
||||||
try {
|
try {
|
||||||
LOG.info("Added: " +
|
LOG.info("Added: " + HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
|
||||||
HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
|
|
||||||
Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
|
Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
|
||||||
int count = count(hri, -1, false);
|
int count = count(hri, -1, false);
|
||||||
assertEquals(count, count(hri, 100, true)); // do a true concurrent background thread flush
|
assertEquals(count, count(hri, 100, true)); // do a true concurrent background thread flush
|
||||||
|
@ -495,7 +498,7 @@ public class TestScanner {
|
||||||
LOG.error("Failed", e);
|
LOG.error("Failed", e);
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
HRegion.closeHRegion(this.region);
|
HRegion.closeHRegion(this.r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -509,8 +512,8 @@ public class TestScanner {
|
||||||
@SuppressWarnings("deprecation")
|
@SuppressWarnings("deprecation")
|
||||||
public void testScanAndConcurrentMajorCompact() throws Exception {
|
public void testScanAndConcurrentMajorCompact() throws Exception {
|
||||||
HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name.getMethodName());
|
HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name.getMethodName());
|
||||||
this.region = TEST_UTIL.createLocalHRegion(htd, null, null);
|
this.r = TEST_UTIL.createLocalHRegion(htd, null, null);
|
||||||
Table hri = new RegionTable(region);
|
HRegionIncommon hri = new HRegionIncommon(r);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
|
HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
|
||||||
|
@ -521,18 +524,18 @@ public class TestScanner {
|
||||||
Delete dc = new Delete(firstRowBytes);
|
Delete dc = new Delete(firstRowBytes);
|
||||||
/* delete column1 of firstRow */
|
/* delete column1 of firstRow */
|
||||||
dc.deleteColumns(fam1, col1);
|
dc.deleteColumns(fam1, col1);
|
||||||
region.delete(dc);
|
r.delete(dc);
|
||||||
region.flush(true);
|
r.flush(true);
|
||||||
|
|
||||||
HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
|
HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
|
||||||
secondRowBytes, thirdRowBytes);
|
secondRowBytes, thirdRowBytes);
|
||||||
HBaseTestCase.addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
|
HBaseTestCase.addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
|
||||||
secondRowBytes, thirdRowBytes);
|
secondRowBytes, thirdRowBytes);
|
||||||
region.flush(true);
|
r.flush(true);
|
||||||
|
|
||||||
InternalScanner s = region.getScanner(new Scan());
|
InternalScanner s = r.getScanner(new Scan());
|
||||||
// run a major compact, column1 of firstRow will be cleaned.
|
// run a major compact, column1 of firstRow will be cleaned.
|
||||||
region.compact(true);
|
r.compact(true);
|
||||||
|
|
||||||
List<Cell> results = new ArrayList<Cell>();
|
List<Cell> results = new ArrayList<Cell>();
|
||||||
s.next(results);
|
s.next(results);
|
||||||
|
@ -552,7 +555,7 @@ public class TestScanner {
|
||||||
assertTrue(CellUtil.matchingFamily(results.get(0), fam1));
|
assertTrue(CellUtil.matchingFamily(results.get(0), fam1));
|
||||||
assertTrue(CellUtil.matchingFamily(results.get(1), fam2));
|
assertTrue(CellUtil.matchingFamily(results.get(1), fam2));
|
||||||
} finally {
|
} finally {
|
||||||
HRegion.closeHRegion(this.region);
|
HRegion.closeHRegion(this.r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -564,20 +567,19 @@ public class TestScanner {
|
||||||
* @return Count of rows found.
|
* @return Count of rows found.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
private int count(final Table countTable, final int flushIndex, boolean concurrent)
|
private int count(final HRegionIncommon hri, final int flushIndex,
|
||||||
|
boolean concurrent)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
LOG.info("Taking out counting scan");
|
LOG.info("Taking out counting scan");
|
||||||
Scan scan = new Scan();
|
ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
|
||||||
for (byte [] qualifier: EXPLICIT_COLS) {
|
HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
|
||||||
scan.addColumn(HConstants.CATALOG_FAMILY, qualifier);
|
List<Cell> values = new ArrayList<Cell>();
|
||||||
}
|
|
||||||
ResultScanner s = countTable.getScanner(scan);
|
|
||||||
int count = 0;
|
int count = 0;
|
||||||
boolean justFlushed = false;
|
boolean justFlushed = false;
|
||||||
while (s.next() != null) {
|
while (s.next(values)) {
|
||||||
if (justFlushed) {
|
if (justFlushed) {
|
||||||
LOG.info("after next() just after next flush");
|
LOG.info("after next() just after next flush");
|
||||||
justFlushed = false;
|
justFlushed=false;
|
||||||
}
|
}
|
||||||
count++;
|
count++;
|
||||||
if (flushIndex == count) {
|
if (flushIndex == count) {
|
||||||
|
@ -585,7 +587,7 @@ public class TestScanner {
|
||||||
Thread t = new Thread() {
|
Thread t = new Thread() {
|
||||||
public void run() {
|
public void run() {
|
||||||
try {
|
try {
|
||||||
region.flush(true);
|
hri.flushcache();
|
||||||
LOG.info("Finishing flush");
|
LOG.info("Finishing flush");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.info("Failed flush cache");
|
LOG.info("Failed flush cache");
|
||||||
|
@ -605,4 +607,5 @@ public class TestScanner {
|
||||||
LOG.info("Found " + count + " items");
|
LOG.info("Found " + count + " items");
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
Loading…
Reference in New Issue