Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-19841 LocalHTU to not enforce stream capabilities

This commit is contained in:
Mike Drob 2018-01-24 00:42:32 -06:00
parent 94a1c051ba
commit 9ea152d235
26 changed files with 54 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
View File

@ -394,7 +394,13 @@ public abstract class CommonFSUtils {
public static FileSystem getWALFileSystem(final Configuration c) throws IOException {
Path p = getWALRootDir(c);
return p.getFileSystem(c);
FileSystem fs = p.getFileSystem(c);
// Need to copy this to the new filesystem we are returning in case it is localFS
String enforceStreamCapabilities = c.get(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE);
if (enforceStreamCapabilities != null) {
fs.getConf().set(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, enforceStreamCapabilities);
}
return fs;
}
private static boolean isValidWALRootDir(Path walDir, final Configuration c) throws IOException {

1
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
View File

@ -113,6 +113,7 @@ public abstract class HBaseTestCase extends TestCase {
if (fs.exists(testDir)) {
fs.delete(testDir, true);
}
conf.setBoolean("hbase.unsafe.stream.capability.enforce",false);
} else {
testDir = FSUtils.getRootDir(conf);
}

10
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase;
import static org.apache.hadoop.hbase.util.CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -298,10 +299,18 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
public static final Collection<Object[]> BLOOM_AND_COMPRESSION_COMBINATIONS =
bloomAndCompressionCombinations();
/**
* Creates a new HBaseTestingUtility for starting a mini-cluster.
* If you are not using the startMini* methods, consider {@link #createLocalHTU()} instead.
*/
public HBaseTestingUtility() {
this(HBaseConfiguration.create());
}
/**
* Creates a new HBaseTestingUtility for starting a mini-cluster.
* If you are not using the startMini* methods, consider {@link #createLocalHTU()} instead.
*/
public HBaseTestingUtility(Configuration conf) {
super(conf);
@ -347,6 +356,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
htu.getConfiguration().set("fs.defaultFS","file:///");
htu.getConfiguration().set(HConstants.HBASE_DIR, "file://" + dataTestDir);
LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir);
htu.getConfiguration().setBoolean(UNSAFE_STREAM_CAPABILITY_ENFORCE,false);
htu.localMode = true;
return htu;
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIntraRowPagination.java
View File

@ -40,7 +40,7 @@ import org.junit.experimental.categories.Category;
@Category({SmallTests.class, ClientTests.class})
public class TestIntraRowPagination {
private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
/**
* Test from client side for scan with maxResultPerCF set

7
hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java
View File

@ -27,12 +27,10 @@ import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MockRegionServerServices;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Durability;
@ -50,8 +48,7 @@ import org.mockito.Mockito;
@Category({CoprocessorTests.class, SmallTests.class})
public class TestRegionObserverStacking extends TestCase {
private static HBaseTestingUtility TEST_UTIL
= new HBaseTestingUtility();
private static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
static final Path DIR = TEST_UTIL.getDataTestDir();
public static class ObserverA implements RegionCoprocessor, RegionObserver {
@ -143,7 +140,7 @@ public class TestRegionObserverStacking extends TestCase {
byte[] A = Bytes.toBytes("A");
byte[][] FAMILIES = new byte[][] { A } ;
Configuration conf = HBaseConfiguration.create();
Configuration conf = TEST_UTIL.getConfiguration();
HRegion region = initHRegion(TABLE, getClass().getName(),
conf, FAMILIES);
RegionCoprocessorHost h = region.getCoprocessorHost();

3
hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java
View File

@ -51,8 +51,7 @@ import org.junit.rules.TestName;
@Category({FilterTests.class, SmallTests.class})
public class TestColumnPrefixFilter {
private final static HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
@Rule
public TestName name = new TestName();

2
hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java
View File

@ -71,7 +71,7 @@ public class TestDependentColumnFilter {
Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
};
private static final byte[] MATCH_VAL = Bytes.toBytes("match");
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
List<KeyValue> testVals;
private HRegion region;

2
hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
View File

@ -71,7 +71,7 @@ import org.slf4j.LoggerFactory;
public class TestFilter {
private final static Logger LOG = LoggerFactory.getLogger(TestFilter.class);
private HRegion region;
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
@Rule
public TestName name = new TestName();

2
hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
View File

@ -50,7 +50,7 @@ import java.util.List;
@Category(SmallTests.class)
public class TestFilterFromRegionSide {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private static HRegion REGION;
private static TableName TABLE_NAME = TableName.valueOf("TestFilterFromRegionSide");

2
hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
View File

@ -59,7 +59,7 @@ public class TestInvocationRecordFilter {
private static final String QUALIFIER_PREFIX = "qualifier";
private static final String VALUE_PREFIX = "value";
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private HRegion region;
@Before

3
hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java
View File

@ -51,8 +51,7 @@ import org.junit.rules.TestName;
@Category({FilterTests.class, SmallTests.class})
public class TestMultipleColumnPrefixFilter {
private final static HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
@Rule
public TestName name = new TestName();

3
hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java
View File

@ -23,6 +23,7 @@ import java.util.concurrent.ExecutionException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.FSUtils;
@ -42,7 +43,7 @@ public class TestLocalAsyncOutput {
private static Class<? extends Channel> CHANNEL_CLASS = NioSocketChannel.class;
private static final HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility();
private static final HBaseCommonTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
@AfterClass
public static void tearDownAfterClass() throws IOException {

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
View File

@ -46,7 +46,7 @@ import org.junit.experimental.categories.Category;
@Category({ IOTests.class, SmallTests.class })
public class TestSeekBeforeWithReverseScan {
private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
private final HBaseTestingUtility testUtil = HBaseTestingUtility.createLocalHTU();
private HRegion region;

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
View File

@ -55,7 +55,7 @@ public class TestBlocksScanned extends HBaseTestCase {
public void setUp() throws Exception {
super.setUp();
TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL = HBaseTestingUtility.createLocalHTU();
}
@Test

6
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
View File

@ -95,11 +95,13 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
protected void compactingSetUp() throws Exception {
super.internalSetUp();
Configuration conf = new Configuration();
HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU();
Configuration conf = hbaseUtility.getConfiguration();
conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("foobar"));
htd.addFamily(hcd);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
View File

@ -47,7 +47,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
public class TestCompactionPolicy {
private final static Logger LOG = LoggerFactory.getLogger(TestCompactionPolicy.class);
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
protected final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
protected Configuration conf;
protected HStore store;

20
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
View File

@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
@ -92,6 +91,9 @@ public class TestDefaultMemStore {
return this.name.getMethodName();
}
private HBaseTestingUtility util;
private Configuration conf;
@Before
public void setUp() throws Exception {
internalSetUp();
@ -107,6 +109,8 @@ public class TestDefaultMemStore {
}
protected void internalSetUp() throws Exception {
this.util = HBaseTestingUtility.createLocalHTU();
this.conf = util.getConfiguration();
this.mvcc = new MultiVersionConcurrencyControl();
}
@ -162,7 +166,6 @@ public class TestDefaultMemStore {
List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0);
Scan scan = new Scan();
List<Cell> result = new ArrayList<>();
Configuration conf = HBaseConfiguration.create();
ScanInfo scanInfo = new ScanInfo(conf, null, 0, 1, HConstants.LATEST_TIMESTAMP,
KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false);
int count = 0;
@ -536,7 +539,7 @@ public class TestDefaultMemStore {
@Test
public void testMultipleVersionsSimple() throws Exception {
DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR);
DefaultMemStore m = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR);
byte [] row = Bytes.toBytes("testRow");
byte [] family = Bytes.toBytes("testFamily");
byte [] qf = Bytes.toBytes("testQualifier");
@ -582,7 +585,6 @@ public class TestDefaultMemStore {
}
}
//starting from each row, validate results should contain the starting row
Configuration conf = HBaseConfiguration.create();
for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
ScanInfo scanInfo =
new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
@ -818,7 +820,6 @@ public class TestDefaultMemStore {
*/
@Test
public void testUpsertMemstoreSize() throws Exception {
Configuration conf = HBaseConfiguration.create();
memstore = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR);
MemStoreSize oldSize = memstore.size();
@ -901,7 +902,6 @@ public class TestDefaultMemStore {
*/
@Test
public void testShouldFlush() throws Exception {
Configuration conf = new Configuration();
conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
checkShouldFlush(conf, true);
// test disable flush
@ -913,10 +913,8 @@ public class TestDefaultMemStore {
try {
EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
EnvironmentEdgeManager.injectEdge(edge);
HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
String cf = "foo";
HRegion region =
hbaseUtility.createTestRegion("foobar", ColumnFamilyDescriptorBuilder.of(cf));
HRegion region = util.createTestRegion("foobar", ColumnFamilyDescriptorBuilder.of(cf));
edge.setCurrentTimeMillis(1234);
Put p = new Put(Bytes.toBytes("r"));
@ -937,10 +935,8 @@ public class TestDefaultMemStore {
// write an edit in the META and ensure the shouldFlush (that the periodic memstore
// flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though
// the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value)
Configuration conf = new Configuration();
conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10);
HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
Path testDir = hbaseUtility.getDataTestDir();
Path testDir = util.getDataTestDir();
EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
EnvironmentEdgeManager.injectEdge(edge);
edge.setCurrentTimeMillis(1234);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
View File

@ -65,7 +65,7 @@ public class TestHRegionInfo {
@Test
public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException {
HBaseTestingUtility htu = new HBaseTestingUtility();
HBaseTestingUtility htu = HBaseTestingUtility.createLocalHTU();
HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
Path basedir = htu.getDataTestDir();
// Create a region. That'll write the .regioninfo file.

6
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
View File

@ -146,7 +146,7 @@ public class TestHStore {
long id = System.currentTimeMillis();
Get get = new Get(row);
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private static final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
@ -234,7 +234,7 @@ public class TestHStore {
public void testFlushSizeSizing() throws Exception {
LOG.info("Setting up a faulty file system that cannot write in " +
this.name.getMethodName());
final Configuration conf = HBaseConfiguration.create();
final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
// Only retry once.
conf.setInt("hbase.hstore.flush.retries.number", 1);
User user = User.createUserForTesting(conf, this.name.getMethodName(),
@ -661,7 +661,7 @@ public class TestHStore {
public void testHandleErrorsInFlush() throws Exception {
LOG.info("Setting up a faulty file system that cannot write");
final Configuration conf = HBaseConfiguration.create();
final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
User user = User.createUserForTesting(conf,
"testhandleerrorsinflush", new String[]{"foo"});
// Inject our faulty LocalFileSystem

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionInfoBuilder.java
View File

@ -91,7 +91,7 @@ public class TestRegionInfoBuilder {
@Test
public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException {
HBaseTestingUtility htu = new HBaseTestingUtility();
HBaseTestingUtility htu = HBaseTestingUtility.createLocalHTU();
RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO;
Path basedir = htu.getDataTestDir();
// Create a region. That'll write the .regioninfo file.

3
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java
View File

@ -45,8 +45,7 @@ public class TestResettingCounters {
@Test
public void testResettingCounters() throws Exception {
HBaseTestingUtility htu = new HBaseTestingUtility();
HBaseTestingUtility htu = HBaseTestingUtility.createLocalHTU();
Configuration conf = htu.getConfiguration();
FileSystem fs = FileSystem.get(conf);
byte [] table = Bytes.toBytes(name.getMethodName());

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
View File

@ -70,7 +70,7 @@ public class TestStoreFileRefresherChore {
@Before
public void setUp() throws IOException {
TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL = HBaseTestingUtility.createLocalHTU();
testDir = TEST_UTIL.getDataTestDir("TestStoreFileRefresherChore");
FSUtils.setRootDir(TEST_UTIL.getConfiguration(), testDir);
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALMonotonicallyIncreasingSeqId.java
View File

@ -77,7 +77,7 @@ import org.slf4j.LoggerFactory;
@Category({ RegionServerTests.class, SmallTests.class })
public class TestWALMonotonicallyIncreasingSeqId {
private final Logger LOG = LoggerFactory.getLogger(getClass());
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private static Path testDir = TEST_UTIL.getDataTestDir("TestWALMonotonicallyIncreasingSeqId");
private WALFactory wals;
private FileSystem fileSystem;

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
View File

@ -55,7 +55,7 @@ import org.junit.experimental.categories.Category;
public class TestWALActionsListener {
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
HBaseTestingUtility.createLocalHTU();
private final static byte[] SOME_BYTES = Bytes.toBytes("t");
private static Configuration conf;

2
hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java
View File

@ -56,7 +56,7 @@ public class TestWALMethods {
private static final TableName TEST_TABLE =
TableName.valueOf("test_table");
private final HBaseTestingUtility util = new HBaseTestingUtility();
private final HBaseTestingUtility util = HBaseTestingUtility.createLocalHTU();
@Test
public void testServerNameFromWAL() throws Exception {

9
hbase-server/src/test/resources/hbase-site.xml
View File

@ -158,13 +158,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.unsafe.stream.capability.enforce</name>
<value>false</value>
<description>
Controls whether HBase will check for stream capabilities (hflush/hsync).
Disable this if you intend to run on LocalFileSystem.
WARNING: Doing so may expose you to additional risk of data loss!
</description>
</property>
</configuration>