HBASE-14456 Implement a namespace-based region grouping strategy for RegionGroupingProvider (Yu Li)

This commit is contained in:
tedyu 2015-09-22 09:26:56 -07:00
parent 1592750eb1
commit 27bc559681
41 changed files with 149 additions and 77 deletions

View File

@ -1843,9 +1843,10 @@ public class HRegionServer extends HasThread implements
roller = ensureMetaWALRoller(); roller = ensureMetaWALRoller();
wal = walFactory.getMetaWAL(regionInfo.getEncodedNameAsBytes()); wal = walFactory.getMetaWAL(regionInfo.getEncodedNameAsBytes());
} else if (regionInfo == null) { } else if (regionInfo == null) {
wal = walFactory.getWAL(UNSPECIFIED_REGION); wal = walFactory.getWAL(UNSPECIFIED_REGION, null);
} else { } else {
wal = walFactory.getWAL(regionInfo.getEncodedNameAsBytes()); byte[] namespace = regionInfo.getTable().getNamespace();
wal = walFactory.getWAL(regionInfo.getEncodedNameAsBytes(), namespace);
} }
roller.addWAL(wal); roller.addWAL(wal);
return wal; return wal;

View File

@ -1342,7 +1342,7 @@ public class HBaseFsck extends Configured implements Closeable {
WAL wal = (new WALFactory(confForWAL, WAL wal = (new WALFactory(confForWAL,
Collections.<WALActionsListener>singletonList(new MetricsWAL()), Collections.<WALActionsListener>singletonList(new MetricsWAL()),
"hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))). "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
getWAL(metaHRI.getEncodedNameAsBytes()); getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
return meta; return meta;

View File

@ -193,11 +193,13 @@ class HMerge {
for (int i = 0; i < info.length - 1; i++) { for (int i = 0; i < info.length - 1; i++) {
if (currentRegion == null) { if (currentRegion == null) {
currentRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i], this.htd, currentRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i], this.htd,
walFactory.getWAL(info[i].getEncodedNameAsBytes())); walFactory.getWAL(info[i].getEncodedNameAsBytes(),
info[i].getTable().getNamespace()));
currentSize = currentRegion.getLargestHStoreSize(); currentSize = currentRegion.getLargestHStoreSize();
} }
nextRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i + 1], this.htd, nextRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i + 1], this.htd,
walFactory.getWAL(info[i+1].getEncodedNameAsBytes())); walFactory.getWAL(info[i + 1].getEncodedNameAsBytes(),
info[i + 1].getTable().getNamespace()));
nextSize = nextRegion.getLargestHStoreSize(); nextSize = nextRegion.getLargestHStoreSize();
if ((currentSize + nextSize) <= (maxFilesize / 2)) { if ((currentSize + nextSize) <= (maxFilesize / 2)) {

View File

@ -95,7 +95,9 @@ public class MetaUtils {
this.walFactory = new WALFactory(walConf, null, logName); this.walFactory = new WALFactory(walConf, null, logName);
} }
final byte[] region = info.getEncodedNameAsBytes(); final byte[] region = info.getEncodedNameAsBytes();
return info.isMetaRegion() ? walFactory.getMetaWAL(region) : walFactory.getWAL(region); final byte[] namespace = info.getTable().getNamespace();
return info.isMetaRegion() ? walFactory.getMetaWAL(region) : walFactory.getWAL(region,
namespace);
} }
/** /**

View File

@ -42,7 +42,7 @@ public class BoundedGroupingStrategy implements RegionGroupingStrategy{
private String[] groupNames; private String[] groupNames;
@Override @Override
public String group(byte[] identifier) { public String group(byte[] identifier, byte[] namespace) {
String idStr = Bytes.toString(identifier); String idStr = Bytes.toString(identifier);
String groupName = groupNameCache.get(idStr); String groupName = groupNameCache.get(idStr);
if (null == groupName) { if (null == groupName) {

View File

@ -118,7 +118,7 @@ public class DefaultWALProvider implements WALProvider {
} }
@Override @Override
public WAL getWAL(final byte[] identifier) throws IOException { public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException {
if (log == null) { if (log == null) {
// only lock when need to create wal, and need to lock since // only lock when need to create wal, and need to lock since
// creating hlog on fs is time consuming // creating hlog on fs is time consuming

View File

@ -67,7 +67,7 @@ class DisabledWALProvider implements WALProvider {
} }
@Override @Override
public WAL getWAL(final byte[] identifier) throws IOException { public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException {
return disabled; return disabled;
} }

View File

@ -0,0 +1,52 @@
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.wal;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider.RegionGroupingStrategy;
/**
* A WAL grouping strategy based on namespace.
* Notice: the wal-group mapping might change if we support dynamic namespace updating later,
* and special attention needed if we support feature like group-based replication.
*/
@InterfaceAudience.Private
public class NamespaceGroupingStrategy implements RegionGroupingStrategy {
String providerId;
@Override
public String group(byte[] identifier, byte[] namespace) {
String namespaceString;
if (namespace == null || namespace.length == 0) {
namespaceString = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
} else {
namespaceString = Bytes.toString(namespace);
}
return providerId + GROUP_NAME_DELIMITER + namespaceString;
}
@Override
public void init(Configuration config, String providerId) {
this.providerId = providerId;
}
}

View File

@ -65,12 +65,11 @@ class RegionGroupingProvider implements WALProvider {
*/ */
public static interface RegionGroupingStrategy { public static interface RegionGroupingStrategy {
String GROUP_NAME_DELIMITER = "."; String GROUP_NAME_DELIMITER = ".";
/** /**
* Given an identifier, pick a group. * Given an identifier and a namespace, pick a group.
* the byte[] returned for a given group must always use the same instance, since we
* will be using it as a hash key.
*/ */
String group(final byte[] identifier); String group(final byte[] identifier, byte[] namespace);
void init(Configuration config, String providerId); void init(Configuration config, String providerId);
} }
@ -80,7 +79,8 @@ class RegionGroupingProvider implements WALProvider {
static enum Strategies { static enum Strategies {
defaultStrategy(BoundedGroupingStrategy.class), defaultStrategy(BoundedGroupingStrategy.class),
identity(IdentityGroupingStrategy.class), identity(IdentityGroupingStrategy.class),
bounded(BoundedGroupingStrategy.class); bounded(BoundedGroupingStrategy.class),
namespace(NamespaceGroupingStrategy.class);
final Class<? extends RegionGroupingStrategy> clazz; final Class<? extends RegionGroupingStrategy> clazz;
Strategies(Class<? extends RegionGroupingStrategy> clazz) { Strategies(Class<? extends RegionGroupingStrategy> clazz) {
@ -200,12 +200,12 @@ class RegionGroupingProvider implements WALProvider {
} }
@Override @Override
public WAL getWAL(final byte[] identifier) throws IOException { public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException {
final String group; final String group;
if (META_WAL_PROVIDER_ID.equals(this.providerId)) { if (META_WAL_PROVIDER_ID.equals(this.providerId)) {
group = META_WAL_GROUP_NAME; group = META_WAL_GROUP_NAME;
} else { } else {
group = strategy.group(identifier); group = strategy.group(identifier, namespace);
} }
return getWAL(group); return getWAL(group);
} }
@ -254,7 +254,7 @@ class RegionGroupingProvider implements WALProvider {
@Override @Override
public void init(Configuration config, String providerId) {} public void init(Configuration config, String providerId) {}
@Override @Override
public String group(final byte[] identifier) { public String group(final byte[] identifier, final byte[] namespace) {
return Bytes.toString(identifier); return Bytes.toString(identifier);
} }
} }

View File

@ -224,9 +224,10 @@ public class WALFactory {
/** /**
* @param identifier may not be null, contents will not be altered * @param identifier may not be null, contents will not be altered
* @param namespace could be null, and will use default namespace if null
*/ */
public WAL getWAL(final byte[] identifier) throws IOException { public WAL getWAL(final byte[] identifier, final byte[] namespace) throws IOException {
return provider.getWAL(identifier); return provider.getWAL(identifier, namespace);
} }
/** /**
@ -246,7 +247,7 @@ public class WALFactory {
metaProvider = this.metaProvider.get(); metaProvider = this.metaProvider.get();
} }
} }
return metaProvider.getWAL(identifier); return metaProvider.getWAL(identifier, null);
} }
public Reader createReader(final FileSystem fs, final Path path) throws IOException { public Reader createReader(final FileSystem fs, final Path path) throws IOException {

View File

@ -53,9 +53,10 @@ public interface WALProvider {
/** /**
* @param identifier may not be null. contents will not be altered. * @param identifier may not be null. contents will not be altered.
* @param namespace could be null, and will use default namespace if null
* @return a WAL for writing entries for the given region. * @return a WAL for writing entries for the given region.
*/ */
WAL getWAL(final byte[] identifier) throws IOException; WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException;
/** /**
* persist outstanding WALs to storage and stop accepting new appends. * persist outstanding WALs to storage and stop accepting new appends.

View File

@ -2569,7 +2569,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
return (new WALFactory(confForWAL, return (new WALFactory(confForWAL,
Collections.<WALActionsListener>singletonList(new MetricsWAL()), Collections.<WALActionsListener>singletonList(new MetricsWAL()),
"hregion-" + RandomStringUtils.randomNumeric(8))). "hregion-" + RandomStringUtils.randomNumeric(8))).
getWAL(hri.getEncodedNameAsBytes()); getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
} }
/** /**

View File

@ -164,7 +164,7 @@ public class TestWALObserver {
*/ */
@Test @Test
public void testWALObserverWriteToWAL() throws Exception { public void testWALObserverWriteToWAL() throws Exception {
final WAL log = wals.getWAL(UNSPECIFIED_REGION); final WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.class), false); verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.class), false);
} }
@ -175,7 +175,7 @@ public class TestWALObserver {
*/ */
@Test @Test
public void testLegacyWALObserverWriteToWAL() throws Exception { public void testLegacyWALObserverWriteToWAL() throws Exception {
final WAL log = wals.getWAL(UNSPECIFIED_REGION); final WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.Legacy.class), true); verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.Legacy.class), true);
} }
@ -279,7 +279,7 @@ public class TestWALObserver {
final Configuration newConf = HBaseConfiguration.create(this.conf); final Configuration newConf = HBaseConfiguration.create(this.conf);
final WAL wal = wals.getWAL(UNSPECIFIED_REGION); final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class); final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class);
newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
final SampleRegionWALObserver oldApi = getCoprocessor(wal, final SampleRegionWALObserver oldApi = getCoprocessor(wal,
@ -351,7 +351,7 @@ public class TestWALObserver {
final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
final AtomicLong sequenceId = new AtomicLong(0); final AtomicLong sequenceId = new AtomicLong(0);
WAL log = wals.getWAL(UNSPECIFIED_REGION); WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
try { try {
SampleRegionWALObserver cp = getCoprocessor(log, SampleRegionWALObserver.class); SampleRegionWALObserver cp = getCoprocessor(log, SampleRegionWALObserver.class);
@ -396,7 +396,7 @@ public class TestWALObserver {
final Configuration newConf = HBaseConfiguration.create(this.conf); final Configuration newConf = HBaseConfiguration.create(this.conf);
// WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf); // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
WAL wal = wals.getWAL(UNSPECIFIED_REGION); WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
// Put p = creatPutWith2Families(TEST_ROW); // Put p = creatPutWith2Families(TEST_ROW);
WALEdit edit = new WALEdit(); WALEdit edit = new WALEdit();
long now = EnvironmentEdgeManager.currentTime(); long now = EnvironmentEdgeManager.currentTime();
@ -421,7 +421,7 @@ public class TestWALObserver {
FileSystem newFS = FileSystem.get(newConf); FileSystem newFS = FileSystem.get(newConf);
// Make a new wal for new region open. // Make a new wal for new region open.
final WALFactory wals2 = new WALFactory(conf, null, currentTest.getMethodName()+"2"); final WALFactory wals2 = new WALFactory(conf, null, currentTest.getMethodName()+"2");
WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION);; WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION, null);;
HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir,
hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null); hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
long seqid2 = region.getOpenSeqNum(); long seqid2 = region.getOpenSeqNum();
@ -449,7 +449,7 @@ public class TestWALObserver {
*/ */
@Test @Test
public void testWALObserverLoaded() throws Exception { public void testWALObserverLoaded() throws Exception {
WAL log = wals.getWAL(UNSPECIFIED_REGION); WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
assertNotNull(getCoprocessor(log, SampleRegionWALObserver.class)); assertNotNull(getCoprocessor(log, SampleRegionWALObserver.class));
} }

View File

@ -119,7 +119,7 @@ public class TestWALRecordReader {
@Test @Test
public void testPartialRead() throws Exception { public void testPartialRead() throws Exception {
final WALFactory walfactory = new WALFactory(conf, null, getName()); final WALFactory walfactory = new WALFactory(conf, null, getName());
WAL log = walfactory.getWAL(info.getEncodedNameAsBytes()); WAL log = walfactory.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
// This test depends on timestamp being millisecond based and the filename of the WAL also // This test depends on timestamp being millisecond based and the filename of the WAL also
// being millisecond based. // being millisecond based.
long ts = System.currentTimeMillis(); long ts = System.currentTimeMillis();
@ -182,7 +182,7 @@ public class TestWALRecordReader {
@Test @Test
public void testWALRecordReader() throws Exception { public void testWALRecordReader() throws Exception {
final WALFactory walfactory = new WALFactory(conf, null, getName()); final WALFactory walfactory = new WALFactory(conf, null, getName());
WAL log = walfactory.getWAL(info.getEncodedNameAsBytes()); WAL log = walfactory.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
byte [] value = Bytes.toBytes("value"); byte [] value = Bytes.toBytes("value");
final AtomicLong sequenceId = new AtomicLong(0); final AtomicLong sequenceId = new AtomicLong(0);
WALEdit edit = new WALEdit(); WALEdit edit = new WALEdit();

View File

@ -174,7 +174,7 @@ public class TestCacheOnWriteInSchema {
walFactory = new WALFactory(conf, null, id); walFactory = new WALFactory(conf, null, id);
region = TEST_UTIL.createLocalHRegion(info, htd, region = TEST_UTIL.createLocalHRegion(info, htd,
walFactory.getWAL(info.getEncodedNameAsBytes())); walFactory.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()));
store = new HStore(region, hcd, conf); store = new HStore(region, hcd, conf);
} }

View File

@ -100,8 +100,8 @@ public class TestDefaultCompactSelection extends TestCase {
region = HBaseTestingUtility.createRegionAndWAL(info, basedir, conf, htd); region = HBaseTestingUtility.createRegionAndWAL(info, basedir, conf, htd);
HBaseTestingUtility.closeRegionAndWAL(region); HBaseTestingUtility.closeRegionAndWAL(region);
Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes()), fs, conf, info, htd, region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(), info.getTable()
null); .getNamespace()), fs, conf, info, htd, null);
store = new HStore(region, hcd, conf); store = new HStore(region, hcd, conf);

View File

@ -962,7 +962,7 @@ public class TestDefaultMemStore extends TestCase {
desc.addFamily(new HColumnDescriptor("foo".getBytes())); desc.addFamily(new HColumnDescriptor("foo".getBytes()));
HRegion r = HRegion r =
HRegion.createHRegion(hri, testDir, conf, desc, HRegion.createHRegion(hri, testDir, conf, desc,
wFactory.getWAL(hri.getEncodedNameAsBytes())); wFactory.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()));
HRegion.addRegionToMETA(meta, r); HRegion.addRegionToMETA(meta, r);
edge.setCurrentTimeMillis(1234 + 100); edge.setCurrentTimeMillis(1234 + 100);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();

View File

@ -160,8 +160,8 @@ public class TestHMobStore {
final Configuration walConf = new Configuration(conf); final Configuration walConf = new Configuration(conf);
FSUtils.setRootDir(walConf, basedir); FSUtils.setRootDir(walConf, basedir);
final WALFactory wals = new WALFactory(walConf, null, methodName); final WALFactory wals = new WALFactory(walConf, null, methodName);
region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes()), fs, conf, region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(),
info, htd, null); info.getTable().getNamespace()), fs, conf, info, htd, null);
store = new HMobStore(region, hcd, conf); store = new HMobStore(region, hcd, conf);
if(testStore) { if(testStore) {
init(conf, hcd); init(conf, hcd);

View File

@ -362,7 +362,7 @@ public class TestHRegion {
FSUtils.setRootDir(walConf, logDir); FSUtils.setRootDir(walConf, logDir);
return (new WALFactory(walConf, return (new WALFactory(walConf,
Collections.<WALActionsListener>singletonList(new MetricsWAL()), callingMethod)) Collections.<WALActionsListener>singletonList(new MetricsWAL()), callingMethod))
.getWAL(tableName.toBytes()); .getWAL(tableName.toBytes(), tableName.getNamespace());
} }
/** /**
@ -934,7 +934,7 @@ public class TestHRegion {
final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
FSUtils.setRootDir(walConf, logDir); FSUtils.setRootDir(walConf, logDir);
final WALFactory wals = new WALFactory(walConf, null, method); final WALFactory wals = new WALFactory(walConf, null, method);
final WAL wal = wals.getWAL(tableName.getName()); final WAL wal = wals.getWAL(tableName.getName(), tableName.getNamespace());
this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW,
HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family); HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
@ -4762,7 +4762,7 @@ public class TestHRegion {
final Configuration walConf = new Configuration(conf); final Configuration walConf = new Configuration(conf);
FSUtils.setRootDir(walConf, logDir); FSUtils.setRootDir(walConf, logDir);
final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString()); final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
final WAL wal = spy(wals.getWAL(tableName.getName())); final WAL wal = spy(wals.getWAL(tableName.getName(), tableName.getNamespace()));
this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW,
HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal, HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
new byte[][] { family }); new byte[][] { family });

View File

@ -158,8 +158,10 @@ public class TestHRegionReplayEvents {
false, time, 1); false, time, 1);
wals = TestHRegion.createWALFactory(CONF, rootDir); wals = TestHRegion.createWALFactory(CONF, rootDir);
walPrimary = wals.getWAL(primaryHri.getEncodedNameAsBytes()); walPrimary = wals.getWAL(primaryHri.getEncodedNameAsBytes(),
walSecondary = wals.getWAL(secondaryHri.getEncodedNameAsBytes()); primaryHri.getTable().getNamespace());
walSecondary = wals.getWAL(secondaryHri.getEncodedNameAsBytes(),
secondaryHri.getTable().getNamespace());
rss = mock(RegionServerServices.class); rss = mock(RegionServerServices.class);
when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1)); when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1));

View File

@ -415,8 +415,9 @@ public class TestRegionMergeTransaction {
HRegion a = HBaseTestingUtility.createRegionAndWAL(hri, testdir, HRegion a = HBaseTestingUtility.createRegionAndWAL(hri, testdir,
TEST_UTIL.getConfiguration(), htd); TEST_UTIL.getConfiguration(), htd);
HBaseTestingUtility.closeRegionAndWAL(a); HBaseTestingUtility.closeRegionAndWAL(a);
return HRegion.openHRegion(testdir, hri, htd, wals.getWAL(hri.getEncodedNameAsBytes()), return HRegion.openHRegion(testdir, hri, htd,
TEST_UTIL.getConfiguration()); wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()),
TEST_UTIL.getConfiguration());
} }
private int countRows(final HRegion r) throws IOException { private int countRows(final HRegion r) throws IOException {

View File

@ -380,7 +380,8 @@ public class TestSplitTransaction {
HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, testdir, TEST_UTIL.getConfiguration(), HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, testdir, TEST_UTIL.getConfiguration(),
htd); htd);
HBaseTestingUtility.closeRegionAndWAL(r); HBaseTestingUtility.closeRegionAndWAL(r);
return HRegion.openHRegion(testdir, hri, htd, wals.getWAL(hri.getEncodedNameAsBytes()), return HRegion.openHRegion(testdir, hri, htd,
wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()),
TEST_UTIL.getConfiguration()); TEST_UTIL.getConfiguration());
} }

View File

@ -186,8 +186,8 @@ public class TestStore {
final Configuration walConf = new Configuration(conf); final Configuration walConf = new Configuration(conf);
FSUtils.setRootDir(walConf, basedir); FSUtils.setRootDir(walConf, basedir);
final WALFactory wals = new WALFactory(walConf, null, methodName); final WALFactory wals = new WALFactory(walConf, null, methodName);
HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes()), fs, conf, HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(),
info, htd, null); info.getTable().getNamespace()), fs, conf, info, htd, null);
store = new HStore(region, hcd, conf); store = new HStore(region, hcd, conf);
return store; return store;

View File

@ -106,7 +106,9 @@ public class TestStoreFileRefresherChore {
final Configuration walConf = new Configuration(conf); final Configuration walConf = new Configuration(conf);
FSUtils.setRootDir(walConf, tableDir); FSUtils.setRootDir(walConf, tableDir);
final WALFactory wals = new WALFactory(walConf, null, "log_" + replicaId); final WALFactory wals = new WALFactory(walConf, null, "log_" + replicaId);
HRegion region = new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes()), conf, htd, null); HRegion region =
new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()),
conf, htd, null);
region.initialize(); region.initialize();

View File

@ -85,7 +85,7 @@ public class TestDurability {
public void testDurability() throws Exception { public void testDurability() throws Exception {
final WALFactory wals = new WALFactory(CONF, null, "TestDurability"); final WALFactory wals = new WALFactory(CONF, null, "TestDurability");
byte[] tableName = Bytes.toBytes("TestDurability"); byte[] tableName = Bytes.toBytes("TestDurability");
final WAL wal = wals.getWAL(tableName); final WAL wal = wals.getWAL(tableName, null);
HRegion region = createHRegion(tableName, "region", wal, Durability.USE_DEFAULT); HRegion region = createHRegion(tableName, "region", wal, Durability.USE_DEFAULT);
HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, Durability.ASYNC_WAL); HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, Durability.ASYNC_WAL);
@ -148,7 +148,7 @@ public class TestDurability {
// Setting up region // Setting up region
final WALFactory wals = new WALFactory(CONF, null, "TestIncrement"); final WALFactory wals = new WALFactory(CONF, null, "TestIncrement");
byte[] tableName = Bytes.toBytes("TestIncrement"); byte[] tableName = Bytes.toBytes("TestIncrement");
final WAL wal = wals.getWAL(tableName); final WAL wal = wals.getWAL(tableName, null);
HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT); HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT);
// col1: amount = 1, 1 write back to WAL // col1: amount = 1, 1 write back to WAL
@ -206,7 +206,7 @@ public class TestDurability {
// Setting up region // Setting up region
final WALFactory wals = new WALFactory(CONF, null, "testIncrementWithReturnResultsSetToFalse"); final WALFactory wals = new WALFactory(CONF, null, "testIncrementWithReturnResultsSetToFalse");
byte[] tableName = Bytes.toBytes("testIncrementWithReturnResultsSetToFalse"); byte[] tableName = Bytes.toBytes("testIncrementWithReturnResultsSetToFalse");
final WAL wal = wals.getWAL(tableName); final WAL wal = wals.getWAL(tableName, null);
HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT); HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT);
Increment inc1 = new Increment(row1); Increment inc1 = new Increment(row1);

View File

@ -190,7 +190,8 @@ public class TestLogRollAbort {
TableName.valueOf(this.getClass().getName()); TableName.valueOf(this.getClass().getName());
HRegionInfo regioninfo = new HRegionInfo(tableName, HRegionInfo regioninfo = new HRegionInfo(tableName,
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes()); final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes(),
regioninfo.getTable().getNamespace());
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);

View File

@ -199,7 +199,7 @@ public class TestLogRolling {
final Configuration conf = TEST_UTIL.getConfiguration(); final Configuration conf = TEST_UTIL.getConfiguration();
final WALFactory wals = new WALFactory(conf, null, final WALFactory wals = new WALFactory(conf, null,
ServerName.valueOf("test.com",8080, 1).toString()); ServerName.valueOf("test.com",8080, 1).toString());
final WAL newLog = wals.getWAL(new byte[]{}); final WAL newLog = wals.getWAL(new byte[]{}, null);
try { try {
// Now roll the log before we write anything. // Now roll the log before we write anything.
newLog.rollWriter(true); newLog.rollWriter(true);

View File

@ -65,7 +65,7 @@ public class TestLogRollingNoCluster {
final Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); final Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
FSUtils.setRootDir(conf, dir); FSUtils.setRootDir(conf, dir);
final WALFactory wals = new WALFactory(conf, null, TestLogRollingNoCluster.class.getName()); final WALFactory wals = new WALFactory(conf, null, TestLogRollingNoCluster.class.getName());
final WAL wal = wals.getWAL(new byte[]{}); final WAL wal = wals.getWAL(new byte[]{}, null);
Appender [] appenders = null; Appender [] appenders = null;

View File

@ -91,7 +91,7 @@ public class TestWALActionsListener {
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);
HRegionInfo hri = new HRegionInfo(TableName.valueOf(SOME_BYTES), HRegionInfo hri = new HRegionInfo(TableName.valueOf(SOME_BYTES),
SOME_BYTES, SOME_BYTES, false); SOME_BYTES, SOME_BYTES, false);
final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes()); final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
for (int i = 0; i < 20; i++) { for (int i = 0; i < 20; i++) {
byte[] b = Bytes.toBytes(i+""); byte[] b = Bytes.toBytes(i+"");

View File

@ -201,7 +201,7 @@ public class TestReplicationSourceManager {
listeners.add(replication); listeners.add(replication);
final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners, final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners,
URLEncoder.encode("regionserver:60020", "UTF8")); URLEncoder.encode("regionserver:60020", "UTF8"));
final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes()); final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);
manager.init(); manager.init();
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("tableame")); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("tableame"));

View File

@ -131,7 +131,7 @@ public class TestReplicationWALReaderManager {
pathWatcher = new PathWatcher(); pathWatcher = new PathWatcher();
listeners.add(pathWatcher); listeners.add(pathWatcher);
final WALFactory wals = new WALFactory(conf, listeners, tn.getMethodName()); final WALFactory wals = new WALFactory(conf, listeners, tn.getMethodName());
log = wals.getWAL(info.getEncodedNameAsBytes()); log = wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
} }
@After @After

View File

@ -275,7 +275,7 @@ public class TestMergeTool extends HBaseTestCase {
// Close the region and delete the log // Close the region and delete the log
HBaseTestingUtility.closeRegionAndWAL(regions[i]); HBaseTestingUtility.closeRegionAndWAL(regions[i]);
} }
WAL log = wals.getWAL(new byte[]{}); WAL log = wals.getWAL(new byte[]{}, null);
// Merge Region 0 and Region 1 // Merge Region 0 and Region 1
HRegion merged = mergeAndVerify("merging regions 0 and 1 ", HRegion merged = mergeAndVerify("merging regions 0 and 1 ",
this.sourceRegions[0].getRegionNameAsString(), this.sourceRegions[0].getRegionNameAsString(),

View File

@ -108,7 +108,7 @@ public class IOTestProvider implements WALProvider {
} }
@Override @Override
public WAL getWAL(final byte[] identifier) throws IOException { public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException {
return log; return log;
} }

View File

@ -172,7 +172,7 @@ public class TestBoundedRegionGroupingStrategy {
int count = 0; int count = 0;
// we know that this should see one of the wals more than once // we know that this should see one of the wals more than once
for (int i = 0; i < temp*8; i++) { for (int i = 0; i < temp*8; i++) {
final WAL maybeNewWAL = wals.getWAL(Bytes.toBytes(random.nextInt())); final WAL maybeNewWAL = wals.getWAL(Bytes.toBytes(random.nextInt()), null);
LOG.info("Iteration " + i + ", checking wal " + maybeNewWAL); LOG.info("Iteration " + i + ", checking wal " + maybeNewWAL);
if (seen.add(maybeNewWAL)) { if (seen.add(maybeNewWAL)) {
count++; count++;

View File

@ -198,7 +198,7 @@ public class TestDefaultWALProvider {
HRegionInfo hri2 = new HRegionInfo(htd2.getTableName(), HRegionInfo hri2 = new HRegionInfo(htd2.getTableName(),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
// we want to mix edits from regions, so pick our own identifier. // we want to mix edits from regions, so pick our own identifier.
final WAL log = wals.getWAL(UNSPECIFIED_REGION); final WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
// Add a single edit and make sure that rolling won't remove the file // Add a single edit and make sure that rolling won't remove the file
// Before HBASE-3198 it used to delete it // Before HBASE-3198 it used to delete it
@ -265,7 +265,7 @@ public class TestDefaultWALProvider {
localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName()); localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName()); final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
try { try {
final WAL wal = wals.getWAL(UNSPECIFIED_REGION); final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal)); assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
HRegionInfo hri1 = HRegionInfo hri1 =
new HRegionInfo(table1.getTableName(), HConstants.EMPTY_START_ROW, new HRegionInfo(table1.getTableName(), HConstants.EMPTY_START_ROW,
@ -351,10 +351,11 @@ public class TestDefaultWALProvider {
final Set<WAL> seen = new HashSet<WAL>(1); final Set<WAL> seen = new HashSet<WAL>(1);
final Random random = new Random(); final Random random = new Random();
assertTrue("first attempt to add WAL from default provider should work.", assertTrue("first attempt to add WAL from default provider should work.",
seen.add(wals.getWAL(Bytes.toBytes(random.nextInt())))); seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()), null)));
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
assertFalse("default wal provider is only supposed to return a single wal, which should " + assertFalse("default wal provider is only supposed to return a single wal, which should "
"compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt())))); + "compare as .equals itself.",
seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()), null)));
} }
} finally { } finally {
wals.close(); wals.close();

View File

@ -91,7 +91,8 @@ public class TestSecureWAL {
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);
// Write the WAL // Write the WAL
final WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes()); final WAL wal =
wals.getWAL(regioninfo.getEncodedNameAsBytes(), regioninfo.getTable().getNamespace());
for (int i = 0; i < total; i++) { for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit(); WALEdit kvs = new WALEdit();

View File

@ -185,7 +185,8 @@ public class TestWALFactory {
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);
for (int ii = 0; ii < howmany; ii++) { for (int ii = 0; ii < howmany; ii++) {
for (int i = 0; i < howmany; i++) { for (int i = 0; i < howmany; i++) {
final WAL log = wals.getWAL(infos[i].getEncodedNameAsBytes()); final WAL log =
wals.getWAL(infos[i].getEncodedNameAsBytes(), infos[i].getTable().getNamespace());
for (int j = 0; j < howmany; j++) { for (int j = 0; j < howmany; j++) {
WALEdit edit = new WALEdit(); WALEdit edit = new WALEdit();
byte [] family = Bytes.toBytes("column"); byte [] family = Bytes.toBytes("column");
@ -246,7 +247,7 @@ public class TestWALFactory {
null,null, false); null,null, false);
HTableDescriptor htd = new HTableDescriptor(tableName); HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor(tableName.getName())); htd.addFamily(new HColumnDescriptor(tableName.getName()));
final WAL wal = wals.getWAL(info.getEncodedNameAsBytes()); final WAL wal = wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
for (int i = 0; i < total; i++) { for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit(); WALEdit kvs = new WALEdit();
@ -361,7 +362,8 @@ public class TestWALFactory {
HRegionInfo regioninfo = new HRegionInfo(tableName, HRegionInfo regioninfo = new HRegionInfo(tableName,
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false); HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
final WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes()); final WAL wal =
wals.getWAL(regioninfo.getEncodedNameAsBytes(), regioninfo.getTable().getNamespace());
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);
final int total = 20; final int total = 20;
@ -498,7 +500,7 @@ public class TestWALFactory {
} }
HRegionInfo info = new HRegionInfo(htd.getTableName(), HRegionInfo info = new HRegionInfo(htd.getTableName(),
row,Bytes.toBytes(Bytes.toString(row) + "1"), false); row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
final WAL log = wals.getWAL(info.getEncodedNameAsBytes()); final WAL log = wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
final long txid = log.append(htd, info, final long txid = log.append(htd, info,
new WALKey(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis()), new WALKey(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis()),
@ -556,7 +558,7 @@ public class TestWALFactory {
} }
HRegionInfo hri = new HRegionInfo(htd.getTableName(), HRegionInfo hri = new HRegionInfo(htd.getTableName(),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
final WAL log = wals.getWAL(hri.getEncodedNameAsBytes()); final WAL log = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
final long txid = log.append(htd, hri, final long txid = log.append(htd, hri,
new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis()), new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis()),
cols, sequenceId, true, null); cols, sequenceId, true, null);
@ -605,7 +607,7 @@ public class TestWALFactory {
HRegionInfo hri = new HRegionInfo(tableName, HRegionInfo hri = new HRegionInfo(tableName,
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
final WAL log = wals.getWAL(hri.getEncodedNameAsBytes()); final WAL log = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
log.registerWALActionsListener(visitor); log.registerWALActionsListener(visitor);
for (int i = 0; i < COL_COUNT; i++) { for (int i = 0; i < COL_COUNT; i++) {
WALEdit cols = new WALEdit(); WALEdit cols = new WALEdit();
@ -634,7 +636,7 @@ public class TestWALFactory {
@Test @Test
public void testWALCoprocessorLoaded() throws Exception { public void testWALCoprocessorLoaded() throws Exception {
// test to see whether the coprocessor is loaded or not. // test to see whether the coprocessor is loaded or not.
WALCoprocessorHost host = wals.getWAL(UNSPECIFIED_REGION).getCoprocessorHost(); WALCoprocessorHost host = wals.getWAL(UNSPECIFIED_REGION, null).getCoprocessorHost();
Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName()); Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName());
assertNotNull(c); assertNotNull(c);
} }

View File

@ -80,7 +80,7 @@ public class TestWALMethods {
final Configuration walConf = new Configuration(util.getConfiguration()); final Configuration walConf = new Configuration(util.getConfiguration());
FSUtils.setRootDir(walConf, regiondir); FSUtils.setRootDir(walConf, regiondir);
(new WALFactory(walConf, null, "dummyLogName")).getWAL(new byte[]{}); (new WALFactory(walConf, null, "dummyLogName")).getWAL(new byte[] {}, null);
NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir); NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir);
assertEquals(7, files.size()); assertEquals(7, files.size());

View File

@ -109,7 +109,8 @@ public class TestWALReaderOnSecureWAL {
final AtomicLong sequenceId = new AtomicLong(1); final AtomicLong sequenceId = new AtomicLong(1);
// Write the WAL // Write the WAL
WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes()); WAL wal =
wals.getWAL(regioninfo.getEncodedNameAsBytes(), regioninfo.getTable().getNamespace());
for (int i = 0; i < total; i++) { for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit(); WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value)); kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));

View File

@ -1073,7 +1073,7 @@ public class TestWALSplit {
REGIONS.add(regionName); REGIONS.add(regionName);
generateWALs(-1); generateWALs(-1);
wals.getWAL(Bytes.toBytes(regionName)); wals.getWAL(Bytes.toBytes(regionName), null);
FileStatus[] logfiles = fs.listStatus(WALDIR); FileStatus[] logfiles = fs.listStatus(WALDIR);
assertTrue("There should be some log file", assertTrue("There should be some log file",
logfiles != null && logfiles.length > 0); logfiles != null && logfiles.length > 0);

View File

@ -485,7 +485,8 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
// Initialize HRegion // Initialize HRegion
HRegionInfo regionInfo = new HRegionInfo(htd.getTableName()); HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
// Initialize WAL // Initialize WAL
final WAL wal = wals.getWAL(regionInfo.getEncodedNameAsBytes()); final WAL wal =
wals.getWAL(regionInfo.getEncodedNameAsBytes(), regionInfo.getTable().getNamespace());
// If we haven't already, attach a listener to this wal to handle rolls and metrics. // If we haven't already, attach a listener to this wal to handle rolls and metrics.
if (walsListenedTo.add(wal)) { if (walsListenedTo.add(wal)) {
roller.addWAL(wal); roller.addWAL(wal);