Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21247 Allow WAL Provider to be specified by configuration without explicit enum in Providers

This commit is contained in:
tedyu 2018-10-10 08:27:16 -07:00
parent fe579a1bb3
commit 0789f54454
4 changed files with 72 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
View File

@ -121,6 +121,8 @@ public class RegionGroupingProvider implements WALProvider {
/** delegate provider for WAL creation/roll/close */
public static final String DELEGATE_PROVIDER = "hbase.wal.regiongrouping.delegate.provider";
public static final String DELEGATE_PROVIDER_CLASS =
"hbase.wal.regiongrouping.delegate.provider.class";
public static final String DEFAULT_DELEGATE_PROVIDER = WALFactory.Providers.defaultProvider
.name();
@ -155,7 +157,8 @@ public class RegionGroupingProvider implements WALProvider {
}
this.providerId = sb.toString();
this.strategy = getStrategy(conf, REGION_GROUPING_STRATEGY, DEFAULT_REGION_GROUPING_STRATEGY);
this.providerClass = factory.getProviderClass(DELEGATE_PROVIDER, DEFAULT_DELEGATE_PROVIDER);
this.providerClass = factory.getProviderClass(DELEGATE_PROVIDER_CLASS, DELEGATE_PROVIDER,
DEFAULT_DELEGATE_PROVIDER);
}
private WALProvider createProvider(String group) throws IOException {

34
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
View File

@ -80,8 +80,11 @@ public class WALFactory {
public static final String WAL_PROVIDER = "hbase.wal.provider";
static final String DEFAULT_WAL_PROVIDER = Providers.defaultProvider.name();
public static final String WAL_PROVIDER_CLASS = "hbase.wal.provider.class";
static final Class<? extends WALProvider> DEFAULT_WAL_PROVIDER_CLASS = AsyncFSWALProvider.class;
public static final String META_WAL_PROVIDER = "hbase.wal.meta_provider";
public static final String META_WAL_PROVIDER_CLASS = "hbase.wal.meta_provider.class";
final String factoryId;
private final WALProvider provider;
@ -125,7 +128,25 @@ public class WALFactory {
}
@VisibleForTesting
public Class<? extends WALProvider> getProviderClass(String key, String defaultValue) {
/*
* @param clsKey config key for provider classname
* @param key config key for provider enum
* @param defaultValue default value for provider enum
* @return Class which extends WALProvider
*/
public Class<? extends WALProvider> getProviderClass(String clsKey, String key,
String defaultValue) {
String clsName = conf.get(clsKey);
if (clsName == null || clsName.isEmpty()) {
clsName = conf.get(key, defaultValue);
}
if (clsName != null && !clsName.isEmpty()) {
try {
return (Class<? extends WALProvider>) Class.forName(clsName);
} catch (ClassNotFoundException exception) {
// try with enum key next
}
}
try {
Providers provider = Providers.valueOf(conf.get(key, defaultValue));
@ -149,7 +170,7 @@ public class WALFactory {
// Fall back to them specifying a class name
// Note that the passed default class shouldn't actually be used, since the above only fails
// when there is a config value present.
return conf.getClass(key, Providers.defaultProvider.clazz, WALProvider.class);
return conf.getClass(key, AsyncFSWALProvider.class, WALProvider.class);
}
}
@ -196,7 +217,8 @@ public class WALFactory {
this.factoryId = factoryId;
// end required early initialization
if (conf.getBoolean("hbase.regionserver.hlog.enabled", true)) {
WALProvider provider = createProvider(getProviderClass(WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
WALProvider provider = createProvider(
getProviderClass(WAL_PROVIDER_CLASS, WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
if (enableSyncReplicationWALProvider) {
provider = new SyncReplicationWALProvider(provider);
}
@ -260,8 +282,10 @@ public class WALFactory {
if (provider != null) {
return provider;
}
provider = createProvider(getProviderClass(META_WAL_PROVIDER,
conf.get(WAL_PROVIDER, DEFAULT_WAL_PROVIDER)));
boolean metaWALProvPresent = conf.get(META_WAL_PROVIDER_CLASS) != null;
provider = createProvider(getProviderClass(
metaWALProvPresent ? META_WAL_PROVIDER_CLASS : WAL_PROVIDER_CLASS,
META_WAL_PROVIDER, conf.get(WAL_PROVIDER, DEFAULT_WAL_PROVIDER)));
provider.init(this, conf, AbstractFSWALProvider.META_WAL_PROVIDER_ID);
provider.addWALActionsListener(new MetricsWAL());
if (metaProvider.compareAndSet(null, provider)) {

5
hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
View File

@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -87,6 +89,7 @@ public class IOTestProvider implements WALProvider {
private volatile FSHLog log;
private String providerId;
protected AtomicBoolean initialized = new AtomicBoolean(false);
private List<WALActionsListener> listeners = new ArrayList<>();
/**
@ -97,7 +100,7 @@ public class IOTestProvider implements WALProvider {
*/
@Override
public void init(WALFactory factory, Configuration conf, String providerId) throws IOException {
if (factory != null) {
if (!initialized.compareAndSet(false, true)) {
throw new IllegalStateException("WALProvider.init should only be called once.");
}
this.factory = factory;

43
hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
View File

@ -680,6 +680,7 @@ public class TestWALFactory {
@Test
public void testWALProviders() throws IOException {
Configuration conf = new Configuration();
conf.set(HConstants.HBASE_DIR, TestWALFactory.conf.get(HConstants.HBASE_DIR));
// if providers are not set but enable SyncReplicationWALProvider by default for master node
// with not only system tables
WALFactory walFactory = new WALFactory(conf, this.currentServername.toString());
@ -696,28 +697,30 @@ public class TestWALFactory {
@Test
public void testOnlySetWALProvider() throws IOException {
Configuration conf = new Configuration();
conf.set(WAL_PROVIDER, WALFactory.Providers.multiwal.name());
conf.set(WAL_PROVIDER, RegionGroupingProvider.class.getName());
conf.set(HConstants.HBASE_DIR, TestWALFactory.conf.get(HConstants.HBASE_DIR));
WALFactory walFactory = new WALFactory(conf, this.currentServername.toString());
WALProvider wrappedWALProvider = ((SyncReplicationWALProvider) walFactory.getWALProvider())
.getWrappedProvider();
assertEquals(SyncReplicationWALProvider.class, walFactory.getWALProvider().getClass());
// class of WALProvider and metaWALProvider are the same when metaWALProvider is not set
assertEquals(WALFactory.Providers.multiwal.clazz, wrappedWALProvider.getClass());
assertEquals(WALFactory.Providers.multiwal.clazz, walFactory.getMetaProvider().getClass());
assertEquals(RegionGroupingProvider.class, wrappedWALProvider.getClass());
assertEquals(RegionGroupingProvider.class, walFactory.getMetaProvider().getClass());
}
@Test
public void testOnlySetMetaWALProvider() throws IOException {
Configuration conf = new Configuration();
conf.set(META_WAL_PROVIDER, WALFactory.Providers.asyncfs.name());
conf.set(META_WAL_PROVIDER, AsyncFSWALProvider.class.getName());
conf.set(HConstants.HBASE_DIR, TestWALFactory.conf.get(HConstants.HBASE_DIR));
WALFactory walFactory = new WALFactory(conf, this.currentServername.toString());
WALProvider wrappedWALProvider = ((SyncReplicationWALProvider) walFactory.getWALProvider())
.getWrappedProvider();
assertEquals(SyncReplicationWALProvider.class, walFactory.getWALProvider().getClass());
assertEquals(WALFactory.Providers.defaultProvider.clazz, wrappedWALProvider.getClass());
assertEquals(WALFactory.Providers.asyncfs.clazz, walFactory.getMetaProvider().getClass());
assertEquals(AsyncFSWALProvider.class, wrappedWALProvider.getClass());
assertEquals(AsyncFSWALProvider.class, walFactory.getMetaProvider().getClass());
}
@Test
@ -726,7 +729,7 @@ public class TestWALFactory {
// AsyncFSWal is the default, we should be able to request any WAL.
final WALFactory normalWalFactory = new WALFactory(conf, this.currentServername.toString());
Class<? extends WALProvider> fshLogProvider = normalWalFactory.getProviderClass(
WALFactory.WAL_PROVIDER, Providers.filesystem.name());
WALFactory.WAL_PROVIDER_CLASS, WALFactory.WAL_PROVIDER, Providers.filesystem.name());
assertEquals(Providers.filesystem.clazz, fshLogProvider);
// Imagine a world where MultiWAL is the default
@ -739,7 +742,31 @@ public class TestWALFactory {
};
// If we don't specify a WALProvider, we should get the default implementation.
Class<? extends WALProvider> multiwalProviderClass = customizedWalFactory.getProviderClass(
WALFactory.WAL_PROVIDER, Providers.multiwal.name());
WALFactory.WAL_PROVIDER_CLASS, WALFactory.WAL_PROVIDER, Providers.multiwal.name());
assertEquals(Providers.multiwal.clazz, multiwalProviderClass);
}
@Test
public void testCustomProvider() throws IOException {
final Configuration config = new Configuration();
config.set(WALFactory.WAL_PROVIDER_CLASS, IOTestProvider.class.getName());
final WALFactory walFactory = new WALFactory(config, this.currentServername.toString());
Class<? extends WALProvider> walProvider = walFactory.getProviderClass(
WALFactory.WAL_PROVIDER_CLASS, WALFactory.WAL_PROVIDER, Providers.filesystem.name());
assertEquals(IOTestProvider.class, walProvider);
WALProvider metaWALProvider = walFactory.getMetaProvider();
assertEquals(IOTestProvider.class, metaWALProvider.getClass());
}
@Test
public void testCustomMetaProvider() throws IOException {
final Configuration config = new Configuration();
config.set(WALFactory.META_WAL_PROVIDER_CLASS, IOTestProvider.class.getName());
final WALFactory walFactory = new WALFactory(config, this.currentServername.toString());
Class<? extends WALProvider> walProvider = walFactory.getProviderClass(
WALFactory.WAL_PROVIDER_CLASS, WALFactory.WAL_PROVIDER, Providers.filesystem.name());
assertEquals(Providers.filesystem.clazz, walProvider);
WALProvider metaWALProvider = walFactory.getMetaProvider();
assertEquals(IOTestProvider.class, metaWALProvider.getClass());
}
}