HBASE-3360 ReplicationLogCleaner is enabled by default in 0.90 -- causes NPE
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1049680 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
1842a1ee85
commit
39330cff75
|
@ -782,6 +782,7 @@ Release 0.90.0 - Unreleased
|
||||||
HBASE-3356 Add more checks in replication if RS is stopped
|
HBASE-3356 Add more checks in replication if RS is stopped
|
||||||
HBASE-3358 Recovered replication queue wait on themselves when terminating
|
HBASE-3358 Recovered replication queue wait on themselves when terminating
|
||||||
HBASE-3359 LogRoller not added as a WAL listener when replication is enabled
|
HBASE-3359 LogRoller not added as a WAL listener when replication is enabled
|
||||||
|
HBASE-3360 ReplicationLogCleaner is enabled by default in 0.90 -- causes NPE
|
||||||
|
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
|
@ -361,6 +361,9 @@ public final class HConstants {
|
||||||
/** HBCK special code name used as server name when manipulating ZK nodes */
|
/** HBCK special code name used as server name when manipulating ZK nodes */
|
||||||
public static final String HBCK_CODE_NAME = "HBCKServerName";
|
public static final String HBCK_CODE_NAME = "HBCKServerName";
|
||||||
|
|
||||||
|
public static final String HBASE_MASTER_LOGCLEANER_PLUGINS =
|
||||||
|
"hbase.master.logcleaner.plugins";
|
||||||
|
|
||||||
private HConstants() {
|
private HConstants() {
|
||||||
// Can't be instantiated with this ctor.
|
// Can't be instantiated with this ctor.
|
||||||
}
|
}
|
||||||
|
|
|
@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
|
||||||
import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
|
import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
|
||||||
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
|
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
|
import org.apache.hadoop.hbase.replication.regionserver.Replication;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.InfoServer;
|
import org.apache.hadoop.hbase.util.InfoServer;
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
|
@ -203,6 +204,8 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
// set the thread name now we have an address
|
// set the thread name now we have an address
|
||||||
setName(MASTER + "-" + this.address);
|
setName(MASTER + "-" + this.address);
|
||||||
|
|
||||||
|
Replication.decorateMasterConfiguration(this.conf);
|
||||||
|
|
||||||
this.rpcServer.startThreads();
|
this.rpcServer.startThreads();
|
||||||
|
|
||||||
// Hack! Maps DFSClient => Master for logs. HDFS made this
|
// Hack! Maps DFSClient => Master for logs. HDFS made this
|
||||||
|
|
|
@ -34,6 +34,8 @@ import org.apache.hadoop.hbase.RemoteExceptionHandler;
|
||||||
import org.apache.hadoop.hbase.Stoppable;
|
import org.apache.hadoop.hbase.Stoppable;
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This Chore, everytime it runs, will clear the wal logs in the old logs folder
|
* This Chore, everytime it runs, will clear the wal logs in the old logs folder
|
||||||
* that are deletable for each log cleaner in the chain, in order to limit the
|
* that are deletable for each log cleaner in the chain, in order to limit the
|
||||||
|
@ -79,7 +81,7 @@ public class LogCleaner extends Chore {
|
||||||
* ReplicationLogCleaner and SnapshotLogCleaner.
|
* ReplicationLogCleaner and SnapshotLogCleaner.
|
||||||
*/
|
*/
|
||||||
private void initLogCleanersChain() {
|
private void initLogCleanersChain() {
|
||||||
String[] logCleaners = conf.getStrings("hbase.master.logcleaner.plugins");
|
String[] logCleaners = conf.getStrings(HBASE_MASTER_LOGCLEANER_PLUGINS);
|
||||||
if (logCleaners != null) {
|
if (logCleaners != null) {
|
||||||
for (String className : logCleaners) {
|
for (String className : logCleaners) {
|
||||||
LogCleanerDelegate logCleaner = newLogCleaner(className, conf);
|
LogCleanerDelegate logCleaner = newLogCleaner(className, conf);
|
||||||
|
|
|
@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.KeyValue;
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
|
@ -36,9 +35,14 @@ import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
|
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.WALObserver;
|
import org.apache.hadoop.hbase.regionserver.wal.WALObserver;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
|
import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
|
||||||
|
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
|
||||||
|
import static org.apache.hadoop.hbase.HConstants.REPLICATION_ENABLE_KEY;
|
||||||
|
import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_LOCAL;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gateway to Replication. Used by {@link org.apache.hadoop.hbase.regionserver.HRegionServer}.
|
* Gateway to Replication. Used by {@link org.apache.hadoop.hbase.regionserver.HRegionServer}.
|
||||||
*/
|
*/
|
||||||
|
@ -82,7 +86,7 @@ public class Replication implements WALObserver {
|
||||||
* @return True if replication is enabled.
|
* @return True if replication is enabled.
|
||||||
*/
|
*/
|
||||||
public static boolean isReplication(final Configuration c) {
|
public static boolean isReplication(final Configuration c) {
|
||||||
return c.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false);
|
return c.getBoolean(REPLICATION_ENABLE_KEY, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -134,7 +138,7 @@ public class Replication implements WALObserver {
|
||||||
for (KeyValue kv : logEdit.getKeyValues()) {
|
for (KeyValue kv : logEdit.getKeyValues()) {
|
||||||
family = kv.getFamily();
|
family = kv.getFamily();
|
||||||
int scope = info.getTableDesc().getFamily(family).getScope();
|
int scope = info.getTableDesc().getFamily(family).getScope();
|
||||||
if (scope != HConstants.REPLICATION_SCOPE_LOCAL &&
|
if (scope != REPLICATION_SCOPE_LOCAL &&
|
||||||
!scopes.containsKey(family)) {
|
!scopes.containsKey(family)) {
|
||||||
scopes.put(family, scope);
|
scopes.put(family, scope);
|
||||||
}
|
}
|
||||||
|
@ -149,6 +153,22 @@ public class Replication implements WALObserver {
|
||||||
getReplicationManager().logRolled(p);
|
getReplicationManager().logRolled(p);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method modifies the master's configuration in order to inject
|
||||||
|
* replication-related features
|
||||||
|
* @param conf
|
||||||
|
*/
|
||||||
|
public static void decorateMasterConfiguration(Configuration conf) {
|
||||||
|
if (!isReplication(conf)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String plugins = conf.get(HBASE_MASTER_LOGCLEANER_PLUGINS);
|
||||||
|
if (!plugins.contains(ReplicationLogCleaner.class.toString())) {
|
||||||
|
conf.set(HBASE_MASTER_LOGCLEANER_PLUGINS,
|
||||||
|
plugins + "," + ReplicationLogCleaner.class.getCanonicalName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void logRollRequested() {
|
public void logRollRequested() {
|
||||||
// Not interested
|
// Not interested
|
||||||
|
|
|
@ -290,7 +290,7 @@
|
||||||
</property>
|
</property>
|
||||||
<property>
|
<property>
|
||||||
<name>hbase.master.logcleaner.plugins</name>
|
<name>hbase.master.logcleaner.plugins</name>
|
||||||
<value>org.apache.hadoop.hbase.master.TimeToLiveLogCleaner,org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner</value>
|
<value>org.apache.hadoop.hbase.master.TimeToLiveLogCleaner</value>
|
||||||
<description>A comma-separated list of LogCleanerDelegate invoked by
|
<description>A comma-separated list of LogCleanerDelegate invoked by
|
||||||
the LogsCleaner service. These WAL/HLog cleaners are called in order,
|
the LogsCleaner service. These WAL/HLog cleaners are called in order,
|
||||||
so put the HLog cleaner that prunes the most HLog files in front. To
|
so put the HLog cleaner that prunes the most HLog files in front. To
|
||||||
|
|
Loading…
Reference in New Issue