HBASE-9875 NamespaceJanitor chore is not used
This commit is contained in:
stack
parent
095a0e9aa2
commit
fd78e1c553
@ -170,7 +170,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
|
|||||||
|
|
||||||
/** Namespace stuff */
|
/** Namespace stuff */
|
||||||
private TableNamespaceManager tableNamespaceManager;
|
private TableNamespaceManager tableNamespaceManager;
|
||||||
private NamespaceJanitor namespaceJanitorChore;
|
|
||||||
|
|
||||||
// Metrics for the HMaster
|
// Metrics for the HMaster
|
||||||
final MetricsMaster metricsMaster;
|
final MetricsMaster metricsMaster;
|
||||||
@ -627,22 +626,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Useful for testing purpose also where we have
|
|
||||||
* master restart scenarios.
|
|
||||||
*/
|
|
||||||
protected void startCatalogJanitorChore() {
|
|
||||||
Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Useful for testing purpose also where we have
|
|
||||||
* master restart scenarios.
|
|
||||||
*/
|
|
||||||
protected void startNamespaceJanitorChore() {
|
|
||||||
Threads.setDaemonThreadRunning(namespaceJanitorChore.getThread());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a {@link ServerManager} instance.
|
* Create a {@link ServerManager} instance.
|
||||||
* @param master
|
* @param master
|
||||||
@ -907,9 +890,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
|
|||||||
if (this.clusterStatusPublisherChore != null){
|
if (this.clusterStatusPublisherChore != null){
|
||||||
clusterStatusPublisherChore.interrupt();
|
clusterStatusPublisherChore.interrupt();
|
||||||
}
|
}
|
||||||
if (this.namespaceJanitorChore != null){
|
|
||||||
namespaceJanitorChore.interrupt();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@ -1,149 +0,0 @@
|
|||||||
/**
|
|
||||||
*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hbase.master;
|
|
||||||
|
|
||||||
import com.google.common.collect.Sets;
|
|
||||||
import org.apache.commons.logging.Log;
|
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
import org.apache.hadoop.hbase.Chore;
|
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.util.FSUtils;
|
|
||||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
|
||||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
|
||||||
import org.apache.zookeeper.KeeperException;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A janitor for the namespace artifacts.
|
|
||||||
* Traverses hdfs and zk to remove orphaned directories/znodes
|
|
||||||
*/
|
|
||||||
@InterfaceAudience.Private
|
|
||||||
public class NamespaceJanitor extends Chore {
|
|
||||||
private static final Log LOG = LogFactory.getLog(NamespaceJanitor.class.getName());
|
|
||||||
private final MasterServices services;
|
|
||||||
private AtomicBoolean enabled = new AtomicBoolean(true);
|
|
||||||
|
|
||||||
public NamespaceJanitor(final MasterServices services) {
|
|
||||||
super("NamespaceJanitor-" + services.getServerName().toShortString(),
|
|
||||||
services.getConfiguration().getInt("hbase.namespacejanitor.interval", 300000),
|
|
||||||
services);
|
|
||||||
this.services = services;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected boolean initialChore() {
|
|
||||||
try {
|
|
||||||
if (this.enabled.get()) removeOrphans();
|
|
||||||
} catch (IOException e) {
|
|
||||||
LOG.warn("Failed NamespaceJanitor chore", e);
|
|
||||||
return false;
|
|
||||||
} catch (KeeperException e) {
|
|
||||||
LOG.warn("Failed NamespaceJanitor chore", e);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param enabled
|
|
||||||
*/
|
|
||||||
public boolean setEnabled(final boolean enabled) {
|
|
||||||
return this.enabled.getAndSet(enabled);
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean getEnabled() {
|
|
||||||
return this.enabled.get();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void chore() {
|
|
||||||
try {
|
|
||||||
if (this.enabled.get()) {
|
|
||||||
removeOrphans();
|
|
||||||
} else {
|
|
||||||
LOG.warn("NamepsaceJanitor disabled! Not running scan.");
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
LOG.warn("Failed NamespaceJanitor chore", e);
|
|
||||||
} catch (KeeperException e) {
|
|
||||||
LOG.warn("Failed NamespaceJanitor chore", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void removeOrphans() throws IOException, KeeperException {
|
|
||||||
//cache the info so we don't need to keep the master nsLock for long
|
|
||||||
//and not be wasteful with rpc calls
|
|
||||||
FileSystem fs = services.getMasterFileSystem().getFileSystem();
|
|
||||||
Set<String> descs = Sets.newHashSet();
|
|
||||||
for(NamespaceDescriptor ns : services.listNamespaceDescriptors()) {
|
|
||||||
descs.add(ns.getName());
|
|
||||||
}
|
|
||||||
|
|
||||||
//cleanup hdfs orphans
|
|
||||||
for (FileStatus nsStatus : FSUtils.listStatus(fs,
|
|
||||||
new Path(FSUtils.getRootDir(services.getConfiguration()), HConstants.BASE_NAMESPACE_DIR))) {
|
|
||||||
if (!descs.contains(nsStatus.getPath().getName()) &&
|
|
||||||
!NamespaceDescriptor.RESERVED_NAMESPACES.contains(nsStatus.getPath().getName())) {
|
|
||||||
boolean isEmpty = true;
|
|
||||||
for(FileStatus status : fs.listStatus(nsStatus.getPath())) {
|
|
||||||
if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
|
|
||||||
isEmpty = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(isEmpty) {
|
|
||||||
try {
|
|
||||||
if (!fs.delete(nsStatus.getPath(), true)) {
|
|
||||||
LOG.error("Failed to remove namespace directory: " + nsStatus.getPath());
|
|
||||||
}
|
|
||||||
} catch (IOException ex) {
|
|
||||||
LOG.error("Failed to remove namespace directory: " + nsStatus.getPath(),
|
|
||||||
ex);
|
|
||||||
}
|
|
||||||
LOG.debug("Removed namespace directory: "+nsStatus.getPath());
|
|
||||||
} else {
|
|
||||||
LOG.debug("Skipping non-empty namespace directory: " + nsStatus.getPath());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
String baseZnode = ZooKeeperWatcher.namespaceZNode;
|
|
||||||
for(String child : ZKUtil.listChildrenNoWatch(services.getZooKeeper(), baseZnode)) {
|
|
||||||
if (!descs.contains(child) &&
|
|
||||||
!NamespaceDescriptor.RESERVED_NAMESPACES.contains(child)) {
|
|
||||||
String znode = ZKUtil.joinZNode(baseZnode, child);
|
|
||||||
try {
|
|
||||||
ZKUtil.deleteNode(services.getZooKeeper(), znode);
|
|
||||||
LOG.debug("Removed namespace znode: " + znode);
|
|
||||||
} catch (KeeperException ex) {
|
|
||||||
LOG.debug("Failed to remove namespace znode: " + znode, ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1373,11 +1373,6 @@ public class TestSplitTransactionOnCluster {
|
|||||||
InterruptedException {
|
InterruptedException {
|
||||||
super(conf, cp);
|
super(conf, cp);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void startCatalogJanitorChore() {
|
|
||||||
LOG.debug("Customised master executed.");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class SplittingNodeCreationFailedException extends IOException {
|
private static class SplittingNodeCreationFailedException extends IOException {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user