HBASE-4255 Expose CatalogJanitor controls

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1364127 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2012-07-21 17:07:03 +00:00
parent 75e5903748
commit 4b07b3985d
15 changed files with 2821 additions and 56 deletions

View File

@ -26,6 +26,7 @@ ServerName metaLocation = null;
List<ServerName> servers = null;
Set<ServerName> deadServers = null;
boolean showAppendWarning = false;
boolean catalogJanitorEnabled = true;
String filter = "general";
String format = "html";
</%args>
@ -119,6 +120,12 @@ org.apache.hadoop.hbase.HBaseConfiguration;
for details.
</div>
</%if>
<%if !catalogJanitorEnabled %>
<div class="alert alert-error">
Please note that your cluster is running with the CatalogJanitor disabled. It can be
re-enabled from the hbase shell by running the command 'catalogjanitor_switch true'
</div>
</%if>
<h2>Region Servers</h2>

View File

@ -25,6 +25,8 @@ import org.apache.hadoop.hbase.security.TokenInfo;
import org.apache.hadoop.hbase.security.KerberosInfo;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnRequest;
@ -35,8 +37,12 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableR
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyColumnResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyTableRequest;
@ -303,4 +309,41 @@ public interface MasterAdminProtocol extends
@Override
public IsMasterRunningResponse isMasterRunning(RpcController c, IsMasterRunningRequest req)
throws ServiceException;
/**
* Run a scan of the catalog table
* @param c Unused (set to null).
* @param req CatalogScanRequest
* @return CatalogScanResponse that contains the int return code corresponding
* to the number of entries cleaned
* @throws ServiceException
*/
@Override
public CatalogScanResponse runCatalogScan(RpcController c,
CatalogScanRequest req) throws ServiceException;
/**
* Enable/Disable the catalog janitor
* @param c Unused (set to null).
* @param req EnableCatalogJanitorRequest that contains:<br>
* - enable: If true, enable catalog janitor. If false, disable janitor.<br>
* @return EnableCatalogJanitorResponse that contains:<br>
* - prevValue: true, if it was enabled previously; false, otherwise
* @throws ServiceException
*/
@Override
public EnableCatalogJanitorResponse enableCatalogJanitor(RpcController c,
EnableCatalogJanitorRequest req) throws ServiceException;
/**
* Query whether the catalog janitor is enabled
* @param c Unused (set to null).
* @param req IsCatalogJanitorEnabledRequest
* @return IsCatalogCatalogJanitorEnabledResponse that contains:<br>
* - value: true, if it is enabled; false, otherwise
* @throws ServiceException
*/
@Override
public IsCatalogJanitorEnabledResponse isCatalogJanitorEnabled(RpcController c,
IsCatalogJanitorEnabledRequest req) throws ServiceException;
}

View File

@ -1502,6 +1502,55 @@ public class HBaseAdmin implements Abortable, Closeable {
}
}
/**
* Enable/Disable the catalog janitor
* @param enable if true enables the catalog janitor
* @return the previous state
* @throws ServiceException
* @throws MasterNotRunningException
*/
public boolean enableCatalogJanitor(boolean enable)
throws ServiceException, MasterNotRunningException {
MasterAdminKeepAliveConnection master = connection.getKeepAliveMasterAdmin();
try {
return master.enableCatalogJanitor(null,
RequestConverter.buildEnableCatalogJanitorRequest(enable)).getPrevValue();
} finally {
master.close();
}
}
/**
* Ask for a scan of the catalog table
* @return the number of entries cleaned
* @throws ServiceException
* @throws MasterNotRunningException
*/
public int runCatalogScan() throws ServiceException, MasterNotRunningException {
MasterAdminKeepAliveConnection master = connection.getKeepAliveMasterAdmin();
try {
return master.runCatalogScan(null,
RequestConverter.buildCatalogScanRequest()).getScanResult();
} finally {
master.close();
}
}
/**
* Query on the catalog janitor state (Enabled/Disabled?)
* @throws ServiceException
* @throws MasterNotRunningException
*/
public boolean isCatalogJanitorEnabled() throws ServiceException, MasterNotRunningException {
MasterAdminKeepAliveConnection master = connection.getKeepAliveMasterAdmin();
try {
return master.isCatalogJanitorEnabled(null,
RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue();
} finally {
master.close();
}
}
/**
* Split a table or an individual region.
* Asynchronous operation.

View File

@ -25,6 +25,7 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@ -62,7 +63,8 @@ class CatalogJanitor extends Chore {
private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
private final Server server;
private final MasterServices services;
private boolean enabled = true;
private AtomicBoolean enabled = new AtomicBoolean(true);
private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
CatalogJanitor(final Server server, final MasterServices services) {
super(server.getServerName() + "-CatalogJanitor",
@ -75,7 +77,7 @@ class CatalogJanitor extends Chore {
@Override
protected boolean initialChore() {
try {
if (this.enabled) scan();
if (this.enabled.get()) scan();
} catch (IOException e) {
LOG.warn("Failed initial scan of catalog table", e);
return false;
@ -86,14 +88,22 @@ class CatalogJanitor extends Chore {
/**
* @param enabled
*/
public void setEnabled(final boolean enabled) {
this.enabled = enabled;
public boolean setEnabled(final boolean enabled) {
return this.enabled.getAndSet(enabled);
}
boolean getEnabled() {
return this.enabled.get();
}
@Override
protected void chore() {
try {
if (this.enabled.get()) {
scan();
} else {
LOG.warn("CatalogJanitor disabled! Not running scan.");
}
} catch (IOException e) {
LOG.warn("Failed scan of catalog table", e);
}
@ -135,6 +145,10 @@ class CatalogJanitor extends Chore {
* @throws IOException
*/
int scan() throws IOException {
try {
if (!alreadyRunning.compareAndSet(false, true)) {
return 0;
}
Pair<Integer, Map<HRegionInfo, Result>> pair = getSplitParents();
int count = pair.getFirst();
Map<HRegionInfo, Result> splitParents = pair.getSecond();
@ -159,6 +173,9 @@ class CatalogJanitor extends Chore {
" unreferenced parent region(s)");
}
return cleaned;
} finally {
alreadyRunning.set(false);
}
}
/**

View File

@ -81,6 +81,7 @@ import org.apache.hadoop.hbase.MasterAdminProtocol;
import org.apache.hadoop.hbase.RegionServerStatusProtocol;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.ipc.ProtocolSignature;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
@ -130,6 +131,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnReq
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AssignRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnRequest;
@ -138,8 +141,12 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableR
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetSchemaAlterStatusRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetSchemaAlterStatusResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsRequest;
@ -1156,6 +1163,30 @@ Server {
return IsMasterRunningResponse.newBuilder().setIsMasterRunning(isMasterRunning()).build();
}
@Override
public CatalogScanResponse runCatalogScan(RpcController c,
CatalogScanRequest req) throws ServiceException {
try {
return ResponseConverter.buildCatalogScanResponse(catalogJanitorChore.scan());
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
}
@Override
public EnableCatalogJanitorResponse enableCatalogJanitor(RpcController c,
EnableCatalogJanitorRequest req) throws ServiceException {
return EnableCatalogJanitorResponse.newBuilder().
setPrevValue(catalogJanitorChore.setEnabled(req.getEnable())).build();
}
@Override
public IsCatalogJanitorEnabledResponse isCatalogJanitorEnabled(RpcController c,
IsCatalogJanitorEnabledRequest req) throws ServiceException {
return IsCatalogJanitorEnabledResponse.newBuilder().
setValue(catalogJanitorChore.getEnabled()).build();
}
/**
* @return Maximum time we should run balancer for
*/

View File

@ -36,7 +36,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
import com.google.protobuf.ServiceException;
/**
* The servlet responsible for rendering the index page of the
@ -65,13 +67,20 @@ public class MasterStatusServlet extends HttpServlet {
Set<ServerName> deadServers = master.getServerManager().getDeadServers();
response.setContentType("text/html");
MasterStatusTmpl tmpl = new MasterStatusTmpl()
MasterStatusTmpl tmpl;
try {
tmpl = new MasterStatusTmpl()
.setFrags(frags)
.setShowAppendWarning(shouldShowAppendWarning(conf))
.setRootLocation(rootLocation)
.setMetaLocation(metaLocation)
.setServers(servers)
.setDeadServers(deadServers);
.setDeadServers(deadServers)
.setCatalogJanitorEnabled(master.isCatalogJanitorEnabled(null,
RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue());
} catch (ServiceException s) {
throw new IOException(s);
}
if (request.getParameter("filter") != null)
tmpl.setFilter(request.getParameter("filter"));
if (request.getParameter("format") != null)

View File

@ -82,11 +82,14 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetSchemaAlterStatusRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyColumnRequest;
@ -1111,4 +1114,28 @@ public final class RequestConverter {
public static GetClusterStatusRequest buildGetClusterStatusRequest() {
return GetClusterStatusRequest.newBuilder().build();
}
/**
* Creates a request for running a catalog scan
* @return A {@link CatalogScanRequest}
*/
public static CatalogScanRequest buildCatalogScanRequest() {
return CatalogScanRequest.newBuilder().build();
}
/**
* Creates a request for enabling/disabling the catalog janitor
* @return A {@link EnableCatalogJanitorRequest}
*/
public static EnableCatalogJanitorRequest buildEnableCatalogJanitorRequest(boolean enable) {
return EnableCatalogJanitorRequest.newBuilder().setEnable(enable).build();
}
/**
* Creates a request for querying the master whether the catalog janitor is enabled
* @return A {@link IsCatalogJanitorEnabledRequest}
*/
public static IsCatalogJanitorEnabledRequest buildIsCatalogJanitorEnabledRequest() {
return IsCatalogJanitorEnabledRequest.newBuilder().build();
}
}

View File

@ -36,6 +36,8 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ActionResult;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorResponse;
import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
import org.apache.hadoop.util.StringUtils;
@ -219,5 +221,21 @@ public final class ResponseConverter {
return builder.build();
}
/**
* Creates a response for the catalog scan request
* @return A CatalogScanResponse
*/
public static CatalogScanResponse buildCatalogScanResponse(int numCleaned) {
return CatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
}
/**
* Creates a response for the catalog scan request
* @return A EnableCatalogJanitorResponse
*/
public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
}
// End utilities for Admin
}

View File

@ -154,6 +154,28 @@ message SetBalancerRunningResponse {
optional bool prevBalanceValue = 1;
}
message CatalogScanRequest {
}
message CatalogScanResponse {
optional int32 scanResult = 1;
}
message EnableCatalogJanitorRequest {
required bool enable = 1;
}
message EnableCatalogJanitorResponse {
optional bool prevValue = 1;
}
message IsCatalogJanitorEnabledRequest {
}
message IsCatalogJanitorEnabledResponse {
required bool value = 1;
}
service MasterAdminService {
/** Adds a column to the specified table. */
rpc addColumn(AddColumnRequest)
@ -236,4 +258,19 @@ service MasterAdminService {
rpc setBalancerRunning(SetBalancerRunningRequest)
returns(SetBalancerRunningResponse);
/** Get a run of the catalog janitor */
rpc runCatalogScan(CatalogScanRequest)
returns(CatalogScanResponse);
/**
* Enable the catalog janitor on or off.
*/
rpc enableCatalogJanitor(EnableCatalogJanitorRequest)
returns(EnableCatalogJanitorResponse);
/**
* Query whether the catalog janitor is enabled.
*/
rpc isCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest)
returns(IsCatalogJanitorEnabledResponse);
}

View File

@ -92,6 +92,27 @@ module Hbase
java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false))
end
#----------------------------------------------------------------------------------------------
# Request a scan of the catalog table (for garbage collection)
# Returns an int signifying the number of entries cleaned
def catalogjanitor_run()
@admin.runCatalogScan()
end
#----------------------------------------------------------------------------------------------
# Enable/disable the catalog janitor
# Returns previous catalog janitor switch setting.
def catalogjanitor_switch(enableDisable)
@admin.enableCatalogJanitor(java.lang.Boolean::valueOf(enableDisable))
end
#----------------------------------------------------------------------------------------------
# Query on the catalog janitor state (enabled/disabled?)
# Returns catalog janitor state (true signifies enabled).
def catalogjanitor_enabled()
@admin.isCatalogJanitorEnabled()
end
#----------------------------------------------------------------------------------------------
# Enables a table
def enable(table_name)

View File

@ -287,6 +287,9 @@ Shell.load_command_group(
unassign
zk_dump
hlog_roll
catalogjanitor_run
catalogjanitor_switch
catalogjanitor_enabled
]
)

View File

@ -0,0 +1,40 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
module Shell
module Commands
class CatalogjanitorEnabled < Command
def help
return <<-EOF
Query for the CatalogJanitor state (enabled/disabled?)
Examples:
hbase> catalogjanitor_enabled
EOF
end
def command()
format_simple_command do
formatter.row([
admin.catalogjanitor_enabled()? "true" : "false"
])
end
end
end
end
end

View File

@ -0,0 +1,37 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
module Shell
module Commands
class CatalogjanitorRun < Command
def help
return <<-EOF
Catalog janitor command to run the (garbage collection) scan from command line.
hbase> catalogjanitor_run
EOF
end
def command()
format_simple_command do
admin.catalogjanitor_run()
end
end
end
end
end

View File

@ -0,0 +1,41 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
module Shell
module Commands
class CatalogjanitorSwitch < Command
def help
return <<-EOF
Enable/Disable CatalogJanitor. Returns previous CatalogJanitor state.
Examples:
hbase> catalogjanitor_switch true
hbase> catalogjanitor_switch false
EOF
end
def command(enableDisable)
format_simple_command do
formatter.row([
admin.catalogjanitor_switch(enableDisable)? "true" : "false"
])
end
end
end
end
end