From dfe84fb1f9ac10a338a4dc1423877715c82df511 Mon Sep 17 00:00:00 2001 From: larsh Date: Mon, 24 Jun 2013 20:29:07 +0000 Subject: [PATCH] HBASE-5083 Backup HMaster should have http infoport open with link to the active master (Cody Marcel) git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1496198 13f79535-47bb-0310-9956-ffa450edef68 --- .../tmpl/master/BackupMasterListTmpl.jamon | 69 --------------- .../tmpl/master/BackupMasterStatusTmpl.jamon | 74 ++++++++++++++++ .../hbase/tmpl/master/MasterStatusTmpl.jamon | 84 +++++++++++-------- .../hbase/master/ActiveMasterManager.java | 2 +- .../apache/hadoop/hbase/master/HMaster.java | 36 +++++--- .../hbase/master/MasterStatusServlet.java | 20 +++-- .../hbase/master/TestMasterStatusServlet.java | 6 ++ 7 files changed, 168 insertions(+), 123 deletions(-) delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterListTmpl.jamon create mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterListTmpl.jamon deleted file mode 100644 index 64660d20880..00000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterListTmpl.jamon +++ /dev/null @@ -1,69 +0,0 @@ -<%doc> -Copyright The Apache Software Foundation - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -<%args> -HMaster master; - - -<%import> - java.util.*; - org.apache.hadoop.util.StringUtils; - org.apache.hadoop.hbase.util.Bytes; - org.apache.hadoop.hbase.master.HMaster; - org.apache.hadoop.hbase.HConstants; - org.apache.hadoop.hbase.ServerName; - org.apache.hadoop.hbase.ClusterStatus; - - -<%java> -Collection backupMasters = null; -if (master.isActiveMaster()) { - ClusterStatus status = master.getClusterStatus(); - backupMasters = status.getBackupMasters(); -} - - - -<%if (backupMasters != null && backupMasters.size() > 0)%> - - - - - -<%java> - ServerName [] serverNames = backupMasters.toArray(new ServerName[backupMasters.size()]); - Arrays.sort(serverNames); - for (ServerName serverName: serverNames) { - - - - - - -<%java> - } - - - -
ServerNamePortStart Time
<% serverName.getHostname() %><% serverName.getPort() %><% new Date(serverName.getStartcode()) %>
Total:<% (backupMasters != null) ? backupMasters.size() : 0 %>
- - - - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon new file mode 100644 index 00000000000..f872ce3317a --- /dev/null +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon @@ -0,0 +1,74 @@ +<%doc> + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%args> +HMaster master; + +<%import> +java.util.*; +org.apache.hadoop.hbase.ServerName; +org.apache.hadoop.hbase.ClusterStatus; +org.apache.hadoop.hbase.master.HMaster; + +<%java> +Collection masters = null; + +if (master.isActiveMaster()) { + ClusterStatus status = master.getClusterStatus(); + masters = status.getBackupMasters(); +} else{ + ServerName sn = master.getMasterAddressManager().getMasterAddress(); + assert sn != null : "Failed to retreive master's ServerName!"; + masters = Collections.singletonList(sn); +} + + +<%java> +ServerName [] serverNames = masters.toArray(new ServerName[masters.size()]); + +<%if (!master.isActiveMaster()) %> + <%if serverNames[0] != null %> +

Master

+ /master-status" target="_blank"><% serverNames[0].getHostname() %> + <%else> + Unable to parse master hostname. + +<%else> +

Backup Masters

+ + + + + + + + <%java> + Arrays.sort(serverNames); + for (ServerName serverName : serverNames) { + + + + + + + <%java> + } + + +
ServerNamePortStart Time
/master-status" target="_blank"><% serverName.getHostname() %><% serverName.getPort() %><% new Date(serverName.getStartcode()) %>
Total:<% (masters != null) ? masters.size() : 0 %>
+ diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 83b51876fb0..355b9f01379 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -26,6 +26,8 @@ Set deadServers = null; boolean catalogJanitorEnabled = true; String filter = "general"; String format = "html"; +ServerManager serverManager = null; +AssignmentManager assignmentManager = null; <%import> java.util.*; @@ -34,6 +36,8 @@ org.apache.hadoop.hbase.util.Bytes; org.apache.hadoop.hbase.util.JvmVersion; org.apache.hadoop.hbase.util.FSUtils; org.apache.hadoop.hbase.master.HMaster; +org.apache.hadoop.hbase.master.AssignmentManager; +org.apache.hadoop.hbase.master.ServerManager; org.apache.hadoop.hbase.HConstants; org.apache.hadoop.hbase.ServerLoad; org.apache.hadoop.hbase.ServerName; @@ -47,6 +51,11 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &> <%java return; %> +<%java> +ServerManager serverManager = master.getServerManager(); +AssignmentManager assignmentManager = master.getAssignmentManager(); + + @@ -94,6 +103,7 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+ <%if master.isActiveMaster() %>

Master <% master.getServerName().getHostname() %>

@@ -125,12 +135,9 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; <& deadRegionServers &> -
-

Backup Masters

- <& BackupMasterListTmpl; master = master &> + <& BackupMasterStatusTmpl; master = master &>
-

Tables

@@ -162,8 +169,13 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
- <& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&> + <%else> +
+ <& BackupMasterStatusTmpl; master = master &> +
+ +
<& ../common/TaskMonitorTmpl; filter = filter &> @@ -196,48 +208,50 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; <% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %> When Hadoop version was compiled and by whom - - HBase Root Directory - <% FSUtils.getRootDir(master.getConfiguration()).toString() %> - Location of HBase home directory - - - HBase Cluster ID - <% master.getClusterId() != null ? master.getClusterId() : "Not set" %> - Unique identifier generated for each HBase cluster - - - Load average - <% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %> - Average number of regions per regionserver. Naive computation. - - <%if frags != null %> - - Fragmentation - <% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %> - Overall fragmentation of all tables, including .META. - - Zookeeper Quorum <% master.getZooKeeperWatcher().getQuorum() %> Addresses of all registered ZK servers. For more, see zk dump. - Coprocessors - <% java.util.Arrays.toString(master.getCoprocessors()) %> - Coprocessors currently loaded by the master + HBase Root Directory + <% FSUtils.getRootDir(master.getConfiguration()).toString() %> + Location of HBase home directory HMaster Start Time <% new Date(master.getMasterStartTime()) %> Date stamp of when this HMaster was started - - HMaster Active Time - <% new Date(master.getMasterActiveTime()) %> - Date stamp of when this HMaster became active - + <%if master.isActiveMaster() %> + + HMaster Active Time + <% new Date(master.getMasterActiveTime()) %> + Date stamp of when this HMaster became active + + + HBase Cluster ID + <% master.getClusterId() != null ? master.getClusterId() : "Not set" %> + Unique identifier generated for each HBase cluster + + + Load average + <% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %> + Average number of regions per regionserver. Naive computation. + + <%if frags != null %> + + Fragmentation + <% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %> + Overall fragmentation of all tables, including .META. + + + + Coprocessors + <% java.util.Arrays.toString(master.getCoprocessors()) %> + Coprocessors currently loaded by the master + +
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java index 8d4005ce677..5eeaa82c291 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java @@ -49,7 +49,7 @@ import org.apache.zookeeper.KeeperException; * the active master of the cluster. */ @InterfaceAudience.Private -class ActiveMasterManager extends ZooKeeperListener { +public class ActiveMasterManager extends ZooKeeperListener { private static final Log LOG = LogFactory.getLog(ActiveMasterManager.class); final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 61276c9aecf..ce6583634d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -197,6 +197,7 @@ import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; +import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -258,6 +259,8 @@ MasterServices, Server { private DrainingServerTracker drainingServerTracker; // Tracker for load balancer state private LoadBalancerTracker loadBalancerTracker; + // master address manager and watcher + private MasterAddressTracker masterAddressManager; // RPC server for the HMaster private final RpcServerInterface rpcServer; @@ -528,6 +531,20 @@ MasterServices, Server { masterStartTime = System.currentTimeMillis(); try { this.registeredZKListenersBeforeRecovery = this.zooKeeper.getListeners(); + this.masterAddressManager = new MasterAddressTracker(getZooKeeperWatcher(), this); + this.masterAddressManager.start(); + + // Put up info server. + int port = this.conf.getInt("hbase.master.info.port", 60010); + if (port >= 0) { + String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0"); + this.infoServer = new InfoServer(MASTER, a, port, false, this.conf); + this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class); + this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class); + this.infoServer.setAttribute(MASTER, this); + this.infoServer.start(); + } + /* * Block on becoming the active master. * @@ -1057,6 +1074,14 @@ MasterServices, Server { return this.zooKeeper; } + public ActiveMasterManager getActiveMasterManager() { + return this.activeMasterManager; + } + + public MasterAddressTracker getMasterAddressManager() { + return this.masterAddressManager; + } + /* * Start up all services. If any of these threads gets an unhandled exception * then they just die with a logged message. This should be fine because @@ -1095,17 +1120,6 @@ MasterServices, Server { .getFileSystem(), archiveDir); Threads.setDaemonThreadRunning(hfileCleaner.getThread(), n + ".archivedHFileCleaner"); - // Put up info server. - int port = this.conf.getInt(HConstants.MASTER_INFO_PORT, 60010); - if (port >= 0) { - String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0"); - this.infoServer = new InfoServer(MASTER, a, port, false, this.conf); - this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class); - this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class); - this.infoServer.setAttribute(MASTER, this); - this.infoServer.start(); - } - // Start the health checker if (this.healthCheckChore != null) { Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java index a293803cfe4..268c433f2f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java @@ -59,12 +59,17 @@ public class MasterStatusServlet extends HttpServlet { HBaseAdmin admin = new HBaseAdmin(conf); Map frags = getFragmentationInfo(master, conf); - - ServerName metaLocation = getMetaLocationOrNull(master); - //ServerName metaLocation = master.getCatalogTracker().getMetaLocation(); - List servers = master.getServerManager().getOnlineServersList(); - Set deadServers = master.getServerManager().getDeadServers().copyServerNames(); - + ServerName metaLocation = null; + List servers = null; + Set deadServers = null; + + if(master.isActiveMaster()){ + metaLocation = getMetaLocationOrNull(master); + //ServerName metaLocation = master.getCatalogTracker().getMetaLocation(); + servers = master.getServerManager().getOnlineServersList(); + deadServers = master.getServerManager().getDeadServers().copyServerNames(); + } + response.setContentType("text/html"); MasterStatusTmpl tmpl; try { @@ -76,6 +81,7 @@ public class MasterStatusServlet extends HttpServlet { .setCatalogJanitorEnabled(master.isCatalogJanitorEnabled(null, RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue()); } catch (ServiceException s) { + admin.close(); throw new IOException(s); } if (request.getParameter("filter") != null) @@ -88,7 +94,7 @@ public class MasterStatusServlet extends HttpServlet { private ServerName getMetaLocationOrNull(HMaster master) { try { - return master.getCatalogTracker().getMetaLocation(); + return (master.getCatalogTracker() == null) ? null : master.getCatalogTracker().getMetaLocation(); } catch (InterruptedException e) { LOG.warn("Unable to get meta location", e); return null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java index 7ec0457a4ff..1b70e3e9e88 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java @@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hbase.tmpl.master.AssignmentManagerStatusTmpl; import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl; @@ -90,6 +91,11 @@ public class TestMasterStatusServlet { Mockito.doReturn("fakequorum").when(zkw).getQuorum(); Mockito.doReturn(zkw).when(master).getZooKeeperWatcher(); + // Fake MasterAddressTracker + MasterAddressTracker tracker = Mockito.mock(MasterAddressTracker.class); + Mockito.doReturn(tracker).when(master).getMasterAddressManager(); + Mockito.doReturn(FAKE_HOST).when(tracker).getMasterAddress(); + // Mock admin admin = Mockito.mock(HBaseAdmin.class); }