From b2d329b5d850a6f61e55df088390a1ac3349aced Mon Sep 17 00:00:00 2001 From: Aman Poonia Date: Mon, 19 Jul 2021 15:30:56 +0530 Subject: [PATCH] HBASE-25986 set default value of normalization enabled from hbase site (#3492) (#3372) Signed-off-by: Viraj Jasani --- .../apache/hadoop/hbase/HTableDescriptor.java | 3 +- .../hbase/zookeeper/ZooKeeperWatcher.java | 8 ++--- .../src/main/resources/hbase-default.xml | 7 +++++ .../apache/hadoop/hbase/master/HMaster.java | 30 ++++++++++++++----- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 3bdd7d3eded..ae54df2d354 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -280,7 +280,6 @@ public class HTableDescriptor implements WritableComparable { String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH)); DEFAULT_VALUES.put(DURABILITY, DEFAULT_DURABLITY.name()); //use the enum name DEFAULT_VALUES.put(REGION_REPLICATION, String.valueOf(DEFAULT_REGION_REPLICATION)); - DEFAULT_VALUES.put(NORMALIZATION_ENABLED, String.valueOf(DEFAULT_NORMALIZATION_ENABLED)); DEFAULT_VALUES.put(PRIORITY, String.valueOf(DEFAULT_PRIORITY)); for (String s : DEFAULT_VALUES.keySet()) { RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s))); @@ -684,7 +683,7 @@ public class HTableDescriptor implements WritableComparable { * @return true if region normalization is enabled for this table */ public boolean isNormalizationEnabled() { - return isSomething(NORMALIZATION_ENABLED_KEY, DEFAULT_NORMALIZATION_ENABLED); + return isSomething(NORMALIZATION_ENABLED_KEY, false); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java index b180fb9411d..e3a662b9af5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -185,13 +185,13 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable { /** * Instantiate a ZooKeeper connection and watcher. - * @param identifier string that is passed to RecoverableZookeeper to be used as - * identifier for this instance. Use null for default. + * @param identifier string that is passed to RecoverableZookeeper to be used as identifier for + * this instance. Use null for default. * @throws IOException * @throws ZooKeeperConnectionException */ - public ZooKeeperWatcher(Configuration conf, String identifier, - Abortable abortable) throws ZooKeeperConnectionException, IOException { + public ZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable) + throws ZooKeeperConnectionException, IOException { this(conf, identifier, abortable, false); } diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 2761b4f6d09..93474a20da7 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -642,6 +642,13 @@ possible configurations would overwhelm and obscure the important. The minimum size for a region to be considered for a merge, in whole MBs. + + hbase.table.normalization.enabled + false + This config is used to set default behaviour of normalizer at table level. To override this at table + level one can set NORMALIZATION_ENABLED at table descriptor level and that property will be honored + + hbase.server.thread.wakefrequency 10000 diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 2d052e32454..92fb6d1e43d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -205,7 +205,8 @@ import org.mortbay.jetty.servlet.ServletHolder; @SuppressWarnings("deprecation") public class HMaster extends HRegionServer implements MasterServices, Server { private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); - + public static final String HBASE_TABLE_NORMALIZATION_ENABLED = + "hbase.table.normalization.enabled"; /** * Protection against zombie master. Started once Master accepts active responsibility and * starts taking over responsibilities. Allows a finite time window before giving up ownership. @@ -381,6 +382,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { private long splitPlanCount; private long mergePlanCount; + private boolean defaultNormalizerTableLevel; /** flag used in test cases in order to simulate RS failures during master initialization */ private volatile boolean initializationBeforeMetaAssignment = false; @@ -529,6 +531,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { activeMasterManager = null; } cachedClusterId = new CachedClusterId(conf); + this.defaultNormalizerTableLevel = extractDefaultNormalizerValue(conf); } // return the actual infoPort, -1 means disable info server. @@ -1019,7 +1022,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { LOG.info("Closing excess replica of meta region " + r.getRegion()); // send a close and wait for a max of 30 seconds ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(), - r.getRegion(), 30000); + r.getRegion(), 30000); ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId)); } } @@ -1117,6 +1120,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.setStatus("META assigned."); } + private boolean extractDefaultNormalizerValue(final Configuration configuration) { + String s = configuration.get(HBASE_TABLE_NORMALIZATION_ENABLED); + return Boolean.parseBoolean(s); + } + private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout, Set previouslyFailedRs) throws IOException, KeeperException { ServerName currentServer = regionState.getServerName(); @@ -1692,12 +1700,20 @@ public class HMaster extends HRegionServer implements MasterServices, Server { if (table.isSystemTable()) { continue; } - + boolean normalizationEnabled; HTableDescriptor tableDescriptor = getTableDescriptors().get(table); - if (tableDescriptor != null && !tableDescriptor.isNormalizationEnabled()) { - LOG.debug("Skipping normalization for table: " + table - + ", as it doesn't have auto normalization turned on"); - continue; + if (tableDescriptor != null) { + String defined = tableDescriptor.getValue(HTableDescriptor.NORMALIZATION_ENABLED); + if (defined != null) { + normalizationEnabled = tableDescriptor.isNormalizationEnabled(); + } else { + normalizationEnabled = this.defaultNormalizerTableLevel; + } + if (!normalizationEnabled) { + LOG.debug("Skipping table " + table + " because normalization is disabled in its " + + "table properties and normalization is also disabled at table level by default"); + continue; + } } // make one last check that the cluster isn't shutting down before proceeding. if (skipRegionManagementAction("region normalizer")) {