HBASE-19787 Fix or disable tests broken in branch-2 so can cut beta-1

M dev-support/make_rc.sh
  Disable checkstyle building site. Its an issue being fixed over in HBASE-19780

M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
  The clusterid was being set into the process only after the
  regionserver registers with the Master. That can be too late for some
  test clients in particular. e.g. TestZKAsyncRegistry needs it as soon
  as it goes to run which could be before Master had called its run
  method  which is regionserver run method which then calls back to the
  master to register itself... and only then do we set the clusterid.
  HBASE-19694 changed start order which made it so this test failed.
  Setting the clusterid right after we set it in zk makes the test pass.

  Another change was that backup masters were not going down on stop.
  Backup masters were sleeping for the default zk period which is 90
  seconds. They were not being woken up to check for stop. On stop
  master now tells active master manager.

M hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java
  Prevent creation of acl table. Messes up our being able to go down
  promptly.

M hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java
M hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
M hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java
  Disabled for now because it wants to run with regions on the Master...
  currently broke!

M hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
  Add a bit of debugging.

M hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDLSAsyncFSWAL.java
  Disabled. Fails 40% of the time.

M hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDLSFSHLog.java
  Disabled. Fails 33% of the time.

Disabled stochastic load balancer for favored nodes because it fails on
occasion and we are not doing favored nodes in branch-2.

dev-support/make_rc.sh

@@ -75,9 +75,12 @@ function build_src {
 
 # Build bin tgz
 function build_bin {
-  MAVEN_OPTS="${mvnopts}" ${mvn} clean install -DskipTests -Papache-release -Prelease \
+  MAVEN_OPTS="${mvnopts}" ${mvn} clean install -DskipTests \
+    -Papache-release -Prelease \
     -Dmaven.repo.local=${output_dir}/repository
-  MAVEN_OPTS="${mvnopts}" ${mvn} install -DskipTests site assembly:single -Papache-release -Prelease \
+  MAVEN_OPTS="${mvnopts}" ${mvn} install -DskipTests \
+    -Dcheckstyle.skip=true site assembly:single \
+    -Papache-release -Prelease \
     -Dmaven.repo.local=${output_dir}/repository
   mv ./hbase-assembly/target/hbase-*.tar.gz "${output_dir}"
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -55,6 +55,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.ClusterMetrics;
 import org.apache.hadoop.hbase.ClusterMetrics.Option;
 import org.apache.hadoop.hbase.ClusterMetricsBuilder;
@@ -796,9 +797,13 @@ public class HMaster extends HRegionServer implements MasterServices {
       this.tableDescriptors.getAll();
     }
 
-    // Publish cluster ID
+    // Publish cluster ID; set it in Master too. The superclass RegionServer does this later but
-    status.setStatus("Publishing Cluster ID in ZooKeeper");
+    // only after it has checked in with the Master. At least a few tests ask Master for clusterId
+    // before it has called its run method and before RegionServer has done the reportForDuty.
+    ClusterId clusterId = fileSystemManager.getClusterId();
+    status.setStatus("Publishing Cluster ID " + clusterId + " in ZooKeeper");
     ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
+    this.clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
 
     this.serverManager = createServerManager(this);
 
@@ -845,10 +850,6 @@ public class HMaster extends HRegionServer implements MasterServices {
     if (this.balancer instanceof FavoredNodesPromoter) {
       favoredNodesManager = new FavoredNodesManager(this);
     }
-    // Wait for regionserver to finish initialization.
-    if (LoadBalancer.isTablesOnMaster(conf)) {
-      waitForServerOnline();
-    }
 
     //initialize load balancer
     this.balancer.setMasterServices(this);
@@ -2692,6 +2693,14 @@ public class HMaster extends HRegionServer implements MasterServices {
     stop("Stopped by " + Thread.currentThread().getName());
   }
 
+  @Override
+  public void stop(String msg) {
+    super.stop(msg);
+    if (this.activeMasterManager != null) {
+      this.activeMasterManager.stop();
+    }
+  }
+
   void checkServiceStarted() throws ServerNotRunningYetException {
     if (!serviceStarted) {
       throw new ServerNotRunningYetException("Server is not running yet");

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -444,7 +444,7 @@ public class HRegionServer extends HasThread implements
   /**
    * Unique identifier for the cluster we are a part of.
    */
-  private String clusterId;
+  protected String clusterId;
 
   /**
    * MX Bean for RegionServerInfo

hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java

@@ -258,7 +258,6 @@ public class JVMClusterUtil {
           LOG.error("Exception occurred in HMaster.shutdown()", e);
         }
       }
-
     }
     boolean wasInterrupted = false;
     final long maxTime = System.currentTimeMillis() + 30 * 1000;

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -1033,7 +1033,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
     t.close();
 
     getAdmin(); // create immediately the hbaseAdmin
-    LOG.info("Minicluster is up");
+    LOG.info("Minicluster is up; activeMaster=" + this.getHBaseCluster().getMaster());
 
     return (MiniHBaseCluster)this.hbaseCluster;
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java

@@ -1,4 +1,4 @@
-/**
+/*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -31,9 +31,11 @@ import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
+import org.apache.hadoop.hbase.security.access.AccessControlLists;
 import org.apache.hadoop.hbase.security.access.AccessController;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Threads;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -181,6 +183,12 @@ public class TestJMXConnectorServer {
    * stopMaster/preStopRegionServer/preShutdown explicitly.
    */
   public static class MyAccessController extends AccessController {
+    @Override
+    public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
+      // Do nothing. In particular, stop the creation of the hbase:acl table. It makes the
+      // shutdown take time.
+    }
+
     @Override
     public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> c) throws IOException {
       if (!hasAccess) {

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java

@@ -58,11 +58,13 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@Ignore // Depends on Master being able to host regions. Needs fixing.
 @Category({MediumTests.class, FlakeyTests.class})
 public class TestMultiParallel {
   private static final Logger LOG = LoggerFactory.getLogger(TestMultiParallel.class);

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java

@@ -30,6 +30,7 @@ import java.util.stream.IntStream;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
@@ -43,10 +44,12 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Category({ MediumTests.class, ClientTests.class })
 public class TestZKAsyncRegistry {
-
+  private static final Logger LOG = LoggerFactory.getLogger(TestZKAsyncRegistry.class);
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
   private static ZKAsyncRegistry REGISTRY;
@@ -96,8 +99,11 @@ public class TestZKAsyncRegistry {
 
   @Test
   public void test() throws InterruptedException, ExecutionException, IOException {
-    assertEquals(TEST_UTIL.getHBaseCluster().getClusterStatus().getClusterId(),
+    LOG.info("STARTED TEST");
-      REGISTRY.getClusterId().get());
+    String clusterId = REGISTRY.getClusterId().get();
+    String expectedClusterId = TEST_UTIL.getHBaseCluster().getMaster().getClusterId();
+    assertEquals("Expected " + expectedClusterId + ", found=" + clusterId,
+        expectedClusterId, clusterId);
     assertEquals(TEST_UTIL.getHBaseCluster().getClusterStatus().getServersSize(),
       REGISTRY.getCurrentNrHRS().get().intValue());
     assertEquals(TEST_UTIL.getHBaseCluster().getMaster().getServerName(),

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDLSAsyncFSWAL.java

@@ -19,8 +19,10 @@ package org.apache.hadoop.hbase.master;
 
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.Ignore;
 import org.junit.experimental.categories.Category;
 
+@Ignore // Temporarily disabled. Fails 40% of the time.
 @Category({ MasterTests.class, LargeTests.class })
 public class TestDLSAsyncFSWAL extends AbstractTestDLS {
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDLSFSHLog.java

@@ -19,8 +19,10 @@ package org.apache.hadoop.hbase.master;
 
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.Ignore;
 import org.junit.experimental.categories.Category;
 
+@Ignore // Fails 33% of the time. Disabling for now.
 @Category({ MasterTests.class, LargeTests.class })
 public class TestDLSFSHLog extends AbstractTestDLS {
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableStateManager.java

@@ -58,7 +58,7 @@ public class TestTableStateManager {
   @Test(timeout = 60000)
   public void testUpgradeFromZk() throws Exception {
     final TableName tableName = TableName.valueOf(name.getMethodName());
-    TEST_UTIL.startMiniCluster(1, 1);
+    TEST_UTIL.startMiniCluster(2, 1);
     TEST_UTIL.shutdownMiniHBaseCluster();
     ZKWatcher watcher = TEST_UTIL.getZooKeeperWatcher();
     setTableStateInZK(watcher, tableName, ZooKeeperProtos.DeprecatedTableState.State.DISABLED);

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredStochasticLoadBalancer.java

@@ -68,6 +68,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
 
+@Ignore // Disabled
 @Category(MediumTests.class)
 public class TestFavoredStochasticLoadBalancer extends BalancerTestBase {
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.apache.hadoop.hbase.util.Threads;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -102,6 +103,8 @@ public class TestRegionsOnMasterOptions {
     checkBalance(0, rsCount);
   }
 
+  @Ignore // Fix this. The Master startup doesn't allow Master reporting as a RegionServer, not
+  // until way late after the Master startup finishes. Needs more work.
   @Test
   public void testSystemTablesOnMaster() throws Exception {
     c.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java

@@ -1,4 +1,4 @@
-/**
+/*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -63,11 +63,13 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@Ignore // Depends on Master being able to host regions. Needs fixing.
 @Category(MediumTests.class)
 public class TestRegionServerReadRequestMetrics {
   private static final Logger LOG =