SOLR-13574: Fix many test and test-framework classes to not fail on After/AfterClass cleanup if assumptions fail in Before/BeforeClass setup

solr/contrib/analytics/src/test/org/apache/solr/analytics/function/field/AbstractAnalyticsFieldTest.java

@@ -224,7 +224,10 @@ public class AbstractAnalyticsFieldTest extends SolrTestCaseJ4 {
   
   @AfterClass
   public static void closeSearcher() throws IOException {
-    ref.decref();
+    if (null != ref) {
+      ref.decref();
+      ref = null;
+    }
   }
   
   protected <T> void checkSingleFieldValues(Map<String,T> expected, Map<String,T> found, Set<String> missing) {

solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsCloudTest.java

@@ -54,7 +54,6 @@ public class LegacyAbstractAnalyticsCloudTest extends SolrCloudTestCase {
   
   @After
   public void teardownCollection() throws Exception {
-    cluster.deleteAllCollections();
     shutdownCluster();
   }
 

solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java

@@ -58,7 +58,10 @@ public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCa
   @Override
   @After
   public void tearDown() throws Exception {
-    jetty.stop();
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
     super.tearDown();
   }
 

solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java

@@ -103,7 +103,10 @@ public class TestHierarchicalDocBuilder extends AbstractDataImportHandlerTestCas
   
   @After
   public void after() {
-    req.close();
+    if (null != req) {
+      req.close();
+      req = null;
+    }
     MockDataSource.clearCache();
   }
 

solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java

@@ -96,7 +96,9 @@ public class TestJdbcDataSource extends AbstractDataImportHandlerTestCase {
       System.setProperty("java.naming.factory.initial", sysProp);
     }
     super.tearDown();
-    reset(driver, dataSource, connection);
+    if (null != driver) {
+      reset(driver, dataSource, connection);
+    }
   }
 
   @Test

solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java

@@ -138,8 +138,14 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe
     } catch (Exception e) {
       log.error("Error deleting core", e);
     }
-    jetty.stop();
-    instance.tearDown();
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
+    if (null != instance) {
+      instance.tearDown();
+      instance = null;
+    }
     super.tearDown();
   }
 

solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java

@@ -82,13 +82,15 @@ public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
 
   @AfterClass
   public static void dihZk_afterClass() throws Exception {
-    cc.shutdown();
-    
-    zkServer.shutdown();
-
-    zkServer = null;
+    if (null != cc) {
+      cc.shutdown();
+      cc = null;
+    }
+    if (null != zkServer) {
+      zkServer.shutdown();
+      zkServer = null;
+    }
     zkDir = null;
-    cc = null;
   }
 
   @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")

solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java

@@ -299,7 +299,10 @@ public class TestLTROnSolrCloud extends TestRerankBase {
 
   @AfterClass
   public static void after() throws Exception {
-    FileUtils.deleteDirectory(tmpSolrHome);
+    if (null != tmpSolrHome) {
+      FileUtils.deleteDirectory(tmpSolrHome);
+      tmpSolrHome = null;
+    }
     System.clearProperty("managed.schema.mutable");
   }
 

solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java

@@ -214,11 +214,18 @@ public class TestRerankBase extends RestTestBase {
   }
 
   protected static void aftertest() throws Exception {
-    restTestHarness.close();
-    restTestHarness = null;
-    jetty.stop();
-    jetty = null;
-    FileUtils.deleteDirectory(tmpSolrHome);
+    if (null != restTestHarness) {
+      restTestHarness.close();
+      restTestHarness = null;
+    }
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
+    if (null != tmpSolrHome) {
+      FileUtils.deleteDirectory(tmpSolrHome);
+      tmpSolrHome = null;
+    }
     System.clearProperty("managed.schema.mutable");
     // System.clearProperty("enable.update.log");
     unchooseDefaultFeatureFormat();

solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java

@@ -92,7 +92,10 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {
   public void tearDown() throws Exception {
     super.tearDown();
     IOUtils.closeQuietly(solrCloudScraper);
-    executor.shutdownNow();
+    if (null != executor) {
+      executor.shutdownNow();
+      executor = null;
+    }
   }
 
   @Test

solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java

@@ -83,8 +83,14 @@ public class SolrStandaloneScraperTest extends RestTestBase {
     IOUtils.closeQuietly(solrScraper);
     IOUtils.closeQuietly(solrClient);
     cleanUpHarness();
-    executor.shutdownNow();
-    jetty.stop();
+    if (null != executor) {
+      executor.shutdownNow();
+      executor = null;
+    }
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/TestTolerantSearch.java

@@ -99,12 +99,18 @@ public class TestTolerantSearch extends SolrJettyTestBase {
   
   @AfterClass
   public static void destroyThings() throws Exception {
-    collection1.close();
-    collection2.close();
-    collection1 = null;
-    collection2 = null;
-    jetty.stop();
-    jetty=null;
+    if (null != collection1) {
+      collection1.close();
+      collection1 = null;
+    }
+    if (null != collection2) {
+      collection2.close();
+      collection2 = null;
+    }
+    if (null != jetty) {
+      jetty.stop();
+      jetty=null;
+    }
     resetExceptionIgnores();
     systemClearPropertySolrDisableShardsWhitelist();
   }

solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java

@@ -65,8 +65,10 @@ public class TestEmbeddedSolrServerSchemaAPI extends SolrTestCaseJ4 {
 
   @AfterClass
   public static void destroyClass() throws IOException {
-    server.close(); // doubtful
-    server = null;
+    if (null != server) {
+      server.close(); 
+      server = null;
+    }
     System.clearProperty("managed.schema.mutable");
   }
 

solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java

@@ -49,7 +49,7 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
   @AfterClass
   public static void afterClass() throws Exception {
 
-    if (suiteFailureMarker.wasSuccessful()) {
+    if (null != cluster && suiteFailureMarker.wasSuccessful()) {
       zkClient().printLayoutToStream(System.out);
     }
 

solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java

@@ -82,7 +82,10 @@ public class CreateRoutedAliasTest extends SolrCloudTestCase {
   public void doAfter() throws Exception {
     cluster.deleteAllCollections(); // deletes aliases too
 
-    solrClient.close();
+    if (null != solrClient) {
+      solrClient.close();
+      solrClient = null;
+    }
   }
 
   // This is a fairly complete test where we set many options and see that it both affected the created

solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java

@@ -349,8 +349,13 @@ public class DistributedQueueTest extends SolrTestCaseJ4 {
   }
 
   protected void closeZk() throws Exception {
-    if (zkClient != null)
+    if (null != zkClient) {
       zkClient.close();
-    zkServer.shutdown();
+      zkClient = null;
+    }
+    if (null != zkServer) {
+      zkServer.shutdown();
+      zkServer = null;
+    }
   }
 }

solr/core/src/test/org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java

@@ -100,8 +100,10 @@ public class LeaderVoteWaitTimeoutTest extends SolrCloudTestCase {
   
   @After
   public void tearDown() throws Exception {
-    for (SocketProxy proxy:proxies.values()) {
-      proxy.close();
+    if (null != proxies) {
+      for (SocketProxy proxy : proxies.values()) {
+        proxy.close();
+      }
     }
     shutdownCluster();
     super.tearDown();

solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java

@@ -79,6 +79,10 @@ public class MissingSegmentRecoveryTest extends SolrCloudTestCase {
   
   @After
   public void teardown() throws Exception {
+    if (null == leader) {
+      // test did not initialize, cleanup is No-Op;
+      return;
+    }
     System.clearProperty("CoreInitFailedAction");
     CollectionAdminRequest.deleteCollection(collection).process(cluster.getSolrClient());
   }

solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java

@@ -530,9 +530,15 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
   }
   
   protected void stopComponentUnderTest() throws Exception {
-    underTest.close();
-    thread.interrupt();
-    thread.join();
+    if (null != underTest) {
+      underTest.close();
+      underTest = null;
+    }
+    if (null != thread) {
+      thread.interrupt();
+      thread.join();
+      thread = null;
+    }
   }
 
   protected void issueCreateJob(Integer numberOfSlices,

solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java

@@ -17,6 +17,7 @@
 package org.apache.solr.cloud;
 
 import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -47,15 +48,17 @@ public class RecoveryZkTest extends SolrCloudTestCase {
         .configure();
   }
 
-  private StoppableIndexingThread indexThread;
-  private StoppableIndexingThread indexThread2;
+  private final List<StoppableIndexingThread> threads = new ArrayList<>();
 
   @After
   public void stopThreads() throws InterruptedException {
-    indexThread.safeStop();
-    indexThread2.safeStop();
-    indexThread.join();
-    indexThread2.join();
+    for (StoppableIndexingThread t : threads) {
+      t.safeStop();
+    }
+    for (StoppableIndexingThread t : threads) {
+      t.join();
+    }
+    threads.clear();
   }
 
   @Test
@@ -83,10 +86,14 @@ public class RecoveryZkTest extends SolrCloudTestCase {
     }
     log.info("Indexing {} documents", maxDoc);
     
-    indexThread = new StoppableIndexingThread(null, cluster.getSolrClient(), "1", true, maxDoc, 1, true);
+    final StoppableIndexingThread indexThread
+      = new StoppableIndexingThread(null, cluster.getSolrClient(), "1", true, maxDoc, 1, true);
+    threads.add(indexThread);
     indexThread.start();
     
-    indexThread2 = new StoppableIndexingThread(null, cluster.getSolrClient(), "2", true, maxDoc, 1, true);
+    final StoppableIndexingThread indexThread2
+      = new StoppableIndexingThread(null, cluster.getSolrClient(), "2", true, maxDoc, 1, true);
+    threads.add(indexThread2);
     indexThread2.start();
 
     // give some time to index...

solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java

@@ -108,7 +108,10 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
   public void doAfter() throws Exception {
     cluster.deleteAllCollections(); // deletes aliases too
 
-    solrClient.close();
+    if (null != solrClient) {
+      solrClient.close();
+      solrClient = null;
+    }
 
     TestInjection.reset();
   }

solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java

@@ -55,7 +55,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
 
   @AfterClass
   public static void closeConn() {
-    zkClient.close();
+    if (null != zkClient) {
+      zkClient.close();
+      zkClient = null;
+    }
+    zkAddr = null;
   }
 
   private static String zkAddr;

solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java

@@ -126,7 +126,10 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
   public void doAfter() throws Exception {
     cluster.deleteAllCollections();
 
-    solrClient.close();
+    if (null != solrClient) {
+      solrClient.close();
+      solrClient = null;
+    }
   }
 
   private Map<String, Object> getSchemaField(String name, SchemaResponse schemaResponse) {

solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java

@@ -78,10 +78,12 @@ public class TestCloudConsistency extends SolrCloudTestCase {
 
   @After
   public void tearDownCluster() throws Exception {
-    for (SocketProxy proxy:proxies.values()) {
-      proxy.close();
+    if (null != proxies) {
+      for (SocketProxy proxy : proxies.values()) {
+        proxy.close();
+      }
+      proxies = null;
     }
-    proxies = null;
     jettys = null;
     System.clearProperty("solr.directoryFactory");
     System.clearProperty("solr.ulog.numRecordsToKeep");

solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java

@@ -83,12 +83,30 @@ public class TestCloudDeleteByQuery extends SolrCloudTestCase {
   
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
-    S_ONE_LEADER_CLIENT.close(); S_ONE_LEADER_CLIENT = null;
-    S_TWO_LEADER_CLIENT.close(); S_TWO_LEADER_CLIENT = null;
-    S_ONE_NON_LEADER_CLIENT.close(); S_ONE_NON_LEADER_CLIENT = null;
-    S_TWO_NON_LEADER_CLIENT.close(); S_TWO_NON_LEADER_CLIENT = null;
-    NO_COLLECTION_CLIENT.close(); NO_COLLECTION_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
+    if (null != S_ONE_LEADER_CLIENT) {
+      S_ONE_LEADER_CLIENT.close();
+      S_ONE_LEADER_CLIENT = null;
+    }
+    if (null != S_TWO_LEADER_CLIENT) {
+      S_TWO_LEADER_CLIENT.close();
+      S_TWO_LEADER_CLIENT = null;
+    }
+    if (null != S_ONE_NON_LEADER_CLIENT) {
+      S_ONE_NON_LEADER_CLIENT.close();
+      S_ONE_NON_LEADER_CLIENT = null;
+    }
+    if (null != S_TWO_NON_LEADER_CLIENT) {
+      S_TWO_NON_LEADER_CLIENT.close();
+      S_TWO_NON_LEADER_CLIENT = null;
+    }
+    if (null != NO_COLLECTION_CLIENT) {
+      NO_COLLECTION_CLIENT.close();
+      NO_COLLECTION_CLIENT = null;
+    }
   }
   
   @BeforeClass

solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java

@@ -110,7 +110,10 @@ public class TestCloudPhrasesIdentificationComponent extends SolrCloudTestCase {
 
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
     for (HttpSolrClient client : CLIENTS) {
       client.close();
     }

solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java

@@ -110,7 +110,10 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {
   
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
     for (HttpSolrClient client : CLIENTS) {
       client.close();
     }

solr/core/src/test/org/apache/solr/cloud/TestCloudSearcherWarming.java

@@ -75,9 +75,12 @@ public class TestCloudSearcherWarming extends SolrCloudTestCase {
     coreNodeNameRef.set(null);
     sleepTime.set(-1);
     
-    cluster.deleteAllCollections();
-    cluster.deleteAllConfigSets();
-    cluster.shutdown();
+    if (null != cluster) {
+      cluster.deleteAllCollections();
+      cluster.deleteAllConfigSets();
+      cluster.shutdown();
+      cluster = null;
+    }
     TestInjection.wrongIndexFingerprint = null;
     
     super.tearDown();

solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java

@@ -116,7 +116,10 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    solrCluster.shutdown();
+    if (null != solrCluster) {
+      solrCluster.shutdown();
+      solrCluster = null;
+    }
     super.tearDown();
   }
 

solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIExclusivity.java

@@ -56,7 +56,10 @@ public class TestConfigSetsAPIExclusivity extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    solrCluster.shutdown();
+    if (null != solrCluster) {
+      solrCluster.shutdown();
+      solrCluster = null;
+    }
     super.tearDown();
   }
 

solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java

@@ -88,8 +88,14 @@ public class TestConfigSetsAPIZkFailure extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    solrCluster.shutdown();
-    zkTestServer.shutdown();
+    if (null != solrCluster) {
+      solrCluster.shutdown();
+      solrCluster = null;
+    }
+    if (null != zkTestServer) {
+      zkTestServer.shutdown();
+      zkTestServer = null;
+    }
     super.tearDown();
   }
 

solr/core/src/test/org/apache/solr/cloud/TestDistributedMap.java

@@ -48,8 +48,10 @@ public class TestDistributedMap extends SolrTestCaseJ4 {
       zkServer.shutdown();
       zkServer = null;
     }
-    FileUtils.deleteDirectory(zkDir.toFile());
-    zkDir = null;
+    if (null != zkDir) {
+      FileUtils.deleteDirectory(zkDir.toFile());
+      zkDir = null;
+    }
   }
   
   public void testPut() throws KeeperException, InterruptedException {

solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java

@@ -107,10 +107,12 @@ public class TestPullReplicaErrorHandling extends SolrCloudTestCase {
   
   @AfterClass
   public static void tearDownCluster() throws Exception {
-    for (SocketProxy proxy:proxies.values()) {
-      proxy.close();
+    if (null != proxies) {
+      for (SocketProxy proxy : proxies.values()) {
+        proxy.close();
+      }
+      proxies = null;
     }
-    proxies = null;
     jettys = null;
     TestInjection.reset();
   }

solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java

@@ -155,7 +155,10 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {
 
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
     for (HttpSolrClient client : CLIENTS) {
       client.close();
     }

solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java

@@ -82,12 +82,16 @@ public class TestSolrCloudWithDelegationTokens extends SolrTestCaseJ4 {
   public static void shutdown() throws Exception {
     if (miniCluster != null) {
       miniCluster.shutdown();
+      miniCluster = null;
+    }
+    if (null != solrClientPrimary) {
+      solrClientPrimary.close();
+      solrClientPrimary = null;
+    }
+    if (null != solrClientSecondary) {
+      solrClientSecondary.close();
+      solrClientSecondary = null;
     }
-    miniCluster = null;
-    solrClientPrimary.close();
-    solrClientPrimary = null;
-    solrClientSecondary.close();
-    solrClientSecondary = null;
     System.clearProperty("authenticationPlugin");
     System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED);
     System.clearProperty("solr.kerberos.cookie.domain");

solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java

@@ -158,10 +158,14 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase {
   @AfterClass
   private static void afterClass() throws Exception {
     TestInjection.reset();
-    ExecutorUtil.shutdownAndAwaitTermination(EXEC_SERVICE);
-    EXEC_SERVICE = null;
-    IOUtils.closeQuietly(CLOUD_CLIENT);
-    CLOUD_CLIENT = null;
+    if (null != EXEC_SERVICE) {
+      ExecutorUtil.shutdownAndAwaitTermination(EXEC_SERVICE);
+      EXEC_SERVICE = null;
+    }
+    if (null != CLOUD_CLIENT) {
+      IOUtils.closeQuietly(CLOUD_CLIENT);
+      CLOUD_CLIENT = null;
+    }
     for (HttpSolrClient client : CLIENTS) {
       if (null == client) {
         log.error("CLIENTS contains a null SolrClient???");

solr/core/src/test/org/apache/solr/cloud/TestStressLiveNodes.java

@@ -76,8 +76,10 @@ public class TestStressLiveNodes extends SolrCloudTestCase {
   
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close();
-    CLOUD_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
   }
 
   private static SolrZkClient newSolrZkClient() {

solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java

@@ -58,9 +58,10 @@ public class TestZkChroot extends SolrTestCaseJ4 {
       cores = null;
     }
     
-    zkServer.shutdown();
-    
-    zkServer = null;
+    if (null != zkServer) {
+      zkServer.shutdown();
+      zkServer = null;
+    }
     zkDir = null;
     
     super.tearDown();

solr/core/src/test/org/apache/solr/cloud/ZkFailoverTest.java

@@ -25,7 +25,6 @@ import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.zookeeper.KeeperException;
-import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 public class ZkFailoverTest extends SolrCloudTestCase {
@@ -39,16 +38,6 @@ public class ZkFailoverTest extends SolrCloudTestCase {
         .configure();
   }
 
-  @AfterClass
-  public static void cleanUp() throws Exception {
-    System.clearProperty("waitForZk");
-
-    for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
-      final JettySolrRunner runner = cluster.getJettySolrRunner(i);
-      runner.stop();
-    }
-  }
-
   public void testRestartZkWhenClusterDown() throws Exception {
     String coll = "coll1";
     CollectionAdminRequest.createCollection(coll, 2, 1).process(cluster.getSolrClient());

solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentDeleteAndCreateCollectionTest.java

@@ -55,7 +55,10 @@ public class ConcurrentDeleteAndCreateCollectionTest extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    solrCluster.shutdown();
+    if (null != solrCluster) {
+      solrCluster.shutdown();
+      solrCluster = null;
+    }
     super.tearDown();
   }
   

solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java

@@ -147,7 +147,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
 
   @AfterClass
   public static void teardown() throws Exception {
-    if (!realCluster) {
+    if (cloudManager != null && !realCluster) {
       cloudManager.close();
     }
     solrClient = null;

solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java

@@ -106,6 +106,11 @@ public class TestSimComputePlanAction extends SimSolrCloudTestCase {
 
   @After
   public void printState() throws Exception {
+    if (null == cluster) {
+      // test didn't init, nothing to do
+      return;
+    }
+                          
     log.info("-------------_ FINAL STATE --------------");
     log.info("* Node values: " + Utils.toJSONString(cluster.getSimNodeStateProvider().simGetAllNodeValues()));
     log.info("* Live nodes: " + cluster.getClusterStateProvider().getLiveNodes());

solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java

@@ -70,6 +70,11 @@ public class TestSimExecutePlanAction extends SimSolrCloudTestCase {
 
   @After
   public void printState() throws Exception {
+    if (null == cluster) {
+      // test didn't init, nothing to do
+      return;
+    }
+                          
     log.info("-------------_ FINAL STATE --------------");
     log.info("* Node values: " + Utils.toJSONString(cluster.getSimNodeStateProvider().simGetAllNodeValues()));
     log.info("* Live nodes: " + cluster.getClusterStateProvider().getLiveNodes());

solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java

@@ -56,8 +56,14 @@ public class CdcrOpsAndBoundariesTest extends SolrTestCaseJ4 {
 
   @After
   public void after() throws Exception {
-    target.shutdown();
-    source.shutdown();
+    if (null != target) {
+      target.shutdown();
+      target = null;
+    }
+    if (null != source) {
+      source.shutdown();
+      source = null;
+    }
   }
 
   /**
@@ -320,4 +326,4 @@ public class CdcrOpsAndBoundariesTest extends SolrTestCaseJ4 {
     deleteTargetcollection();
   }
 
-}
+}

solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrWithNodesRestartsTest.java

@@ -62,8 +62,14 @@ public class CdcrWithNodesRestartsTest extends SolrTestCaseJ4 {
 
   @After
   public void after() throws Exception {
-    target.shutdown();
-    source.shutdown();
+    if (null != target) {
+      target.shutdown();
+      target = null;
+    }
+    if (null != source) {
+      source.shutdown();
+      source = null;
+    }
   }
 
   @Test
@@ -350,4 +356,4 @@ public class CdcrWithNodesRestartsTest extends SolrTestCaseJ4 {
     deleteTargetcollection();
   }
 
-}
+}

solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java

@@ -61,7 +61,10 @@ public class TestCSVLoader extends SolrTestCaseJ4 {
     // if you override setUp or tearDown, you better call
     // the super classes version
     super.tearDown();
-    Files.delete(file.toPath());
+    if (null != file) {
+      Files.delete(file.toPath());
+      file = null;
+    }
   }
 
   void makeFile(String contents) {

solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java

@@ -147,13 +147,22 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    masterJetty.stop();
-    slaveJetty.stop();
-    masterJetty = slaveJetty = null;
-    master = slave = null;
-    masterClient.close();
-    slaveClient.close();
-    masterClient = slaveClient = null;
+    if (null != masterJetty) {
+      masterJetty.stop();
+      masterJetty = null;
+    }
+    if (null != slaveJetty) {
+      slaveJetty.stop();
+      slaveJetty = null;
+    }
+    if (null != masterClient) {
+      masterClient.close();
+      masterClient = null;
+    }
+    if (null != slaveClient) {
+      slaveClient.close();
+      slaveClient = null;
+    }
     System.clearProperty("solr.indexfetcher.sotimeout");
   }
 

solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java

@@ -115,10 +115,14 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    masterClient.close();
-    masterClient  = null;
-    masterJetty.stop();
-    masterJetty = null;
+    if (null != masterClient) {
+      masterClient.close();
+      masterClient  = null;
+    }
+    if (null != masterJetty) {
+      masterJetty.stop();
+      masterJetty = null;
+    }
     master = null;
   }
 

solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java

@@ -91,13 +91,23 @@ public class TestReplicationHandlerDiskOverFlow extends SolrTestCaseJ4 {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    masterJetty.stop();
-    slaveJetty.stop();
-    masterJetty = slaveJetty = null;
+    if (null != masterJetty) {
+      masterJetty.stop();
+      masterJetty = null;
+    }
+    if (null != slaveJetty) {
+      slaveJetty.stop();
+       slaveJetty = null;
+    }
     master = slave = null;
-    masterClient.close();
-    slaveClient.close();
-    masterClient = slaveClient = null;
+    if (null != masterClient) {
+      masterClient.close();
+      masterClient = null;
+    }
+    if (null != slaveClient) {
+      slaveClient.close();
+      slaveClient = null;
+    }
     System.clearProperty("solr.indexfetcher.sotimeout");
     
     IndexFetcher.usableDiskSpaceProvider = originalDiskSpaceprovider;

solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java

@@ -98,10 +98,14 @@ public class TestRestoreCore extends SolrJettyTestBase {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    masterClient.close();
-    masterClient  = null;
-    masterJetty.stop();
-    masterJetty = null;
+    if (null != masterClient) {
+      masterClient.close();
+      masterClient  = null;
+    }
+    if (null != masterJetty) {
+      masterJetty.stop();
+      masterJetty = null;
+    }
     master = null;
   }
 

solr/core/src/test/org/apache/solr/handler/admin/DaemonStreamApiTest.java

@@ -95,7 +95,10 @@ public class DaemonStreamApiTest extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    cluster.shutdown();
+    if (null != cluster) {
+      cluster.shutdown();
+      cluster = null;
+    }
     super.tearDown();
   }
 

solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java

@@ -52,9 +52,11 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
 
   @AfterClass
   public static void cleanupMetrics() throws Exception {
-    h.getCoreContainer().getMetricManager().registry("solr.jvm"  ).remove("solrtest_foo");
-    h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo");
-    h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar");
+    if (null != h) {
+      h.getCoreContainer().getMetricManager().registry("solr.jvm"  ).remove("solrtest_foo");
+      h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo");
+      h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar");
+    }
   }
   
   @Test

solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java

@@ -98,12 +98,18 @@ public class DistributedDebugComponentTest extends SolrJettyTestBase {
   
   @AfterClass
   public static void destroyThings() throws Exception {
-    collection1.close();
-    collection2.close();
-    collection1 = null;
-    collection2 = null;
-    jetty.stop();
-    jetty=null;
+    if (null != collection1) {
+      collection1.close();
+      collection1 = null;
+    }
+    if (null != collection2) {
+      collection2.close();
+      collection2 = null;
+    }
+    if (null != jetty) {
+      jetty.stop();
+      jetty=null;
+    }
     resetExceptionIgnores();
     systemClearPropertySolrDisableShardsWhitelist();
   }

solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java

@@ -87,7 +87,9 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
   @After
   public void tearDown() throws Exception {
     try {
-      directory.close();
+      if (null != directory) {
+        directory.close();
+      }
     } finally {
       try(FileSystem fs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
         fs.delete(path, true);

solr/core/src/test/org/apache/solr/legacy/TestNumericRangeQuery32.java

@@ -127,10 +127,14 @@ public class TestNumericRangeQuery32 extends SolrTestCase {
   @AfterClass
   public static void afterClass() throws Exception {
     searcher = null;
-    reader.close();
-    reader = null;
-    directory.close();
-    directory = null;
+    if (null != reader) {
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
   }
   
   @Override

solr/core/src/test/org/apache/solr/legacy/TestNumericRangeQuery64.java

@@ -136,10 +136,14 @@ public class TestNumericRangeQuery64 extends SolrTestCase {
   @AfterClass
   public static void afterClass() throws Exception {
     searcher = null;
-    reader.close();
-    reader = null;
-    directory.close();
-    directory = null;
+    if (null != reader) {
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
   }
   
   @Override

solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java

@@ -51,9 +51,11 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 {
 
   @After
   public void afterTest() throws IOException {
-    coreMetricManager.close();
-    assertTrue(metricManager.getReporters(coreMetricManager.getRegistryName()).isEmpty());
-    deleteCore();
+    if (null != coreMetricManager) {
+      coreMetricManager.close();
+      assertTrue(metricManager.getReporters(coreMetricManager.getRegistryName()).isEmpty());
+      deleteCore();
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java

@@ -109,6 +109,10 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 {
 
   @After
   public void afterTest() throws Exception {
+    if (null == metricManager) {
+      return; // test failed to init, nothing to cleanup
+    }
+      
     SolrCoreMetricManager coreMetricManager = h.getCore().getCoreMetricManager();
     Map<String, SolrMetricReporter> reporters = metricManager.getReporters(coreMetricManager.getRegistryName());
 

solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java

@@ -118,6 +118,9 @@ public class SolrJmxReporterTest extends SolrTestCaseJ4 {
 
   @After
   public void afterTest() throws Exception {
+    if (null == metricManager) {
+      return; // test failed to init, nothing to cleanup
+    }
     metricManager.closeReporters(coreMetricManager.getRegistryName());
     Set<ObjectInstance> objects =
         mBeanServer.queryMBeans(ObjectName.getInstance(domain + ":*"), null);

solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedStopFilterFactory.java

@@ -62,8 +62,10 @@ public class TestManagedStopFilterFactory extends RestTestBase {
 
   @After
   private void after() throws Exception {
-    jetty.stop();
-    jetty = null;
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
     System.clearProperty("managed.schema.mutable");
     System.clearProperty("enable.update.log");
     

solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java

@@ -61,9 +61,14 @@ public class TestManagedSynonymFilterFactory extends RestTestBase {
 
   @After
   private void after() throws Exception {
-    jetty.stop();
-    jetty = null;
-    FileUtils.deleteDirectory(tmpSolrHome);
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
+    if (null != tmpSolrHome) {
+      FileUtils.deleteDirectory(tmpSolrHome);
+      tmpSolrHome = null;
+    }
     System.clearProperty("managed.schema.mutable");
     System.clearProperty("enable.update.log");
     

solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java

@@ -63,9 +63,13 @@ public class TestManagedSynonymGraphFilterFactory extends RestTestBase {
 
   @After
   private void after() throws Exception {
-    jetty.stop();
-    jetty = null;
-    FileUtils.deleteDirectory(tmpSolrHome);
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
+    if (null != tmpSolrHome) {
+      FileUtils.deleteDirectory(tmpSolrHome);
+    }
     System.clearProperty("managed.schema.mutable");
     System.clearProperty("enable.update.log");
 

solr/core/src/test/org/apache/solr/schema/TestManagedSchemaThreadSafety.java

@@ -97,8 +97,10 @@ public class TestManagedSchemaThreadSafety extends SolrTestCaseJ4 {
 
   @AfterClass
   public static void stopZkServer() throws Exception {
-    zkServer.shutdown();
-    zkServer = null;
+    if (null != zkServer) {
+      zkServer.shutdown();
+      zkServer = null;
+    }
     loaderPath = null;
   }
 

solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java

@@ -109,7 +109,6 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
   @After
   private void afterTest() throws Exception {
     clearIndex();
-    assertU(commit());
     deleteCore();
     System.clearProperty("managed.schema.mutable");
     System.clearProperty("enable.update.log");

solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java

@@ -184,7 +184,10 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
   
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
     for (HttpSolrClient client : CLIENTS) {
       client.close();
     }

solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java

@@ -206,7 +206,10 @@ public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
   
   @AfterClass
   private static void afterClass() throws Exception {
-    CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+    if (null != CLOUD_CLIENT) {
+      CLOUD_CLIENT.close();
+      CLOUD_CLIENT = null;
+    }
     for (HttpSolrClient client : CLIENTS) {
       client.close();
     }

solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java

@@ -209,7 +209,9 @@ public class TestOrdValues extends SolrTestCase {
 
   @AfterClass
   public static void afterClassFunctionTestSetup() throws Exception {
-    dir.close();
+    if (null != dir) {
+      dir.close();
+    }
     dir = null;
     anlzr = null;
   }

solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetRandomTest.java

@@ -178,9 +178,11 @@ public class BlockJoinFacetRandomTest extends SolrTestCaseJ4 {
 
   @AfterClass
   public static void cleanUp() throws Exception {
-    assertU(delQ("*:*"));
-    optimize();
-    assertU((commit()));
+    if (null != h) {
+      assertU(delQ("*:*"));
+      optimize();
+      assertU((commit()));
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java

@@ -90,7 +90,9 @@ public class CloudMLTQParserTest extends SolrCloudTestCase {
   
   @After
   public void cleanCluster() throws Exception {
-    cluster.shutdown();
+    if (null != cluster) {
+      cluster.shutdown();
+    }
   }
 
   public static final String COLLECTION = "mlt-collection";

solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java

@@ -87,7 +87,9 @@ public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase {
   @Override
   @After
   public void tearDown() throws Exception {
-    testHarness.get().close();
+    if (null != testHarness.get()) {
+      testHarness.get().close();
+    }
     super.tearDown();
   }
   

solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java

@@ -107,7 +107,10 @@ public class AuditLoggerPluginTest extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    plugin.close();
+    if (null != plugin) {
+      plugin.close();
+      plugin = null;
+    }
     super.tearDown();
   }
 
@@ -173,4 +176,4 @@ public class AuditLoggerPluginTest extends SolrTestCaseJ4 {
         plugin.formatter.formatEvent(EVENT_AUTHENTICATED));
   } 
   
-}
+}

solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java

@@ -76,7 +76,10 @@ public class BasicAuthStandaloneTest extends SolrTestCaseJ4 {
   @Override
   @After
   public void tearDown() throws Exception {
-    jetty.stop();
+    if (null != jetty) {
+      jetty.stop();
+      jetty = null;
+    }
     super.tearDown();
   }
 

solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java

@@ -137,7 +137,10 @@ public class JWTAuthPluginTest extends SolrTestCaseJ4 {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    plugin.close();
+    if (null != plugin) {
+      plugin.close();
+      plugin = null;
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java

@@ -67,6 +67,9 @@ public class SolrLogAuditLoggerPluginTest extends SolrTestCaseJ4 {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    plugin.close();
+    if (null != plugin) {
+      plugin.close();
+      plugin = null;
+    }
   }
-}
+}

solr/core/src/test/org/apache/solr/servlet/ResponseHeaderTest.java

@@ -51,7 +51,9 @@ public class ResponseHeaderTest extends SolrJettyTestBase {
   
   @AfterClass
   public static void afterTest() throws Exception {
-    cleanUpJettyHome(solrHomeDirectory);
+    if (null != solrHomeDirectory) {
+      cleanUpJettyHome(solrHomeDirectory);
+    }
   }
   
   @Test

solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java

@@ -129,7 +129,10 @@ public class BlockDirectoryTest extends SolrTestCaseJ4 {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    directory.close();
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java

@@ -126,10 +126,14 @@ public class TestFieldCache extends SolrTestCase {
 
   @AfterClass
   public static void afterClass() throws Exception {
-    reader.close();
-    reader = null;
-    directory.close();
-    directory = null;
+    if (null != reader) {
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
     unicodeStrings = null;
     multiValued = null;
   }

solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java

@@ -95,10 +95,14 @@ public class TestLegacyFieldCache extends SolrTestCase {
 
   @AfterClass
   public static void afterClass() throws Exception {
-    reader.close();
-    reader = null;
-    directory.close();
-    directory = null;
+    if (null != reader) {
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
   }
 
   public void test() throws IOException {

solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java

@@ -107,11 +107,15 @@ public class TestNumericTerms32 extends SolrTestCase {
   @AfterClass
   public static void afterClass() throws Exception {
     searcher = null;
-    TestUtil.checkReader(reader);
-    reader.close();
-    reader = null;
-    directory.close();
-    directory = null;
+    if (null != reader) {
+      TestUtil.checkReader(reader);
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
   }
   
   private void testSorting(int precisionStep) throws Exception {

solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java

@@ -112,11 +112,15 @@ public class TestNumericTerms64 extends SolrTestCase {
   @AfterClass
   public static void afterClass() throws Exception {
     searcher = null;
-    TestUtil.checkReader(reader);
-    reader.close();
-    reader = null;
-    directory.close();
-    directory = null;
+    if (null != reader) {
+      TestUtil.checkReader(reader);
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
   }
   
   private void testSorting(int precisionStep) throws Exception {

solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java

@@ -153,9 +153,10 @@ public class AddBlockUpdateTest extends SolrTestCaseJ4 {
 
   @AfterClass
   public static void afterClass() throws Exception {
-    exe.shutdownNow();
-
-    exe = null;
+    if (null != exe) {
+      exe.shutdownNow();
+      exe = null;
+    }
     inputFactory = null;
   }
 

solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java

@@ -96,8 +96,10 @@ public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
 
   @After
   public void tearDown() throws Exception {
-    monitor.assertSaneOffers();
-    monitor.clear();
+    if (null != monitor) {
+      monitor.assertSaneOffers();
+      monitor.clear();
+    }
     super.tearDown();
     System.clearProperty("solr.ulog");
     deleteCore();

solr/core/src/test/org/apache/solr/update/processor/CategoryRoutedAliasUpdateProcessorTest.java

@@ -40,7 +40,6 @@ import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.update.UpdateCommand;
 import org.apache.solr.util.LogLevel;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -80,13 +79,10 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
 
   @After
   public void doAfter() throws Exception {
-    solrClient.close();
-    shutdownCluster();
-  }
-
-  @AfterClass
-  public static void finish() throws Exception {
     IOUtils.close(solrClient);
+    if (null != cluster) {
+      shutdownCluster();
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorTest.java

@@ -73,9 +73,18 @@ public class ClassificationUpdateProcessorTest extends SolrTestCaseJ4 {
 
   @Override
   public void tearDown() throws Exception {
-    reader.close();
-    directory.close();
-    analyzer.close();
+    if (null != reader) {
+      reader.close();
+      reader = null;
+    }
+    if (null != directory) {
+      directory.close();
+      directory = null;
+    }
+    if (null != analyzer) {
+      analyzer.close();
+      analyzer = null;
+    }
     super.tearDown();
   }
 

solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java

@@ -90,7 +90,9 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
 
   @After
   public void doAfter() throws Exception {
-    solrClient.close();
+    if (null != solrClient) {
+      solrClient.close();
+    }
     shutdownCluster();
   }
 

solr/core/src/test/org/apache/solr/util/AuthToolTest.java

@@ -52,7 +52,10 @@ public class AuthToolTest extends SolrCloudTestCase {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile());
+    if (null != dir) {
+      org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile());
+      dir = null;
+    }
   }
 
   @Test

solr/core/src/test/org/apache/solr/util/UtilsToolTest.java

@@ -77,7 +77,10 @@ public class UtilsToolTest extends SolrTestCaseJ4 {
   @After
   public void tearDown() throws Exception {
     super.tearDown();
-    org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile());
+    if (null != dir) {
+      org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile());
+      dir = null;
+    }
   }
   
   @Test

solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientConPoolTest.java

@@ -58,8 +58,10 @@ public class HttpSolrClientConPoolTest extends SolrJettyTestBase {
   
   @AfterClass
   public static void stopYetty() throws Exception {
-    yetty.stop();
-    yetty = null;
+    if (null != yetty) {
+      yetty.stop();
+      yetty = null;
+    }
   }
   
   public void testPoolSize() throws SolrServerException, IOException {

solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java

@@ -58,7 +58,9 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase {
   
   @After
   public void tearDownCluster() throws Exception {
-    executor.shutdown();
+    if (null != executor) {
+      executor.shutdown();
+    }
     shutdownCluster();
     executor = null;
   }

solr/solrj/src/test/org/apache/solr/common/cloud/TestDocCollectionWatcher.java

@@ -59,7 +59,9 @@ public class TestDocCollectionWatcher extends SolrCloudTestCase {
   
   @After
   public void tearDownCluster() throws Exception {
-    executor.shutdown();
+    if (null!= executor) {
+      executor.shutdown();
+    }
     shutdownCluster();
     executor = null;
   }

solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java

@@ -49,7 +49,9 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {
 
   @AfterClass
   public static void shutdownZkServer() throws IOException, InterruptedException {
-    zkServer.shutdown();
+    if (null != zkServer) {
+      zkServer.shutdown();
+    }
     zkServer = null;
   }
 

solr/test-framework/src/java/org/apache/solr/SolrTestCase.java

@@ -22,9 +22,13 @@ import java.lang.invoke.MethodHandles;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.util.StartupLoggingUtils;
 import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
+
 /**
  * All Solr test cases should derive from this class eventually. This is originally a result of async logging, see:
  * SOLR-12055 and associated. To enable async logging, we must gracefully shut down logging. Many Solr tests subclass
@@ -41,6 +45,32 @@ public class SolrTestCase extends LuceneTestCase {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  /** 
+   * Special hook for sanity checking if any tests trigger failures when an
+   * Assumption failure occures in a {@link BeforeClass} method
+   * @lucene.internal
+   */
+  @BeforeClass
+  public static void checkSyspropForceBeforeClassAssumptionFailure() {
+    // ant test -Dargs="-Dtests.force.assumption.failure.beforeclass=true"
+    final String PROP = "tests.force.assumption.failure.beforeclass";
+    assumeFalse(PROP + " == true",
+                systemPropertyAsBoolean(PROP, false));
+  }
+  
+  /** 
+   * Special hook for sanity checking if any tests trigger failures when an
+   * Assumption failure occures in a {@link Before} method
+   * @lucene.internal
+   */
+  @Before
+  public void checkSyspropForceBeforeAssumptionFailure() {
+    // ant test -Dargs="-Dtests.force.assumption.failure.before=true"
+    final String PROP = "tests.force.assumption.failure.before";
+    assumeFalse(PROP + " == true",
+                systemPropertyAsBoolean(PROP, false));
+  }
+  
   @AfterClass
   public static void shutdownLogger() throws Exception {
     StartupLoggingUtils.shutdown();

solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java

@@ -325,7 +325,9 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
         log.error("Error deleting SolrCore.");
       }
       
-      ExecutorUtil.shutdownAndAwaitTermination(testExecutor);
+      if (null != testExecutor) {
+        ExecutorUtil.shutdownAndAwaitTermination(testExecutor);
+      }
 
       resetExceptionIgnores();
 
@@ -1286,6 +1288,11 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
    * @see #deleteByQueryAndGetVersion
    */
   public void clearIndex() {
+    if (null == h) {
+      // harness not initialized, treat as No-Op so safe to call in cleanup methods
+      // even if no tests run
+      return;
+    }
     try {
       deleteByQueryAndGetVersion("*:*", params("_version_", Long.toString(-Long.MAX_VALUE),
                                                DISTRIB_UPDATE_PARAM,DistribPhase.FROMLEADER.toString()));