SOLR-13464: Test work arounds

* Refactor existing work around in BasicAuthIntegrationTest up into SolrCloudAuthTestCase for re-use in JWTAuthPluginIntegrationTest

 * Simplify BasicAuthOnSingleNodeTest and PKIAuthenticationIntegrationTest to use their existing (static) security settings on creation of MiniSolrCloud.  Since they no longer modify security.json once the nodes are alive, the issue no longer affects them

solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java

@@ -24,7 +24,6 @@ import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -347,32 +346,16 @@ public class BasicAuthIntegrationTest extends SolrCloudAuthTestCase {
     update.commit(cluster.getSolrClient(), COLLECTION);
   }
 
-  /** @see #executeCommand */
-  private static Map<String,Object> getAuthPlugins(String url) {
-    Map<String,Object> plugins = new HashMap<>();
-    if (url.endsWith("authentication")) {
-      for (JettySolrRunner r : cluster.getJettySolrRunners()) {
-        plugins.put(r.getNodeName(), r.getCoreContainer().getAuthenticationPlugin());
-      }
-    } else if (url.endsWith("authorization")) {
-      for (JettySolrRunner r : cluster.getJettySolrRunners()) {
-        plugins.put(r.getNodeName(), r.getCoreContainer().getAuthorizationPlugin());
-      }
-    } else {
-      fail("Test helper method assumptions broken: " + url);
-    }
-    return plugins;
-  }
-  
   public static void executeCommand(String url, HttpClient cl, String payload,
                                     String user, String pwd) throws Exception {
 
-    // HACK: (attempted) work around for SOLR-13464...
+    // HACK: work around for SOLR-13464...
     //
     // note the authz/authn objects in use on each node before executing the command,
     // then wait until we see new objects on every node *after* executing the command
     // before returning...
-    final Set<Map.Entry<String,Object>> initialPlugins = getAuthPlugins(url).entrySet();
+    final Set<Map.Entry<String,Object>> initialPlugins
+      = getAuthPluginsInUseForCluster(url).entrySet();
     
     HttpPost httpPost;
     HttpResponse r;
@@ -387,10 +370,11 @@ public class BasicAuthIntegrationTest extends SolrCloudAuthTestCase {
     Utils.consumeFully(r.getEntity());
 
     // HACK (continued)...
-    final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+    final TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
     timeout.waitFor("core containers never fully updated their auth plugins",
                     () -> {
-                      final Set<Map.Entry<String,Object>> tmpSet = getAuthPlugins(url).entrySet();
+                      final Set<Map.Entry<String,Object>> tmpSet
+                        = getAuthPluginsInUseForCluster(url).entrySet();
                       tmpSet.retainAll(initialPlugins);
                       return tmpSet.isEmpty();
                     });

solr/core/src/test/org/apache/solr/security/BasicAuthOnSingleNodeTest.java

@@ -30,8 +30,6 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 public class BasicAuthOnSingleNodeTest extends SolrCloudAuthTestCase {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -45,14 +43,14 @@ public class BasicAuthOnSingleNodeTest extends SolrCloudAuthTestCase {
   public void setupCluster() throws Exception {
     configureCluster(1)
         .addConfig("conf", configset("cloud-minimal"))
+        .withSecurityJson(STD_CONF)
         .configure();
     CollectionAdminRequest.createCollection(COLLECTION, "conf", 4, 1)
         .setMaxShardsPerNode(100)
+        .setBasicAuthCredentials("solr", "solr")
         .process(cluster.getSolrClient());
     cluster.waitForActiveCollection(COLLECTION, 4, 4);
 
-    zkClient().setData("/security.json", STD_CONF.getBytes(UTF_8), true);
-
     JettySolrRunner jetty = cluster.getJettySolrRunner(0);
     jetty.stop();
     cluster.waitForJettyToStop(jetty);

solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java

@@ -24,6 +24,9 @@ import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
@@ -38,14 +41,15 @@ import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.cloud.SolrCloudAuthTestCase;
 import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.TimeOut;
 import org.jose4j.jwk.PublicJsonWebKey;
 import org.jose4j.jwk.RsaJsonWebKey;
 import org.jose4j.jwk.RsaJwkGenerator;
 import org.jose4j.jws.AlgorithmIdentifiers;
 import org.jose4j.jws.JsonWebSignature;
 import org.jose4j.jwt.JwtClaims;
-import org.jose4j.lang.JoseException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -238,7 +242,17 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase {
     cluster.waitForActiveCollection(collectionName, 2, 2);
   }
 
-  private void executeCommand(String url, HttpClient cl, String payload, JsonWebSignature jws) throws IOException, JoseException {
+  private void executeCommand(String url, HttpClient cl, String payload, JsonWebSignature jws)
+    throws Exception {
+    
+    // HACK: work around for SOLR-13464...
+    //
+    // note the authz/authn objects in use on each node before executing the command,
+    // then wait until we see new objects on every node *after* executing the command
+    // before returning...
+    final Set<Map.Entry<String,Object>> initialPlugins
+      = getAuthPluginsInUseForCluster(url).entrySet();
+    
     HttpPost httpPost;
     HttpResponse r;
     httpPost = new HttpPost(url);
@@ -251,5 +265,16 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase {
     assertEquals("Non-200 response code. Response was " + response, 200, r.getStatusLine().getStatusCode());
     assertFalse("Response contained errors: " + response, response.contains("errorMessages"));
     Utils.consumeFully(r.getEntity());
+
+    // HACK (continued)...
+    final TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+    timeout.waitFor("core containers never fully updated their auth plugins",
+                    () -> {
+                      final Set<Map.Entry<String,Object>> tmpSet
+                        = getAuthPluginsInUseForCluster(url).entrySet();
+                      tmpSet.retainAll(initialPlugins);
+                      return tmpSet.isEmpty();
+                    });
+    
   }
 }

solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java

@@ -26,7 +26,6 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.cloud.SolrCloudAuthTestCase;
-import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.Utils;
 import org.junit.After;
@@ -45,9 +44,14 @@ public class PKIAuthenticationIntegrationTest extends SolrCloudAuthTestCase {
   
   @BeforeClass
   public static void setupCluster() throws Exception {
+    final String SECURITY_CONF = Utils.toJSONString
+      (makeMap("authorization", singletonMap("class", MockAuthorizationPlugin.class.getName()),
+               "authentication", singletonMap("class", MockAuthenticationPlugin.class.getName())));
+    
     configureCluster(2)
-        .addConfig("conf", configset("cloud-minimal"))
-        .configure();
+      .addConfig("conf", configset("cloud-minimal"))
+      .withSecurityJson(SECURITY_CONF)
+      .configure();
 
     CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1).process(cluster.getSolrClient());
 
@@ -56,11 +60,6 @@ public class PKIAuthenticationIntegrationTest extends SolrCloudAuthTestCase {
 
   @Test
   public void testPkiAuth() throws Exception {
-    // TODO make a SolrJ helper class for this
-    byte[] bytes = Utils.toJSON(makeMap("authorization", singletonMap("class", MockAuthorizationPlugin.class.getName()),
-        "authentication", singletonMap("class", MockAuthenticationPlugin.class.getName())));
-    zkClient().setData(ZkStateReader.SOLR_SECURITY_CONF_PATH, bytes, true);
-
     HttpClient httpClient = cluster.getSolrClient().getHttpClient();
     for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
       String baseUrl = jetty.getBaseUrl().toString();

solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java

@@ -39,6 +39,7 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.message.AbstractHttpMessage;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.common.util.Base64;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.Utils;
@@ -235,4 +236,37 @@ public class SolrCloudAuthTestCase extends SolrCloudTestCase {
     httpMsg.setHeader(new BasicHeader("Authorization", headerString));
     log.info("Added Authorization Header {}", headerString);
   }
+
+  /**
+   * This helper method can be used by tests to monitor the current state of either 
+   * <code>"authentication"</code> or <code>"authorization"</code> plugins in use each
+   * node of the current cluster.
+   * <p>
+   * This can be useful in a {@line TimeOut#waitFor} loop to monitor a cluster and "wait for"
+   * A change in security settings to affect all nodes by comparing the objects in the current 
+   * Map with the one in use prior to executing some test command. (providing a work around 
+   * for the security user experienence limitations identified in
+   * <a href="https://issues.apache.org/jira/browse/SOLR-13464">SOLR-13464</a> )
+   * </p>
+   * 
+   * @param url A REST url (or any arbitrary String) ending in 
+   *    <code>"authentication"</code> or <code>"authorization"</code> used to specify the type of
+   *    plugins to introspect
+   * @return A Map from <code>nodeName</code> to auth plugin
+   */
+  public static Map<String,Object> getAuthPluginsInUseForCluster(String url) {
+    Map<String,Object> plugins = new HashMap<>();
+    if (url.endsWith("authentication")) {
+      for (JettySolrRunner r : cluster.getJettySolrRunners()) {
+        plugins.put(r.getNodeName(), r.getCoreContainer().getAuthenticationPlugin());
+      }
+    } else if (url.endsWith("authorization")) {
+      for (JettySolrRunner r : cluster.getJettySolrRunners()) {
+        plugins.put(r.getNodeName(), r.getCoreContainer().getAuthorizationPlugin());
+      }
+    } else {
+      fail("Test helper method assumptions broken: " + url);
+    }
+    return plugins;
+  }
 }