SOLR-13510: Intermittent 401's for internode requests with basicauth enabled

solr/CHANGES.txt

@@ -128,6 +128,8 @@ Bug Fixes
 * SOLR-13489: Stop the leader from trying to rejoin the election on session expiration and harden our zk reconnect code path.
   (Mark Miller, Anshum Gupta)
 
+* SOLR-13510: Intermittent 401's for internode requests with basicauth enabled (Cao Manh Dat, Colvin Cowie)
+
 Other Changes
 ----------------------
 

solr/core/src/test/org/apache/solr/security/BasicAuthOnSingleNodeTest.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.security;
+
+import java.lang.invoke.MethodHandles;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudAuthTestCase;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public class BasicAuthOnSingleNodeTest extends SolrCloudAuthTestCase {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static final String COLLECTION = "authCollection";
+
+  static {
+    System.setProperty("basicauth", "solr:solr");
+  }
+
+  @Before
+  public void setupCluster() throws Exception {
+    configureCluster(1)
+        .addConfig("conf", configset("cloud-minimal"))
+        .configure();
+    CollectionAdminRequest.createCollection(COLLECTION, "conf", 4, 1)
+        .setMaxShardsPerNode(100)
+        .process(cluster.getSolrClient());
+    cluster.waitForActiveCollection(COLLECTION, 4, 4);
+
+    zkClient().setData("/security.json", STD_CONF.getBytes(UTF_8), true);
+
+    JettySolrRunner jetty = cluster.getJettySolrRunner(0);
+    jetty.stop();
+    cluster.waitForJettyToStop(jetty);
+    jetty.start();
+    cluster.waitForAllNodes(30);
+    cluster.waitForActiveCollection(COLLECTION, 4, 4);
+  }
+
+  @Test
+  public void basicTest() throws Exception {
+    try (Http2SolrClient client = new Http2SolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString())
+        .build()){
+      PreemptiveBasicAuthClientBuilderFactory factory = new PreemptiveBasicAuthClientBuilderFactory();
+      factory.setup(client);
+
+      // SOLR-13510, this will be failed if the listener (handling inject credential in header) is called in another
+      // thread since SolrRequestInfo will return null in that case.
+      for (int i = 0; i < 30; i++) {
+        client.query(COLLECTION, new SolrQuery("*:*"));
+      }
+    }
+  }
+
+  protected static final String STD_CONF = "{\n" +
+      "  \"authentication\":{\n" +
+      "   \"blockUnknown\": true,\n" +
+      "   \"class\":\"solr.BasicAuthPlugin\",\n" +
+      "   \"credentials\":{\"solr\":\"EEKn7ywYk5jY8vG9TyqlG2jvYuvh1Q7kCCor6Hqm320= 6zkmjMjkMKyJX6/f0VarEWQujju5BzxZXub6WOrEKCw=\"}\n" +
+      "  },\n" +
+      "  \"authorization\":{\n" +
+      "   \"class\":\"solr.RuleBasedAuthorizationPlugin\",\n" +
+      "   \"permissions\":[\n" +
+      " {\"name\":\"security-edit\", \"role\":\"admin\"},\n" +
+      " {\"name\":\"collection-admin-edit\", \"role\":\"admin\"},\n" +
+      " {\"name\":\"core-admin-edit\", \"role\":\"admin\"}\n" +
+      "   ],\n" +
+      "   \"user-role\":{\"solr\":\"admin\"}\n" +
+      "  }\n" +
+      "}";
+}
+
+

solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java

@@ -465,7 +465,7 @@ public class Http2SolrClient extends SolrClient {
     setBasicAuthHeader(solrRequest, req);
     for (HttpListenerFactory factory : listenerFactory) {
       HttpListenerFactory.RequestResponseListener listener = factory.get();
-      req.onRequestQueued(listener);
+      listener.onQueued(req);
       req.onRequestBegin(listener);
       req.onComplete(listener);
     }

solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpListenerFactory.java

@@ -23,9 +23,23 @@ import org.eclipse.jetty.client.api.Result;
 
 public interface HttpListenerFactory {
   abstract class RequestResponseListener implements Request.BeginListener, Response.CompleteListener, Request.QueuedListener {
+
+    /**
+     * Callback method invoked when the request begins being processed in order to be sent.
+     * This is the last opportunity to modify the request.
+     * This method will NOT be ensured to be called on the same thread as the thread calling {@code Http2SolrClient} methods.
+     *
+     * @param request the request that begins being processed
+     */
     @Override
     public void onBegin(Request request){}
 
+    /**
+     * Callback method invoked when the request is queued, waiting to be sent.
+     * This method will be ensured to be called on the same thread as the thread calling {@code Http2SolrClient} methods.
+     *
+     * @param request the request being queued
+     */
     @Override
     public void onQueued(Request request) {}