SOLR-7955: Auto create .system collection on first request if it does not exist

solr/CHANGES.txt

@@ -124,6 +124,9 @@ New Features
 * SOLR-9933: SolrCoreParser now supports configuration of custom SpanQueryBuilder classes.
   (Daniel Collins, Christine Poerschke)
 
+* SOLR-7955: Auto create .system collection on first request if it does not exist (noble)
+
+
 Bug Fixes
 ----------------------
 

solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java

@@ -18,7 +18,6 @@ package org.apache.solr.servlet;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -37,6 +36,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.CloseShieldInputStream;
@@ -73,16 +73,17 @@ import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.ValidatingJsonMap;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.Utils;
+import org.apache.solr.common.util.ValidatingJsonMap;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.handler.ContentStreamHandlerBase;
 import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.request.SolrRequestHandler;
@@ -103,6 +104,7 @@ import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
 import org.apache.solr.util.CommandOperation;
 import org.apache.solr.util.JsonSchemaValidator;
 import org.apache.solr.util.RTimerTree;
+import org.apache.solr.util.TimeOut;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -111,9 +113,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.params.CoreAdminParams.ACTION;
+import static org.apache.solr.handler.admin.CollectionsHandler.SYSTEM_COLL;
 import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
 import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD;
 import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH;
@@ -296,6 +302,9 @@ public class HttpSolrCall {
       // if we couldn't find it locally, look on other nodes
       extractRemotePath(corename, origCorename, idx);
       if (action != null) return;
+      //core is not available locally or remotely
+      autoCreateSystemColl();
+      if(action != null) return;
     }
 
     // With a valid core...
@@ -331,6 +340,35 @@ public class HttpSolrCall {
     action = PASSTHROUGH;
   }
 
+  protected void autoCreateSystemColl() throws Exception {
+    if (SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod()) && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
+      log.info("Going to auto-create .system collection");
+      SolrQueryResponse rsp = new SolrQueryResponse();
+      String repFactor = String.valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size()));
+      cores.getCollectionsHandler().handleRequestBody(new LocalSolrQueryRequest(null,
+          new ModifiableSolrParams()
+              .add(ACTION, CREATE.toString())
+              .add( NAME, SYSTEM_COLL)
+              .add(REPLICATION_FACTOR, repFactor)), rsp);
+      if (rsp.getValues().get("success") == null) {
+        throw new SolrException(ErrorCode.SERVER_ERROR, "Could not auto-create .system collection: "+ Utils.toJSONString(rsp.getValues()));
+      }
+      TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS);
+      for (; ; ) {
+        if (cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) != null) {
+          break;
+        } else {
+          if (timeOut.hasTimedOut()) {
+            throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find .system collection even after 3 seconds");
+          }
+          Thread.sleep(50);
+        }
+      }
+
+      action = RETRY;
+    }
+  }
+
   protected String lookupAliases(String collName) {
     ZkStateReader reader = cores.getZkController().getZkStateReader();
     aliases = reader.getAliases();

solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java

@@ -90,6 +90,13 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
           "field",
           "type")));
 
+      checkBlobPost(baseUrl, cloudClient);
+    }
+  }
+
+  static void checkBlobPost(String baseUrl, CloudSolrClient cloudClient) throws Exception {
+    String url;
+    Map map;
     byte[] bytarr = new byte[1024];
     for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127);
     byte[] bytarr2 = new byte[2048];
@@ -106,9 +113,8 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
     map = (Map) l.get(0);
     assertEquals("" + bytarr.length, String.valueOf(map.get("size")));
 
-      compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2);
-      compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr);
-    }
+    compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2, cloudClient);
+    compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr, cloudClient);
   }
 
   public static void createSystemCollection(SolrClient client) throws SolrServerException, IOException {
@@ -152,7 +158,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
     return new String(Utils.toJSON(map), StandardCharsets.UTF_8);
   }
 
-  private void compareInputAndOutput(String url, byte[] bytarr) throws IOException {
+  static void compareInputAndOutput(String url, byte[] bytarr, CloudSolrClient cloudClient) throws IOException {
 
     HttpClient httpClient = cloudClient.getLbClient().getHttpClient();
 

solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.cloud.DocCollection;
+
+public class TestSystemCollAutoCreate extends AbstractFullDistribZkTestBase {
+  public void testAutoCreate() throws Exception {
+    TestBlobHandler.checkBlobPost(cloudJettys.get(0).jetty.getBaseUrl().toExternalForm(), cloudClient);
+    DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
+  }
+}