diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 44c5316a502..b4f9016c6e8 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -124,6 +124,9 @@ New Features * SOLR-9933: SolrCoreParser now supports configuration of custom SpanQueryBuilder classes. (Daniel Collins, Christine Poerschke) +* SOLR-7955: Auto create .system collection on first request if it does not exist (noble) + + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index b2440153933..521c7bb1adf 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -18,7 +18,6 @@ package org.apache.solr.servlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; - import java.io.EOFException; import java.io.IOException; import java.io.InputStream; @@ -37,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.CloseShieldInputStream; @@ -73,16 +73,17 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; -import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.ContentStreamHandlerBase; import org.apache.solr.logging.MDCLoggingContext; +import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; import org.apache.solr.request.SolrRequestHandler; @@ -103,6 +104,7 @@ import org.apache.solr.update.processor.DistributingUpdateProcessorFactory; import org.apache.solr.util.CommandOperation; import org.apache.solr.util.JsonSchemaValidator; import org.apache.solr.util.RTimerTree; +import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,9 +113,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.params.CoreAdminParams.ACTION; +import static org.apache.solr.handler.admin.CollectionsHandler.SYSTEM_COLL; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH; @@ -296,6 +302,9 @@ public class HttpSolrCall { // if we couldn't find it locally, look on other nodes extractRemotePath(corename, origCorename, idx); if (action != null) return; + //core is not available locally or remotely + autoCreateSystemColl(); + if(action != null) return; } // With a valid core... @@ -331,6 +340,35 @@ public class HttpSolrCall { action = PASSTHROUGH; } + protected void autoCreateSystemColl() throws Exception { + if (SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod()) && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) { + log.info("Going to auto-create .system collection"); + SolrQueryResponse rsp = new SolrQueryResponse(); + String repFactor = String.valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size())); + cores.getCollectionsHandler().handleRequestBody(new LocalSolrQueryRequest(null, + new ModifiableSolrParams() + .add(ACTION, CREATE.toString()) + .add( NAME, SYSTEM_COLL) + .add(REPLICATION_FACTOR, repFactor)), rsp); + if (rsp.getValues().get("success") == null) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Could not auto-create .system collection: "+ Utils.toJSONString(rsp.getValues())); + } + TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS); + for (; ; ) { + if (cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) != null) { + break; + } else { + if (timeOut.hasTimedOut()) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find .system collection even after 3 seconds"); + } + Thread.sleep(50); + } + } + + action = RETRY; + } + } + protected String lookupAliases(String collName) { ZkStateReader reader = cores.getZkController().getZkStateReader(); aliases = reader.getAliases(); diff --git a/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java b/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java index 4fda9268d8b..c395d205b64 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java @@ -90,27 +90,33 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase { "field", "type"))); - byte[] bytarr = new byte[1024]; - for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127); - byte[] bytarr2 = new byte[2048]; - for (int i = 0; i < bytarr2.length; i++) bytarr2[i] = (byte) (i % 127); - String blobName = "test"; - postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr), 1); - postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr2), 2); - - url = baseUrl + "/.system/blob/test/1"; - map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient); - List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs")); - assertNotNull("" + map, l); - assertTrue("" + map, l.size() > 0); - map = (Map) l.get(0); - assertEquals("" + bytarr.length, String.valueOf(map.get("size"))); - - compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2); - compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr); + checkBlobPost(baseUrl, cloudClient); } } + static void checkBlobPost(String baseUrl, CloudSolrClient cloudClient) throws Exception { + String url; + Map map; + byte[] bytarr = new byte[1024]; + for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127); + byte[] bytarr2 = new byte[2048]; + for (int i = 0; i < bytarr2.length; i++) bytarr2[i] = (byte) (i % 127); + String blobName = "test"; + postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr), 1); + postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr2), 2); + + url = baseUrl + "/.system/blob/test/1"; + map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient); + List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs")); + assertNotNull("" + map, l); + assertTrue("" + map, l.size() > 0); + map = (Map) l.get(0); + assertEquals("" + bytarr.length, String.valueOf(map.get("size"))); + + compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2, cloudClient); + compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr, cloudClient); + } + public static void createSystemCollection(SolrClient client) throws SolrServerException, IOException { CollectionAdminResponse response1; CollectionAdminRequest.Create createCollectionRequest = new CollectionAdminRequest.Create() @@ -152,7 +158,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase { return new String(Utils.toJSON(map), StandardCharsets.UTF_8); } - private void compareInputAndOutput(String url, byte[] bytarr) throws IOException { + static void compareInputAndOutput(String url, byte[] bytarr, CloudSolrClient cloudClient) throws IOException { HttpClient httpClient = cloudClient.getLbClient().getHttpClient(); diff --git a/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java b/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java new file mode 100644 index 00000000000..cadda583d83 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler; + + +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; +import org.apache.solr.common.cloud.DocCollection; + +public class TestSystemCollAutoCreate extends AbstractFullDistribZkTestBase { + public void testAutoCreate() throws Exception { + TestBlobHandler.checkBlobPost(cloudJettys.get(0).jetty.getBaseUrl().toExternalForm(), cloudClient); + DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system"); + } +}