Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-7955: Auto create .system collection on first request if it does not exist

This commit is contained in:
Noble Paul 2017-02-02 15:26:26 +10:30
parent c1d9b87d7d
commit e200b8a2a4
4 changed files with 97 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -124,6 +124,9 @@ New Features
* SOLR-9933: SolrCoreParser now supports configuration of custom SpanQueryBuilder classes.
(Daniel Collins, Christine Poerschke)
* SOLR-7955: Auto create .system collection on first request if it does not exist (noble)
Bug Fixes
----------------------

42
solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
View File

@ -18,7 +18,6 @@ package org.apache.solr.servlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
@ -37,6 +36,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
@ -73,16 +73,17 @@ import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ContentStreamHandlerBase;
import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestHandler;
@ -103,6 +104,7 @@ import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.CommandOperation;
import org.apache.solr.util.JsonSchemaValidator;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -111,9 +113,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.CoreAdminParams.ACTION;
import static org.apache.solr.handler.admin.CollectionsHandler.SYSTEM_COLL;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH;
@ -296,6 +302,9 @@ public class HttpSolrCall {
// if we couldn't find it locally, look on other nodes
extractRemotePath(corename, origCorename, idx);
if (action != null) return;
//core is not available locally or remotely
autoCreateSystemColl();
if(action != null) return;
}
// With a valid core...
@ -331,6 +340,35 @@ public class HttpSolrCall {
action = PASSTHROUGH;
}
protected void autoCreateSystemColl() throws Exception {
if (SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod()) && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
log.info("Going to auto-create .system collection");
SolrQueryResponse rsp = new SolrQueryResponse();
String repFactor = String.valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size()));
cores.getCollectionsHandler().handleRequestBody(new LocalSolrQueryRequest(null,
new ModifiableSolrParams()
.add(ACTION, CREATE.toString())
.add( NAME, SYSTEM_COLL)
.add(REPLICATION_FACTOR, repFactor)), rsp);
if (rsp.getValues().get("success") == null) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not auto-create .system collection: "+ Utils.toJSONString(rsp.getValues()));
}
TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS);
for (; ; ) {
if (cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) != null) {
break;
} else {
if (timeOut.hasTimedOut()) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find .system collection even after 3 seconds");
}
Thread.sleep(50);
}
}
action = RETRY;
}
}
protected String lookupAliases(String collName) {
ZkStateReader reader = cores.getZkController().getZkStateReader();
aliases = reader.getAliases();

44
solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
View File

@ -90,27 +90,33 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
"field",
"type")));
byte[] bytarr = new byte[1024];
for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127);
byte[] bytarr2 = new byte[2048];
for (int i = 0; i < bytarr2.length; i++) bytarr2[i] = (byte) (i % 127);
String blobName = "test";
postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr), 1);
postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr2), 2);
url = baseUrl + "/.system/blob/test/1";
map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs"));
assertNotNull("" + map, l);
assertTrue("" + map, l.size() > 0);
map = (Map) l.get(0);
assertEquals("" + bytarr.length, String.valueOf(map.get("size")));
compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2);
compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr);
checkBlobPost(baseUrl, cloudClient);
}
}
static void checkBlobPost(String baseUrl, CloudSolrClient cloudClient) throws Exception {
String url;
Map map;
byte[] bytarr = new byte[1024];
for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127);
byte[] bytarr2 = new byte[2048];
for (int i = 0; i < bytarr2.length; i++) bytarr2[i] = (byte) (i % 127);
String blobName = "test";
postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr), 1);
postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr2), 2);
url = baseUrl + "/.system/blob/test/1";
map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs"));
assertNotNull("" + map, l);
assertTrue("" + map, l.size() > 0);
map = (Map) l.get(0);
assertEquals("" + bytarr.length, String.valueOf(map.get("size")));
compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2, cloudClient);
compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr, cloudClient);
}
public static void createSystemCollection(SolrClient client) throws SolrServerException, IOException {
CollectionAdminResponse response1;
CollectionAdminRequest.Create createCollectionRequest = new CollectionAdminRequest.Create()
@ -152,7 +158,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
return new String(Utils.toJSON(map), StandardCharsets.UTF_8);
}
private void compareInputAndOutput(String url, byte[] bytarr) throws IOException {
static void compareInputAndOutput(String url, byte[] bytarr, CloudSolrClient cloudClient) throws IOException {
HttpClient httpClient = cloudClient.getLbClient().getHttpClient();

29
solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.DocCollection;
public class TestSystemCollAutoCreate extends AbstractFullDistribZkTestBase {
public void testAutoCreate() throws Exception {
TestBlobHandler.checkBlobPost(cloudJettys.get(0).jetty.getBaseUrl().toExternalForm(), cloudClient);
DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
}
}