mirror of https://github.com/apache/lucene.git
SOLR-6223: SearchComponents may throw NPE when using shards.tolerant and there is a failure in the 'GET_FIELDS/GET_HIGHLIGHTS/GET_DEBUG' phase
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1607897 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar
parent
2cc8facde2
commit
271576ed0f
|
|
@ -155,6 +155,9 @@ Bug Fixes
|
|||
* SOLR-6159: A ZooKeeper session expiry during setup can keep LeaderElector from joining elections.
|
||||
(Steven Bower, shalin)
|
||||
|
||||
* SOLR-6223: SearchComponents may throw NPE when using shards.tolerant and there is a failure
|
||||
in the 'GET_FIELDS/GET_HIGHLIGHTS/GET_DEBUG' phase. (Tomás Fernández Löbbe via shalin)
|
||||
|
||||
Other Changes
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -230,7 +230,7 @@ public class DebugComponent extends SearchComponent
|
|||
}
|
||||
|
||||
if (rb.isDebugResults()) {
|
||||
explain = SolrPluginUtils.removeNulls(new SimpleOrderedMap<>(arr));
|
||||
explain = SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>());
|
||||
}
|
||||
|
||||
if (!hasGetDebugResponses) {
|
||||
|
|
|
|||
|
|
@ -182,6 +182,11 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
|
|||
for (ShardRequest sreq : rb.finished) {
|
||||
if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue;
|
||||
for (ShardResponse srsp : sreq.responses) {
|
||||
if (srsp.getException() != null) {
|
||||
// can't expect the highlight content if there was an exception for this request
|
||||
// this should only happen when using shards.tolerant=true
|
||||
continue;
|
||||
}
|
||||
NamedList hl = (NamedList)srsp.getSolrResponse().getResponse().get("highlighting");
|
||||
for (int i=0; i<hl.size(); i++) {
|
||||
String id = hl.getName(i);
|
||||
|
|
@ -193,7 +198,7 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni
|
|||
}
|
||||
|
||||
// remove nulls in case not all docs were able to be retrieved
|
||||
rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(new SimpleOrderedMap(arr)));
|
||||
rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<Object>()));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -133,6 +133,10 @@ public class MoreLikeThisComponent extends SearchComponent {
|
|||
&& rb.req.getParams().getBool(COMPONENT_NAME, false)) {
|
||||
log.debug("ShardRequest.response.size: " + sreq.responses.size());
|
||||
for (ShardResponse r : sreq.responses) {
|
||||
if (r.getException() != null) {
|
||||
// This should only happen in case of using shards.tolerant=true. Omit this ShardResponse
|
||||
continue;
|
||||
}
|
||||
NamedList<?> moreLikeThisReponse = (NamedList<?>) r.getSolrResponse()
|
||||
.getResponse().get("moreLikeThis");
|
||||
log.debug("ShardRequest.response.shard: " + r.getShard());
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ import org.apache.solr.common.SolrDocument;
|
|||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.*;
|
||||
import org.apache.solr.common.params.CursorMarkParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
|
|
@ -1064,7 +1063,9 @@ public class QueryComponent extends SearchComponent
|
|||
populateNextCursorMarkFromMergedShards(rb);
|
||||
|
||||
if (partialResults) {
|
||||
rb.rsp.getResponseHeader().add( "partialResults", Boolean.TRUE );
|
||||
if(rb.rsp.getResponseHeader().get("partialResults") == null) {
|
||||
rb.rsp.getResponseHeader().add("partialResults", Boolean.TRUE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1227,6 +1228,28 @@ public class QueryComponent extends SearchComponent
|
|||
boolean removeKeyField = !rb.rsp.getReturnFields().wantsField(keyFieldName);
|
||||
|
||||
for (ShardResponse srsp : sreq.responses) {
|
||||
if (srsp.getException() != null) {
|
||||
// Don't try to get the documents if there was an exception in the shard
|
||||
if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
|
||||
@SuppressWarnings("unchecked")
|
||||
NamedList<Object> shardInfo = (NamedList<Object>) rb.rsp.getValues().get(ShardParams.SHARDS_INFO);
|
||||
@SuppressWarnings("unchecked")
|
||||
SimpleOrderedMap<Object> nl = (SimpleOrderedMap<Object>) shardInfo.get(srsp.getShard());
|
||||
if (nl.get("error") == null) {
|
||||
// Add the error to the shards info section if it wasn't added before
|
||||
Throwable t = srsp.getException();
|
||||
if(t instanceof SolrServerException) {
|
||||
t = ((SolrServerException)t).getCause();
|
||||
}
|
||||
nl.add("error", t.toString() );
|
||||
StringWriter trace = new StringWriter();
|
||||
t.printStackTrace(new PrintWriter(trace));
|
||||
nl.add("trace", trace.toString() );
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
SolrDocumentList docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
|
||||
|
||||
for (SolrDocument doc : docs) {
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar
|
|||
public void finishStage(ResponseBuilder rb) {
|
||||
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
|
||||
|
||||
NamedList termVectors = new NamedList<>();
|
||||
NamedList<Object> termVectors = new NamedList<>();
|
||||
Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
|
||||
|
||||
for (ShardRequest sreq : rb.finished) {
|
||||
|
|
@ -450,7 +450,7 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar
|
|||
}
|
||||
}
|
||||
// remove nulls in case not all docs were able to be retrieved
|
||||
termVectors.addAll(SolrPluginUtils.removeNulls(new NamedList<>(arr)));
|
||||
termVectors.addAll(SolrPluginUtils.removeNulls(arr, new NamedList<Object>()));
|
||||
rb.rsp.add(TERM_VECTORS, termVectors);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -702,21 +702,25 @@ public class SolrPluginUtils {
|
|||
}
|
||||
return s.toString().replace("\"","");
|
||||
}
|
||||
|
||||
public static NamedList removeNulls(NamedList nl) {
|
||||
for (int i=0; i<nl.size(); i++) {
|
||||
if (nl.getName(i)==null) {
|
||||
NamedList newList = nl instanceof SimpleOrderedMap ? new SimpleOrderedMap() : new NamedList();
|
||||
for (int j=0; j<nl.size(); j++) {
|
||||
String n = nl.getName(j);
|
||||
if (n != null) {
|
||||
newList.add(n, nl.getVal(j));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds to {@code dest} all the not-null elements of {@code entries} that have non-null names
|
||||
*
|
||||
* @param entries The array of entries to be added to the {@link NamedList} {@code dest}
|
||||
* @param dest The {@link NamedList} instance where the not-null elements of entries are added
|
||||
* @return Returns The {@code dest} input object
|
||||
*/
|
||||
public static <T> NamedList<T> removeNulls(Map.Entry<String, T>[] entries, NamedList<T> dest) {
|
||||
for (int i=0; i<entries.length; i++) {
|
||||
Map.Entry<String, T> entry = entries[i];
|
||||
if (entry != null) {
|
||||
String key = entry.getKey();
|
||||
if (key != null) {
|
||||
dest.add(key, entry.getValue());
|
||||
}
|
||||
return newList;
|
||||
}
|
||||
}
|
||||
return nl;
|
||||
return dest;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
<?xml version="1.0" ?>
|
||||
|
||||
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
|
||||
license agreements. See the NOTICE file distributed with this work for additional
|
||||
information regarding copyright ownership. The ASF licenses this file to
|
||||
You under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy of
|
||||
the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
|
||||
by applicable law or agreed to in writing, software distributed under the
|
||||
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
|
||||
OF ANY KIND, either express or implied. See the License for the specific
|
||||
language governing permissions and limitations under the License. -->
|
||||
|
||||
<!-- This is a "kitchen sink" config file that tests can use. When writting
|
||||
a new test, feel free to add *new* items (plugins, config options, etc...)
|
||||
as long as they don't break any existing tests. if you need to test something
|
||||
esoteric please add a new "solrconfig-your-esoteric-purpose.xml" config file.
|
||||
Note in particular that this test is used by MinimalSchemaTest so Anything
|
||||
added to this file needs to work correctly even if there is now uniqueKey
|
||||
or defaultSearch Field. -->
|
||||
|
||||
<config>
|
||||
|
||||
<dataDir>${solr.data.dir:}</dataDir>
|
||||
|
||||
<directoryFactory name="DirectoryFactory"
|
||||
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
|
||||
|
||||
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
|
||||
|
||||
<xi:include href="solrconfig.snippet.randomindexconfig.xml"
|
||||
xmlns:xi="http://www.w3.org/2001/XInclude" />
|
||||
|
||||
<updateHandler class="solr.DirectUpdateHandler2">
|
||||
<commitWithin>
|
||||
<softCommit>${solr.commitwithin.softcommit:true}</softCommit>
|
||||
</commitWithin>
|
||||
|
||||
</updateHandler>
|
||||
<requestHandler name="/select" class="solr.SearchHandler">
|
||||
<lst name="defaults">
|
||||
<str name="echoParams">explicit</str>
|
||||
<str name="indent">true</str>
|
||||
<str name="df">text</str>
|
||||
</lst>
|
||||
|
||||
</requestHandler>
|
||||
|
||||
<queryResponseWriter name="javabin"
|
||||
class="solr.TestTolerantSearch$BadResponseWriter" />
|
||||
|
||||
<requestHandler name="/admin/"
|
||||
class="org.apache.solr.handler.admin.AdminHandlers" />
|
||||
|
||||
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
|
||||
</config>
|
||||
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
package org.apache.solr;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.client.solrj.request.CoreAdminRequest;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.BinaryResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
public class TestTolerantSearch extends SolrJettyTestBase {
|
||||
|
||||
private static SolrServer collection1;
|
||||
private static SolrServer collection2;
|
||||
private static String shard1;
|
||||
private static String shard2;
|
||||
private static File solrHome;
|
||||
|
||||
private static File createSolrHome() throws Exception {
|
||||
File workDir = createTempDir();
|
||||
setupJettyTestHome(workDir, "collection1");
|
||||
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME() + "/collection1/conf/solrconfig-tolerant-search.xml"), new File(workDir, "/collection1/conf/solrconfig.xml"));
|
||||
FileUtils.copyDirectory(new File(workDir, "collection1"), new File(workDir, "collection2"));
|
||||
return workDir;
|
||||
}
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void createThings() throws Exception {
|
||||
solrHome = createSolrHome();
|
||||
createJetty(solrHome.getAbsolutePath(), null, null);
|
||||
String url = jetty.getBaseUrl().toString();
|
||||
collection1 = new HttpSolrServer(url);
|
||||
collection2 = new HttpSolrServer(url + "/collection2");
|
||||
|
||||
String urlCollection1 = jetty.getBaseUrl().toString() + "/" + "collection1";
|
||||
String urlCollection2 = jetty.getBaseUrl().toString() + "/" + "collection2";
|
||||
shard1 = urlCollection1.replaceAll("https?://", "");
|
||||
shard2 = urlCollection2.replaceAll("https?://", "");
|
||||
|
||||
//create second core
|
||||
CoreAdminRequest.Create req = new CoreAdminRequest.Create();
|
||||
req.setCoreName("collection2");
|
||||
collection1.request(req);
|
||||
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.setField("id", "1");
|
||||
doc.setField("subject", "batman");
|
||||
doc.setField("title", "foo bar");
|
||||
collection1.add(doc);
|
||||
collection1.commit();
|
||||
|
||||
doc.setField("id", "2");
|
||||
doc.setField("subject", "superman");
|
||||
collection2.add(doc);
|
||||
collection2.commit();
|
||||
|
||||
doc = new SolrInputDocument();
|
||||
doc.setField("id", "3");
|
||||
doc.setField("subject", "aquaman");
|
||||
doc.setField("title", "foo bar");
|
||||
collection1.add(doc);
|
||||
collection1.commit();
|
||||
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void destroyThings() throws Exception {
|
||||
collection1.shutdown();
|
||||
collection2.shutdown();
|
||||
collection1 = null;
|
||||
collection2 = null;
|
||||
jetty.stop();
|
||||
jetty=null;
|
||||
resetExceptionIgnores();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testGetFieldsPhaseError() throws SolrServerException {
|
||||
BadResponseWriter.failOnGetFields = true;
|
||||
BadResponseWriter.failOnGetTopIds = false;
|
||||
SolrQuery query = new SolrQuery();
|
||||
query.setQuery("subject:batman OR subject:superman");
|
||||
query.addField("id");
|
||||
query.addField("subject");
|
||||
query.set("distrib", "true");
|
||||
query.set("shards", shard1 + "," + shard2);
|
||||
query.set(ShardParams.SHARDS_INFO, "true");
|
||||
query.set("debug", "true");
|
||||
query.set("stats", "true");
|
||||
query.set("stats.field", "id");
|
||||
query.set("mlt", "true");
|
||||
query.set("mlt.fl", "title");
|
||||
query.set("mlt.count", "1");
|
||||
query.set("mlt.mintf", "0");
|
||||
query.set("mlt.mindf", "0");
|
||||
query.setHighlight(true);
|
||||
query.addFacetField("id");
|
||||
query.setFacet(true);
|
||||
|
||||
ignoreException("Dummy exception in BadResponseWriter");
|
||||
try {
|
||||
collection1.query(query);
|
||||
fail("Should get an exception");
|
||||
} catch (Exception e) {
|
||||
//expected
|
||||
}
|
||||
query.set(ShardParams.SHARDS_TOLERANT, "true");
|
||||
QueryResponse response = collection1.query(query);
|
||||
assertTrue(response.getResponseHeader().getBooleanArg("partialResults"));
|
||||
NamedList<Object> shardsInfo = ((NamedList<Object>)response.getResponse().get("shards.info"));
|
||||
boolean foundError = false;
|
||||
for (int i = 0; i < shardsInfo.size(); i++) {
|
||||
if (shardsInfo.getName(i).contains("collection2")) {
|
||||
assertNotNull(((NamedList<Object>)shardsInfo.getVal(i)).get("error"));
|
||||
foundError = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assertTrue(foundError);
|
||||
assertEquals(1, response.getResults().get(0).getFieldValue("id"));
|
||||
assertEquals("batman", response.getResults().get(0).getFirstValue("subject"));
|
||||
unIgnoreException("Dummy exception in BadResponseWriter");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testGetTopIdsPhaseError() throws SolrServerException {
|
||||
BadResponseWriter.failOnGetTopIds = true;
|
||||
BadResponseWriter.failOnGetFields = false;
|
||||
SolrQuery query = new SolrQuery();
|
||||
query.setQuery("subject:batman OR subject:superman");
|
||||
query.addField("id");
|
||||
query.addField("subject");
|
||||
query.set("distrib", "true");
|
||||
query.set("shards", shard1 + "," + shard2);
|
||||
query.set(ShardParams.SHARDS_INFO, "true");
|
||||
query.set("debug", "true");
|
||||
query.set("stats", "true");
|
||||
query.set("stats.field", "id");
|
||||
query.set("mlt", "true");
|
||||
query.set("mlt.fl", "title");
|
||||
query.set("mlt.count", "1");
|
||||
query.set("mlt.mintf", "0");
|
||||
query.set("mlt.mindf", "0");
|
||||
query.setHighlight(true);
|
||||
query.addFacetField("id");
|
||||
query.setFacet(true);
|
||||
|
||||
ignoreException("Dummy exception in BadResponseWriter");
|
||||
try {
|
||||
collection1.query(query);
|
||||
fail("Should get an exception");
|
||||
} catch (Exception e) {
|
||||
//expected
|
||||
}
|
||||
query.set(ShardParams.SHARDS_TOLERANT, "true");
|
||||
QueryResponse response = collection1.query(query);
|
||||
assertTrue(response.getResponseHeader().getBooleanArg("partialResults"));
|
||||
NamedList<Object> shardsInfo = ((NamedList<Object>)response.getResponse().get("shards.info"));
|
||||
boolean foundError = false;
|
||||
for (int i = 0; i < shardsInfo.size(); i++) {
|
||||
if (shardsInfo.getName(i).contains("collection2")) {
|
||||
assertNotNull(((NamedList<Object>)shardsInfo.getVal(i)).get("error"));
|
||||
foundError = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assertTrue(foundError);
|
||||
|
||||
assertEquals(1, response.getResults().get(0).getFieldValue("id"));
|
||||
assertEquals("batman", response.getResults().get(0).getFirstValue("subject"));
|
||||
unIgnoreException("Dummy exception in BadResponseWriter");
|
||||
}
|
||||
|
||||
public static class BadResponseWriter extends BinaryResponseWriter {
|
||||
|
||||
private static boolean failOnGetFields = false;
|
||||
private static boolean failOnGetTopIds = false;
|
||||
|
||||
public BadResponseWriter() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(OutputStream out, SolrQueryRequest req,
|
||||
SolrQueryResponse response) throws IOException {
|
||||
|
||||
// I want to fail on the shard request, not the original user request, and only on the
|
||||
// GET_FIELDS phase
|
||||
if (failOnGetFields &&
|
||||
"collection2".equals(req.getCore().getName())
|
||||
&& "subject:batman OR subject:superman".equals(req.getParams().get("q", ""))
|
||||
&& req.getParams().get("ids") != null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Dummy exception in BadResponseWriter");
|
||||
} else if (failOnGetTopIds
|
||||
&& "collection2".equals(req.getCore().getName())
|
||||
&& "subject:batman OR subject:superman".equals(req.getParams().get("q", ""))
|
||||
&& req.getParams().get("ids") == null
|
||||
&& req.getParams().getBool("isShard", false) == true) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Dummy exception in BadResponseWriter");
|
||||
}
|
||||
super.write(out, req, response);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue