Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5465: SolrCmdDistributor retry logic has a concurrency race bug. 

SOLR-5464: ConcurrentSolrServer does not stream pure delete by id requests.
SOLR-5452: Do not attempt to proxy internal update requests.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1543299 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-11-19 03:13:33 +00:00
parent d6e1114002
commit 8822632048
7 changed files with 120 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
solr/CHANGES.txt
View File

@ -108,14 +108,20 @@ Bug Fixes
* SOLR-5451: SyncStrategy closes it's http connection manager before the
executor that uses it in it's close method. (Mark Miller)
* SOLR-5452: Do not attempt to proxy internal update requests. (Mark Miller)
* SOLR-5460: SolrDispatchFilter#sendError can get a SolrCore that it does not
close. (Mark Miller)
* SOLR-5461: Request proxying should only set con.setDoOutput(true) if the
request is a post. (Mark Miller)
* SOLR-5465: SolrCmdDistributor retry logic has a concurrency race bug.
(Mark Miller)
* SOLR-5464: ConcurrentSolrServer does not stream pure delete by id requests.
(Mark Miller)
* SOLR-5452: Do not attempt to proxy internal update requests. (Mark Miller)
Other Changes
---------------------

9
solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -52,6 +52,7 @@ import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.FastWriter;
import org.slf4j.Logger;
@ -315,9 +316,11 @@ public class SolrDispatchFilter implements Filter
if (core == null && idx > 0) {
String coreUrl = getRemotCoreUrl(cores, corename, origCorename);
// don't proxy for internal update requests
//solrReq = SolrRequestParsers.DEFAULT.parse(null,path, req);
//if (coreUrl != null && solrReq.getParams().get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null) {
if (coreUrl != null) {
SolrParams queryParams = SolrRequestParsers.parseQueryString(req.getQueryString());
if (coreUrl != null
&& queryParams
.get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null
&& queryParams.get(DistributedUpdateProcessor.DISTRIB_FROM) == null) {
path = path.substring(idx);
remoteQuery(coreUrl + path, req, solrReq, resp);
return;

10
solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
View File

@ -69,8 +69,8 @@ public class SolrCmdDistributor {
List<Error> errors = new ArrayList<Error>(this.errors);
errors.addAll(servers.getErrors());
List<Error> resubmitList = new ArrayList<Error>();
boolean blockUntilFinishedAgain = false;
for (Error err : errors) {
String oldNodeUrl = err.req.node.getUrl();
@ -108,8 +108,7 @@ public class SolrCmdDistributor {
log.warn(null, e);
}
submit(err.req);
blockUntilFinishedAgain = true;
resubmitList.add(err);
} else {
allErrors.add(err);
}
@ -120,8 +119,11 @@ public class SolrCmdDistributor {
servers.clearErrors();
this.errors.clear();
for (Error err : resubmitList) {
submit(err.req);
}
if (blockUntilFinishedAgain) {
if (resubmitList.size() > 0) {
servers.blockUntilFinished();
doRetriesIfNeeded();
}

10
solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java
View File

@ -20,8 +20,10 @@ package org.apache.solr.update;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.apache.http.client.HttpClient;
@ -33,6 +35,8 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.SolrCmdDistributor.Error;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -70,7 +74,7 @@ public class StreamingSolrServers {
String url = getFullUrl(req.node.getUrl());
ConcurrentUpdateSolrServer server = solrServers.get(url);
if (server == null) {
server = new ConcurrentUpdateSolrServer(url, httpClient, 100, 1, updateExecutor) {
server = new ConcurrentUpdateSolrServer(url, httpClient, 100, 1, updateExecutor, true) {
@Override
public void handleError(Throwable ex) {
log.error("error", ex);
@ -86,6 +90,10 @@ public class StreamingSolrServers {
server.setParser(new BinaryResponseParser());
server.setRequestWriter(new BinaryRequestWriter());
server.setPollQueueTime(0);
Set<String> queryParams = new HashSet<String>(2);
queryParams.add(DistributedUpdateProcessor.DISTRIB_FROM);
queryParams.add(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM);
server.setQueryParams(queryParams);
solrServers.put(url, server);
}

37
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
View File

@ -22,6 +22,7 @@ import java.io.OutputStream;
import java.util.LinkedList;
import java.util.Locale;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@ -80,6 +81,7 @@ public class ConcurrentUpdateSolrServer extends SolrServer {
final int threadCount;
boolean shutdownExecutor = false;
int pollQueueTime = 250;
private final boolean streamDeletes;
/**
* Uses an internally managed HttpClient instance.
@ -109,12 +111,33 @@ public class ConcurrentUpdateSolrServer extends SolrServer {
*/
public ConcurrentUpdateSolrServer(String solrServerUrl,
HttpClient client, int queueSize, int threadCount, ExecutorService es) {
this(solrServerUrl, client, queueSize, threadCount, es, false);
}
/**
* Uses the supplied HttpClient to send documents to the Solr server.
*/
public ConcurrentUpdateSolrServer(String solrServerUrl,
HttpClient client, int queueSize, int threadCount, ExecutorService es, boolean streamDeletes) {
this.server = new HttpSolrServer(solrServerUrl, client);
this.server.setFollowRedirects(false);
queue = new LinkedBlockingQueue<UpdateRequest>(queueSize);
this.threadCount = threadCount;
runners = new LinkedList<Runner>();
scheduler = es;
this.streamDeletes = streamDeletes;
}
public Set<String> getQueryParams() {
return this.server.getQueryParams();
}
/**
* Expert Method.
* @param queryParams set of param keys to only send via the query string
*/
public void setQueryParams(Set<String> queryParams) {
this.server.setQueryParams(queryParams);
}
/**
@ -261,10 +284,22 @@ public class ConcurrentUpdateSolrServer extends SolrServer {
UpdateRequest req = (UpdateRequest) request;
// this happens for commit...
if (req.getDocuments() == null || req.getDocuments().isEmpty()) {
if (streamDeletes) {
if ((req.getDocuments() == null || req.getDocuments().isEmpty())
&& (req.getDeleteById() == null || req.getDeleteById().isEmpty())
&& (req.getDeleteByIdMap() == null || req.getDeleteByIdMap().isEmpty())) {
blockUntilFinished();
if (req.getDeleteQuery() == null) {
return server.request(request);
}
}
} else {
if ((req.getDocuments() == null || req.getDocuments().isEmpty())) {
blockUntilFinished();
return server.request(request);
}
}
SolrParams params = req.getParams();
if (params != null) {

39
solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
View File

@ -22,9 +22,11 @@ import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
@ -71,6 +73,7 @@ public class HttpSolrServer extends SolrServer {
private static final String UTF_8 = "UTF-8";
private static final String DEFAULT_PATH = "/select";
private static final long serialVersionUID = -946812319974801896L;
/**
* User-Agent String.
*/
@ -117,6 +120,7 @@ public class HttpSolrServer extends SolrServer {
private boolean useMultiPartPost;
private final boolean internalClient;
private Set<String> queryParams = Collections.emptySet();
/**
* @param baseURL
@ -158,6 +162,18 @@ public class HttpSolrServer extends SolrServer {
this.parser = parser;
}
public Set<String> getQueryParams() {
return queryParams;
}
/**
* Expert Method.
* @param queryParams set of param keys to only send via the query string
*/
public void setQueryParams(Set<String> queryParams) {
this.queryParams = queryParams;
}
/**
* Process the request. If
* {@link org.apache.solr.client.solrj.SolrRequest#getResponseParser()} is
@ -207,7 +223,6 @@ public class HttpSolrServer extends SolrServer {
if (invariantParams != null) {
wparams.add(invariantParams);
}
params = wparams;
int tries = maxRetries + 1;
try {
@ -221,7 +236,7 @@ public class HttpSolrServer extends SolrServer {
if( streams != null ) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!" );
}
method = new HttpGet( baseUrl + path + ClientUtils.toQueryString( params, false ) );
method = new HttpGet( baseUrl + path + ClientUtils.toQueryString( wparams, false ) );
}
else if( SolrRequest.METHOD.POST == request.getMethod() ) {
@ -237,9 +252,21 @@ public class HttpSolrServer extends SolrServer {
}
boolean isMultipart = (this.useMultiPartPost || ( streams != null && streams.size() > 1 )) && !hasNullStreamName;
// only send this list of params as query string params
ModifiableSolrParams queryParams = new ModifiableSolrParams();
for (String param : this.queryParams) {
String[] value = wparams.getParams(param) ;
if (value != null) {
for (String v : value) {
queryParams.add(param, v);
}
wparams.remove(param);
}
}
LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>();
if (streams == null || isMultipart) {
HttpPost post = new HttpPost(url);
HttpPost post = new HttpPost(url + ClientUtils.toQueryString( queryParams, false ));
post.setHeader("Content-Charset", "UTF-8");
if (!isMultipart) {
post.addHeader("Content-Type",
@ -247,10 +274,10 @@ public class HttpSolrServer extends SolrServer {
}
List<FormBodyPart> parts = new LinkedList<FormBodyPart>();
Iterator<String> iter = params.getParameterNamesIterator();
Iterator<String> iter = wparams.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = params.getParams(p);
String[] vals = wparams.getParams(p);
if (vals != null) {
for (String v : vals) {
if (isMultipart) {
@ -295,7 +322,7 @@ public class HttpSolrServer extends SolrServer {
}
// It is has one stream, it is the post body, put the params in the URL
else {
String pstr = ClientUtils.toQueryString(params, false);
String pstr = ClientUtils.toQueryString(wparams, false);
HttpPost post = new HttpPost(url + pstr);
// Single stream as body

17
solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
View File

@ -97,6 +97,8 @@ public class LBHttpSolrServer extends SolrServer {
private volatile ResponseParser parser;
private volatile RequestWriter requestWriter;
private Set<String> queryParams;
static {
solrQuery.setRows(0);
}
@ -214,6 +216,18 @@ public class LBHttpSolrServer extends SolrServer {
updateAliveList();
}
public Set<String> getQueryParams() {
return queryParams;
}
/**
* Expert Method.
* @param queryParams set of param keys to only send via the query string
*/
public void setQueryParams(Set<String> queryParams) {
this.queryParams = queryParams;
}
public static String normalize(String server) {
if (server.endsWith("/"))
server = server.substring(0, server.length() - 1);
@ -225,6 +239,9 @@ public class LBHttpSolrServer extends SolrServer {
if (requestWriter != null) {
s.setRequestWriter(requestWriter);
}
if (queryParams != null) {
s.setQueryParams(queryParams);
}
return s;
}