diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 26e6399fa71..bdfaee454c4 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -18,11 +18,10 @@ package org.apache.solr.handler.dataimport; */ import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; @@ -83,7 +82,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { * @return a {@link HttpClient} instance used for interfacing with a source Solr service */ protected HttpClient getHttpClient() { - return new DefaultHttpClient(new ThreadSafeClientConnManager()); + return HttpClientUtil.createClient(null); } @Override diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java index ed515e73280..3867749da2e 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java @@ -27,10 +27,10 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.commons.io.FileUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.common.SolrInputDocument; import org.junit.After; @@ -274,7 +274,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe sidl.add(sd); } - DefaultHttpClient client = new DefaultHttpClient(new ThreadSafeClientConnManager()); + HttpClient client = HttpClientUtil.createClient(null); URL url = new URL(getSourceUrl(jetty.getLocalPort())); HttpSolrServer solrServer = new HttpSolrServer(url.toExternalForm(), client); solrServer.add(sidl); diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index 9ca9eecfeac..5232ca15725 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -23,11 +23,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery; import org.apache.solr.common.SolrException; @@ -51,25 +49,21 @@ import org.slf4j.LoggerFactory; public class SyncStrategy { protected final Logger log = LoggerFactory.getLogger(getClass()); - private HttpShardHandlerFactory shardHandlerFactory; - - private ShardHandler shardHandler; + private final ShardHandler shardHandler; - private static ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); - private static DefaultHttpClient client = new DefaultHttpClient(mgr); + private final static HttpClient client; static { - mgr.setDefaultMaxPerRoute(20); - mgr.setMaxTotal(10000); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 30000); - client.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000); - // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20); + params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_SO_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_USE_RETRY, false); + client = HttpClientUtil.createClient(params); } public SyncStrategy() { - shardHandlerFactory = new HttpShardHandlerFactory(); - shardHandler = shardHandlerFactory.getShardHandler(client); + shardHandler = new HttpShardHandlerFactory().getShardHandler(client); } private static class SyncShardRequest extends ShardRequest { diff --git a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java index 9179fffb657..eb4a5f3039c 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java @@ -20,20 +20,16 @@ import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.message.AbstractHttpMessage; import org.apache.http.message.BasicNameValuePair; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.util.EntityUtils; import org.apache.lucene.index.IndexCommit; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.FastInputStream; import org.apache.solr.util.FileUtils; import org.apache.solr.common.util.JavaBinCodec; @@ -122,26 +118,23 @@ public class SnapPuller { private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword) { if (connTimeout == null && readTimeout == null && client != null) return client; - ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); + final ModifiableSolrParams httpClientParams = new ModifiableSolrParams(); + httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000"); + httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000"); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword); // Keeping a very high number so that if you have a large number of cores // no requests are kept waiting for an idle connection. - mgr.setDefaultMaxPerRoute(10000); - mgr.setMaxTotal(10000); - DefaultHttpClient httpClient = new DefaultHttpClient(mgr); - httpClient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, readTimeout == null ? 20000 : Integer.parseInt(readTimeout)); //20 secs - httpClient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connTimeout == null ? 5000 : Integer.parseInt(connTimeout)); //5 secs + httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 10000); + HttpClient httpClient = HttpClientUtil.createClient(httpClientParams); if (client == null && connTimeout == null && readTimeout == null) client = httpClient; - - if (httpBasicAuthUser != null && httpBasicAuthPassword != null) { - httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, - new UsernamePasswordCredentials(httpBasicAuthUser, httpBasicAuthPassword)); - } - return httpClient; } public SnapPuller(NamedList initArgs, ReplicationHandler handler, SolrCore sc) { solrCore = sc; + SolrParams params = SolrParams.toSolrParams(initArgs); masterUrl = (String) initArgs.get(MASTER_URL); if (masterUrl == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, @@ -152,10 +145,10 @@ public class SnapPuller { String compress = (String) initArgs.get(COMPRESSION); useInternal = INTERNAL.equals(compress); useExternal = EXTERNAL.equals(compress); - String connTimeout = (String) initArgs.get(HTTP_CONN_TIMEOUT); - String readTimeout = (String) initArgs.get(HTTP_READ_TIMEOUT); - String httpBasicAuthUser = (String) initArgs.get(HTTP_BASIC_AUTH_USER); - String httpBasicAuthPassword = (String) initArgs.get(HTTP_BASIC_AUTH_PASSWORD); + String connTimeout = (String) initArgs.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT); + String readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT); + String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); + String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); myHttpClient = createHttpClient(connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword); if (pollInterval != null && pollInterval > 0) { startExecutorService(); @@ -1251,14 +1244,6 @@ public class SnapPuller { private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); - private static final String HTTP_CONN_TIMEOUT = "httpConnTimeout"; - - private static final String HTTP_READ_TIMEOUT = "httpReadTimeout"; - - private static final String HTTP_BASIC_AUTH_USER = "httpBasicAuthUser"; - - private static final String HTTP_BASIC_AUTH_PASSWORD = "httpBasicAuthPassword"; - static final String INDEX_REPLICATED_AT = "indexReplicatedAt"; static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated"; diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 9daebb43a88..3a4bbe7fae8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -161,7 +161,7 @@ public class HttpShardHandler extends ShardHandler { if (urls.size() <= 1) { String url = urls.get(0); srsp.setShardAddress(url); - SolrServer server = new HttpSolrServer(url, httpClient == null ? httpShardHandlerFactory.client : httpClient); + SolrServer server = new HttpSolrServer(url, httpClient); ssr.nl = server.request(req); } else { LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.loadbalancer.request(new LBHttpSolrServer.Req(req, urls)); diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java index d77bf8085b6..f8e33dcffb6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java @@ -21,12 +21,10 @@ import java.util.Random; import java.util.concurrent.*; import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.LBHttpSolrServer; import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.PluginInfo; import org.apache.solr.util.DefaultSolrThreadFactory; @@ -51,37 +49,25 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug new DefaultSolrThreadFactory("httpShardExecutor") ); - HttpClient client; - Random r = new Random(); + private HttpClient defaultClient; LBHttpSolrServer loadbalancer; - int soTimeout = 0; //current default values - int connectionTimeout = 0; //current default values + //default values: + int soTimeout = 0; + int connectionTimeout = 0; int maxConnectionsPerHost = 20; int corePoolSize = 0; - int maximumPoolSize = 10; + int maximumPoolSize = Integer.MAX_VALUE; int keepAliveTime = 5; - int queueSize = 1; - boolean accessPolicy = true; + int queueSize = -1; + boolean accessPolicy = false; public String scheme = "http://"; //current default values - private ThreadSafeClientConnManager mgr; - // socket timeout measured in ms, closes a socket if read - // takes longer than x ms to complete. throws - // java.net.SocketTimeoutException: Read timed out exception - static final String INIT_SO_TIMEOUT = "socketTimeout"; - - // connection timeout measures in ms, closes a socket if connection - // cannot be established within x ms. with a - // java.net.SocketTimeoutException: Connection timed out - static final String INIT_CONNECTION_TIMEOUT = "connTimeout"; + final Random r = new Random(); // URL scheme to be used in distributed search. static final String INIT_URL_SCHEME = "urlScheme"; - // Maximum connections allowed per host - static final String INIT_MAX_CONNECTION_PER_HOST = "maxConnectionsPerHost"; - // The core size of the threadpool servicing requests static final String INIT_CORE_POOL_SIZE = "corePoolSize"; @@ -97,27 +83,32 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug // Configure if the threadpool favours fairness over throughput static final String INIT_FAIRNESS_POLICY = "fairnessPolicy"; + /** + * Get {@link ShardHandler} that uses the default http client. + */ public ShardHandler getShardHandler() { - return getShardHandler(null); + return getShardHandler(defaultClient); } - public ShardHandler getShardHandler(DefaultHttpClient httpClient){ + /** + * Get {@link ShardHandler} that uses custom http client. + */ + public ShardHandler getShardHandler(final HttpClient httpClient){ return new HttpShardHandler(this, httpClient); } public void init(PluginInfo info) { NamedList args = info.initArgs; - this.soTimeout = getParameter(args, INIT_SO_TIMEOUT, 0); - + this.soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT, soTimeout); this.scheme = getParameter(args, INIT_URL_SCHEME, "http://"); this.scheme = (this.scheme.endsWith("://")) ? this.scheme : this.scheme + "://"; - this.connectionTimeout = getParameter(args, INIT_CONNECTION_TIMEOUT, 0); - this.maxConnectionsPerHost = getParameter(args, INIT_MAX_CONNECTION_PER_HOST, 20); - this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, 0); - this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, Integer.MAX_VALUE); - this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, 5); - this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, -1); - this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, false); + this.connectionTimeout = getParameter(args, HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout); + this.maxConnectionsPerHost = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost); + this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, corePoolSize); + this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, maximumPoolSize); + this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime); + this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize); + this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy); BlockingQueue blockingQueue = (this.queueSize == -1) ? new SynchronousQueue(this.accessPolicy) : @@ -131,23 +122,16 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug new DefaultSolrThreadFactory("httpShardExecutor") ); - mgr = new ThreadSafeClientConnManager(); - mgr.setDefaultMaxPerRoute(256); - mgr.setMaxTotal(10000); - DefaultHttpClient client = new DefaultHttpClient(mgr); - - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); - client.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, soTimeout); - // mgr.getParams().setStaleCheckingEnabled(false); - - - // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); - this.client = client; + ModifiableSolrParams clientParams = new ModifiableSolrParams(); + clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost); + clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, soTimeout); + clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout); + clientParams.set(HttpClientUtil.PROP_USE_RETRY, false); + this.defaultClient = HttpClientUtil.createClient(clientParams); try { - loadbalancer = new LBHttpSolrServer(client); + loadbalancer = new LBHttpSolrServer(defaultClient); } catch (MalformedURLException e) { // should be impossible since we're not passing any URLs here throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); @@ -169,7 +153,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug @Override public void close() { try { - mgr.shutdown(); + defaultClient.getConnectionManager().shutdown(); } catch (Throwable e) { SolrException.log(log, e); } diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index abff7ba9344..58052b74192 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -27,12 +27,10 @@ import java.util.List; import java.util.Set; import org.apache.http.NoHttpResponseException; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.client.HttpClient; import org.apache.lucene.util.BytesRef; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -80,17 +78,15 @@ public class PeerSync { private Set requestedUpdateSet; private long ourLowThreshold; // 20th percentile private long ourHighThreshold; // 80th percentile - private static ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); - private static DefaultHttpClient client = new DefaultHttpClient(mgr); + private static final HttpClient client; static { - mgr.setDefaultMaxPerRoute(20); - mgr.setMaxTotal(10000); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 30000); - client.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000); - - // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_SO_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_USE_RETRY, false); + client = HttpClientUtil.createClient(params); } // comparator that sorts by absolute value, putting highest first diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index b0a82bd26a2..acd2229a1d7 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -34,8 +34,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.UpdateRequestExt; @@ -56,13 +55,13 @@ public class SolrCmdDistributor { Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue(), new DefaultSolrThreadFactory("cmdDistribExecutor")); - static HttpClient client; + static final HttpClient client; static { - ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); - mgr.setDefaultMaxPerRoute(8); - mgr.setMaxTotal(200); - client = new DefaultHttpClient(mgr); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 200); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 8); + client = HttpClientUtil.createClient(params); } CompletionService completionService; diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 3a65bed6b74..7a5bdbd306b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -25,11 +25,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.common.SolrInputDocument; import org.apache.zookeeper.KeeperException; @@ -203,8 +203,7 @@ public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest { } class FullThrottleStopableIndexingThread extends StopableIndexingThread { - ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(); - private DefaultHttpClient httpClient = new DefaultHttpClient(cm) ; + private HttpClient httpClient = HttpClientUtil.createClient(null); private volatile boolean stop = false; int clientIndex = 0; private ConcurrentUpdateSolrServer suss; @@ -301,7 +300,7 @@ public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest { public void safeStop() { stop = true; suss.shutdownNow(); - cm.shutdown(); + httpClient.getConnectionManager().shutdown(); } public int getFails() { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java index d10530cea4e..75d95ea7bcb 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java @@ -28,8 +28,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.TimeoutException; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; @@ -60,16 +59,16 @@ public class CloudSolrServer extends SolrServer { private int zkClientTimeout = 10000; private volatile String defaultCollection; private LBHttpSolrServer lbServer; + private HttpClient myClient; Random rand = new Random(); - private ThreadSafeClientConnManager connManager; /** * @param zkHost The client endpoint of the zookeeper quorum containing the cloud state, * in the form HOST:PORT. */ public CloudSolrServer(String zkHost) throws MalformedURLException { - connManager = new ThreadSafeClientConnManager(); this.zkHost = zkHost; - this.lbServer = new LBHttpSolrServer(new DefaultHttpClient(connManager)); + this.myClient = HttpClientUtil.createClient(null); + this.lbServer = new LBHttpSolrServer(myClient); } /** @@ -206,8 +205,8 @@ public class CloudSolrServer extends SolrServer { zkStateReader = null; } } - if (connManager != null) { - connManager.shutdown(); + if (myClient!=null) { + myClient.getConnectionManager().shutdown(); } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java new file mode 100644 index 00000000000..b71ac7c01b4 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java @@ -0,0 +1,311 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; + +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.HttpClient; +import org.apache.http.client.params.ClientParamBean; +import org.apache.http.entity.HttpEntityWrapper; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.params.HttpConnectionParams; +import org.apache.http.protocol.HttpContext; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for creating/configuring httpclient instances. + */ +public class HttpClientUtil { + + // socket timeout measured in ms, closes a socket if read + // takes longer than x ms to complete. throws + // java.net.SocketTimeoutException: Read timed out exception + public static final String PROP_SO_TIMEOUT = "socketTimeout"; + // connection timeout measures in ms, closes a socket if connection + // cannot be established within x ms. with a + // java.net.SocketTimeoutException: Connection timed out + public static final String PROP_CONNECTION_TIMEOUT = "connTimeout"; + // Maximum connections allowed per host + public static final String PROP_MAX_CONNECTIONS_PER_HOST = "maxConnectionsPerHost"; + // Maximum total connections allowed + public static final String PROP_MAX_CONNECTIONS = "maxConnections"; + // Retry http requests on error + public static final String PROP_USE_RETRY = "retry"; + // Allow compression (deflate,gzip) if server supports it + public static final String PROP_ALLOW_COMPRESSION = "allowCompression"; + // Follow redirects + public static final String PROP_FOLLOW_REDIRECTS = "followRedirects"; + // Basic auth username + public static final String PROP_BASIC_AUTH_USER = "httpBasicAuthUser"; + // Basic auth password + public static final String PROP_BASIC_AUTH_PASS = "httpBasicAuthPassword"; + + private static final Logger logger = LoggerFactory + .getLogger(HttpClientUtil.class); + + static final DefaultHttpRequestRetryHandler NO_RETRY = new DefaultHttpRequestRetryHandler( + 0, false); + + private HttpClientUtil(){} + + /** + * Creates new http client by using the provided configuration. + * + * @param params + * http client configuration, if null a client with default + * configuration (no additional configuration) is created that uses + * ThreadSafeClientConnManager. + */ + public static HttpClient createClient(final SolrParams params) { + final ModifiableSolrParams config = new ModifiableSolrParams(params); + logger.info("Creating new http client, config:" + config); + final ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); + final DefaultHttpClient httpClient = new DefaultHttpClient(mgr); + configureClient(httpClient, config); + return httpClient; + } + + /** + * Configures {@link DefaultHttpClient}, only sets parameters if they are + * present in config. + */ + public static void configureClient(final DefaultHttpClient httpClient, + SolrParams config) { + + if (config.get(PROP_MAX_CONNECTIONS) != null) { + setMaxConnections(httpClient, config.getInt(PROP_MAX_CONNECTIONS)); + } + + if (config.get(PROP_MAX_CONNECTIONS_PER_HOST) != null) { + setMaxConnectionsPerHost(httpClient, config.getInt(PROP_MAX_CONNECTIONS_PER_HOST)); + } + + if (config.get(PROP_CONNECTION_TIMEOUT) != null) { + setConnectionTimeout(httpClient, config.getInt(PROP_CONNECTION_TIMEOUT)); + } + + if (config.get(PROP_SO_TIMEOUT) != null) { + setSoTimeout(httpClient, config.getInt(PROP_SO_TIMEOUT)); + } + + if (config.get(PROP_USE_RETRY) != null) { + setUseRetry(httpClient, config.getBool(PROP_USE_RETRY)); + } + + if (config.get(PROP_FOLLOW_REDIRECTS) != null) { + setFollowRedirects(httpClient, config.getBool(PROP_FOLLOW_REDIRECTS)); + } + + final String basicAuthUser = config.get(PROP_BASIC_AUTH_USER); + final String basicAuthPass = config.get(PROP_BASIC_AUTH_PASS); + setBasicAuth(httpClient, basicAuthUser, basicAuthPass); + + if (config.get(PROP_ALLOW_COMPRESSION) != null) { + setAllowCompression(httpClient, config.getBool(PROP_ALLOW_COMPRESSION)); + } + } + + /** + * Control HTTP payload compression. + * + * @param allowCompression + * true will enable compression (needs support from server), false + * will disable compression. + */ + public static void setAllowCompression(DefaultHttpClient httpClient, + boolean allowCompression) { + httpClient + .removeRequestInterceptorByClass(UseCompressionRequestInterceptor.class); + httpClient + .removeResponseInterceptorByClass(UseCompressionResponseInterceptor.class); + if (allowCompression) { + httpClient.addRequestInterceptor(new UseCompressionRequestInterceptor()); + httpClient + .addResponseInterceptor(new UseCompressionResponseInterceptor()); + } + } + + /** + * Set http basic auth information. If basicAuthUser or basicAuthPass is null + * the basic auth configuration is cleared. Currently this is not preemtive + * authentication. So it is not currently possible to do a post request while + * using this setting. + */ + public static void setBasicAuth(DefaultHttpClient httpClient, + String basicAuthUser, String basicAuthPass) { + if (basicAuthUser != null && basicAuthPass != null) { + httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(basicAuthUser, basicAuthPass)); + } else { + httpClient.getCredentialsProvider().clear(); + } + } + + /** + * Set max connections allowed per host. This call will only work when + * {@link ThreadSafeClientConnManager} is used. + */ + public static void setMaxConnectionsPerHost(HttpClient httpClient, + int max) { + if(httpClient.getConnectionManager() instanceof ThreadSafeClientConnManager) { + ThreadSafeClientConnManager mgr = (ThreadSafeClientConnManager)httpClient.getConnectionManager(); + mgr.setDefaultMaxPerRoute(max); + } + } + + /** + * Set max total connections allowed. This call will only work when + * {@link ThreadSafeClientConnManager} is used. + */ + public static void setMaxConnections(final HttpClient httpClient, + int max) { + if(httpClient.getConnectionManager() instanceof ThreadSafeClientConnManager) { + ThreadSafeClientConnManager mgr = (ThreadSafeClientConnManager)httpClient.getConnectionManager(); + mgr.setMaxTotal(max); + } + } + + + /** + * Defines the socket timeout (SO_TIMEOUT) in milliseconds. A timeout value of + * zero is interpreted as an infinite timeout. + * + * @param timeout timeout in milliseconds + */ + public static void setSoTimeout(HttpClient httpClient, int timeout) { + HttpConnectionParams.setSoTimeout(httpClient.getParams(), + timeout); + } + + /** + * Control retry handler + * @param useRetry when false the client will not try to retry failed requests. + */ + public static void setUseRetry(final DefaultHttpClient httpClient, + boolean useRetry) { + if (!useRetry) { + httpClient.setHttpRequestRetryHandler(NO_RETRY); + } else { + httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler()); + } + } + + /** + * Set connection timeout. A timeout value of zero is interpreted as an + * infinite timeout. + * + * @param timeout + * connection Timeout in milliseconds + */ + public static void setConnectionTimeout(final HttpClient httpClient, + int timeout) { + HttpConnectionParams.setConnectionTimeout(httpClient.getParams(), + timeout); + } + + /** + * Set follow redirects. + * + * @param followRedirects When true the client will follow redirects. + */ + public static void setFollowRedirects(HttpClient httpClient, + boolean followRedirects) { + new ClientParamBean(httpClient.getParams()).setHandleRedirects(followRedirects); + } + + private static class UseCompressionRequestInterceptor implements + HttpRequestInterceptor { + + @Override + public void process(HttpRequest request, HttpContext context) + throws HttpException, IOException { + if (!request.containsHeader("Accept-Encoding")) { + request.addHeader("Accept-Encoding", "gzip, deflate"); + } + } + } + + private static class UseCompressionResponseInterceptor implements + HttpResponseInterceptor { + + public void process(final HttpResponse response, final HttpContext context) + throws HttpException, IOException { + + HttpEntity entity = response.getEntity(); + Header ceheader = entity.getContentEncoding(); + if (ceheader != null) { + HeaderElement[] codecs = ceheader.getElements(); + for (int i = 0; i < codecs.length; i++) { + if (codecs[i].getName().equalsIgnoreCase("gzip")) { + response + .setEntity(new GzipDecompressingEntity(response.getEntity())); + return; + } + if (codecs[i].getName().equalsIgnoreCase("deflate")) { + response.setEntity(new DeflateDecompressingEntity(response + .getEntity())); + return; + } + } + } + } + } + + private static class GzipDecompressingEntity extends HttpEntityWrapper { + public GzipDecompressingEntity(final HttpEntity entity) { + super(entity); + } + + public InputStream getContent() throws IOException, IllegalStateException { + return new GZIPInputStream(wrappedEntity.getContent()); + } + + public long getContentLength() { + return -1; + } + } + + private static class DeflateDecompressingEntity extends + GzipDecompressingEntity { + public DeflateDecompressingEntity(final HttpEntity entity) { + super(entity); + } + + public InputStream getContent() throws IOException, IllegalStateException { + return new InflaterInputStream(wrappedEntity.getContent()); + } + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java index 41461b3ce5c..a91d522fd45 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java @@ -25,17 +25,9 @@ import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; -import org.apache.http.HttpResponseInterceptor; import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; import org.apache.http.NoHttpResponseException; @@ -45,12 +37,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.params.ClientPNames; -import org.apache.http.client.params.ClientParamBean; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.entity.HttpEntityWrapper; +import org.apache.http.conn.ClientConnectionManager; import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.mime.FormBodyPart; import org.apache.http.entity.mime.HttpMultipartMode; @@ -58,11 +45,8 @@ import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.InputStreamBody; import org.apache.http.entity.mime.content.StringBody; import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicNameValuePair; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import org.apache.solr.client.solrj.ResponseParser; import org.apache.solr.client.solrj.SolrRequest; @@ -130,8 +114,9 @@ public class HttpSolrServer extends SolrServer { private int maxRetries = 0; - private ThreadSafeClientConnManager ccm; private boolean useMultiPartPost; + private final boolean internalClient; + /** * @param baseURL @@ -160,31 +145,19 @@ public class HttpSolrServer extends SolrServer { if (client != null) { httpClient = client; + internalClient = false; } else { - httpClient = createClient(); + internalClient = true; + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32); + params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, followRedirects); + httpClient = HttpClientUtil.createClient(params); } this.parser = parser; } - private DefaultHttpClient createClient() { - SchemeRegistry schemeRegistry = new SchemeRegistry(); - schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory - .getSocketFactory())); - schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory - .getSocketFactory())); - - ccm = new ThreadSafeClientConnManager(schemeRegistry); - // Increase default max connection per route to 32 - ccm.setDefaultMaxPerRoute(32); - // Increase max total connection to 128 - ccm.setMaxTotal(128); - DefaultHttpClient httpClient = new DefaultHttpClient(ccm); - httpClient.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, - followRedirects); - return httpClient; - } - /** * Process the request. If * {@link org.apache.solr.client.solrj.SolrRequest#getResponseParser()} is @@ -301,7 +274,6 @@ public class HttpSolrServer extends SolrServer { post.setEntity(entity); } else { //not using multipart - HttpEntity e; post.setEntity(new UrlEncodedFormEntity(postParams, "UTF-8")); } @@ -367,7 +339,7 @@ public class HttpSolrServer extends SolrServer { throw new SolrServerException("error reading streams", ex); } - // TODO: move to a interceptor? + // XXX client already has this set, is this needed? method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, followRedirects); method.addHeader("User-Agent", AGENT); @@ -482,6 +454,9 @@ public class HttpSolrServer extends SolrServer { parser = processor; } + /** + * Return the HttpClient this instance uses. + */ public HttpClient getHttpClient() { return httpClient; } @@ -493,22 +468,22 @@ public class HttpSolrServer extends SolrServer { * Timeout in milliseconds **/ public void setConnectionTimeout(int timeout) { - HttpConnectionParams.setConnectionTimeout(httpClient.getParams(), timeout); + HttpClientUtil.setConnectionTimeout(httpClient, timeout); } /** - * Sets HttpConnectionParams.setSoTimeout (read timeout). This is desirable + * Set SoTimeout (read timeout). This is desirable * for queries, but probably not for indexing. * * @param timeout * Timeout in milliseconds **/ public void setSoTimeout(int timeout) { - HttpConnectionParams.setSoTimeout(httpClient.getParams(), timeout); + HttpClientUtil.setSoTimeout(httpClient, timeout); } /** - * HttpClientParams.setRedirecting + * Configure whether the client should follow redirects or not. *

* This defaults to false under the assumption that if you are following a * redirect to get to a Solr installation, something is misconfigured @@ -516,90 +491,19 @@ public class HttpSolrServer extends SolrServer { *

*/ public void setFollowRedirects(boolean followRedirects) { - this.followRedirects = followRedirects; - new ClientParamBean(httpClient.getParams()) - .setHandleRedirects(followRedirects); - } - - private static class UseCompressionRequestInterceptor implements - HttpRequestInterceptor { - - @Override - public void process(HttpRequest request, HttpContext context) - throws HttpException, IOException { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip, deflate"); - } - } - } - - private static class UseCompressionResponseInterceptor implements - HttpResponseInterceptor { - - public void process(final HttpResponse response, final HttpContext context) - throws HttpException, IOException { - - HttpEntity entity = response.getEntity(); - Header ceheader = entity.getContentEncoding(); - if (ceheader != null) { - HeaderElement[] codecs = ceheader.getElements(); - for (int i = 0; i < codecs.length; i++) { - if (codecs[i].getName().equalsIgnoreCase("gzip")) { - response - .setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - if (codecs[i].getName().equalsIgnoreCase("deflate")) { - response.setEntity(new DeflateDecompressingEntity(response - .getEntity())); - return; - } - } - } - } - } - - private static class GzipDecompressingEntity extends HttpEntityWrapper { - public GzipDecompressingEntity(final HttpEntity entity) { - super(entity); - } - - public InputStream getContent() throws IOException, IllegalStateException { - return new GZIPInputStream(wrappedEntity.getContent()); - } - - public long getContentLength() { - return -1; - } - } - - private static class DeflateDecompressingEntity extends - GzipDecompressingEntity { - public DeflateDecompressingEntity(final HttpEntity entity) { - super(entity); - } - - public InputStream getContent() throws IOException, IllegalStateException { - return new InflaterInputStream(wrappedEntity.getContent()); - } + this.followRedirects = true; + HttpClientUtil.setFollowRedirects(httpClient, followRedirects); } /** * Allow server->client communication to be compressed. Currently gzip and * deflate are supported. If the server supports compression the response will - * be compressed. + * be compressed. This method is only allowed if the http client is of type + * DefatulHttpClient. */ public void setAllowCompression(boolean allowCompression) { if (httpClient instanceof DefaultHttpClient) { - final DefaultHttpClient client = (DefaultHttpClient) httpClient; - client - .removeRequestInterceptorByClass(UseCompressionRequestInterceptor.class); - client - .removeResponseInterceptorByClass(UseCompressionResponseInterceptor.class); - if (allowCompression) { - client.addRequestInterceptor(new UseCompressionRequestInterceptor()); - client.addResponseInterceptor(new UseCompressionResponseInterceptor()); - } + HttpClientUtil.setAllowCompression((DefaultHttpClient) httpClient, allowCompression); } else { throw new UnsupportedOperationException( "HttpClient instance was not of type DefaultHttpClient"); @@ -617,7 +521,7 @@ public class HttpSolrServer extends SolrServer { */ public void setMaxRetries(int maxRetries) { if (maxRetries > 1) { - log.warn("CommonsHttpSolrServer: maximum Retries " + maxRetries + log.warn("HttpSolrServer: maximum Retries " + maxRetries + " > 1. Maximum recommended retries is 1."); } this.maxRetries = maxRetries; @@ -672,15 +576,23 @@ public class HttpSolrServer extends SolrServer { return req.process(this); } + /** + * Close the {@link ClientConnectionManager} from the internal client. + */ public void shutdown() { - if (httpClient != null) { + if (httpClient != null && internalClient) { httpClient.getConnectionManager().shutdown(); } } - + + /** + * Set the maximum number of connections that can be open to a single host at + * any given time. If http client was created outside the operation is not + * allowed. + */ public void setDefaultMaxConnectionsPerHost(int max) { - if (ccm != null) { - ccm.setDefaultMaxPerRoute(max); + if (internalClient) { + HttpClientUtil.setMaxConnectionsPerHost(httpClient, max); } else { throw new UnsupportedOperationException( "Client was created outside of HttpSolrServer"); @@ -689,10 +601,11 @@ public class HttpSolrServer extends SolrServer { /** * Set the maximum number of connections that can be open at any given time. + * If http client was created outside the operation is not allowed. */ public void setMaxTotalConnections(int max) { - if (ccm != null) { - ccm.setMaxTotal(max); + if (internalClient) { + HttpClientUtil.setMaxConnections(httpClient, max); } else { throw new UnsupportedOperationException( "Client was created outside of HttpSolrServer"); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java index 9d68421edbf..a22823a0bee 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java @@ -17,12 +17,9 @@ package org.apache.solr.client.solrj.impl; import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; import org.apache.solr.client.solrj.*; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.SolrException; @@ -82,7 +79,8 @@ public class LBHttpSolrServer extends SolrServer { private ScheduledExecutorService aliveCheckExecutor; - private HttpClient httpClient; + private final HttpClient httpClient; + private final boolean clientIsInternal; private final AtomicInteger counter = new AtomicInteger(-1); private static final SolrQuery solrQuery = new SolrQuery("*:*"); @@ -177,14 +175,7 @@ public class LBHttpSolrServer extends SolrServer { } public LBHttpSolrServer(String... solrServerUrls) throws MalformedURLException { - this(getDefaultClient(), solrServerUrls); - } - - private static HttpClient getDefaultClient(){ - DefaultHttpClient client = new DefaultHttpClient(new ThreadSafeClientConnManager());; - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); - return client; + this(null, solrServerUrls); } /** The provided httpClient should use a multi-threaded connection manager */ @@ -196,7 +187,14 @@ public class LBHttpSolrServer extends SolrServer { /** The provided httpClient should use a multi-threaded connection manager */ public LBHttpSolrServer(HttpClient httpClient, ResponseParser parser, String... solrServerUrl) throws MalformedURLException { - this.httpClient = httpClient; + clientIsInternal = (httpClient == null); + if (httpClient == null) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_USE_RETRY, false); + this.httpClient = HttpClientUtil.createClient(params); + } else { + this.httpClient = httpClient; + } for (String s : solrServerUrl) { ServerWrapper wrapper = new ServerWrapper(makeServer(s)); aliveServers.put(wrapper.getKey(), wrapper); @@ -389,7 +387,7 @@ public class LBHttpSolrServer extends SolrServer { } public void setConnectionTimeout(int timeout) { - httpClient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); + HttpClientUtil.setConnectionTimeout(httpClient, timeout); } /** @@ -397,13 +395,16 @@ public class LBHttpSolrServer extends SolrServer { * not for indexing. */ public void setSoTimeout(int timeout) { - httpClient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, timeout); + HttpClientUtil.setSoTimeout(httpClient, timeout); } public void shutdown() { if (aliveCheckExecutor != null) { aliveCheckExecutor.shutdownNow(); } + if(clientIsInternal) { + httpClient.getConnectionManager().shutdown(); + } } /** diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java index 23310df6ea2..1ec5d6bad6a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java @@ -17,9 +17,9 @@ package org.apache.solr.client.solrj; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; /** @@ -37,8 +37,8 @@ public class SolrExceptionTest extends LuceneTestCase { try { // switched to a local address to avoid going out on the net, ns lookup issues, etc. // set a 1ms timeout to let the connection fail faster. - DefaultHttpClient httpClient = new DefaultHttpClient(new ThreadSafeClientConnManager()); - httpClient.getParams().setIntParameter("http.connection.timeout", 1); + HttpClient httpClient = HttpClientUtil.createClient(null); + HttpClientUtil.setConnectionTimeout(httpClient, 1); SolrServer client = new HttpSolrServer("http://[ff01::114]:11235/solr/", httpClient); SolrQuery query = new SolrQuery("test123"); client.query(query); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java index b1e9e2a3930..5c8e427fa38 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java @@ -19,16 +19,17 @@ package org.apache.solr.client.solrj; import junit.framework.Assert; import org.apache.commons.io.FileUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.LBHttpSolrServer; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.util.AbstractSolrTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -47,7 +48,7 @@ import java.util.Set; */ public class TestLBHttpSolrServer extends LuceneTestCase { SolrInstance[] solr = new SolrInstance[3]; - DefaultHttpClient httpClient; + HttpClient httpClient; // TODO: fix this test to not require FSDirectory static String savedFactory; @@ -70,9 +71,8 @@ public class TestLBHttpSolrServer extends LuceneTestCase { @Override public void setUp() throws Exception { super.setUp(); - httpClient = new DefaultHttpClient(new ThreadSafeClientConnManager()); - - httpClient.getParams().setParameter("http.connection.timeout", new Integer(1000)); + httpClient = HttpClientUtil.createClient(null); + HttpClientUtil.setConnectionTimeout(httpClient, 1000); for (int i = 0; i < solr.length; i++) { solr[i] = new SolrInstance("solr" + i, 0); solr[i].setUp(); @@ -180,10 +180,11 @@ public class TestLBHttpSolrServer extends LuceneTestCase { for (int i = 0; i < solr.length; i++) { s[i] = solr[i].getUrl(); } - DefaultHttpClient myHttpClient = new DefaultHttpClient(new ThreadSafeClientConnManager()); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 250); + params.set(HttpClientUtil.PROP_SO_TIMEOUT, 250); + HttpClient myHttpClient = HttpClientUtil.createClient(params); - myHttpClient.getParams().setParameter("http.connection.timeout", new Integer(250)); - myHttpClient.getParams().setParameter("http.socket.timeout", new Integer(250)); LBHttpSolrServer lbHttpSolrServer = new LBHttpSolrServer(myHttpClient, s); lbHttpSolrServer.setAliveCheckInterval(500); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java index d9d2e9e2fd9..b726a8a419e 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java @@ -32,8 +32,8 @@ import javax.servlet.http.HttpServletResponse; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.DefaultHttpClient; import org.apache.solr.SolrJettyTestBase; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest.METHOD; @@ -384,7 +384,7 @@ public class BasicHttpSolrServerTest extends SolrJettyTestBase { HttpGet get = new HttpGet("http://127.0.0.1:" + jetty.getLocalPort() + "/solr/select?q=foo&wt=xml"); get.setHeader("Accept-Encoding", "gzip"); - DefaultHttpClient client = new DefaultHttpClient(); + HttpClient client = HttpClientUtil.createClient(null); HttpEntity entity = null; try { HttpResponse response = client.execute(get); @@ -408,6 +408,20 @@ public class BasicHttpSolrServerTest extends SolrJettyTestBase { assertEquals(0, response.getStatus()); } + @Test + public void testSetParametersExternalClient(){ + HttpClient client = HttpClientUtil.createClient(null); + HttpSolrServer server = new HttpSolrServer("http://127.0.0.1/", client); + try { + server.setMaxTotalConnections(1); + fail("Operation should not succeed."); + } catch (UnsupportedOperationException e) {} + try { + server.setDefaultMaxConnectionsPerHost(1); + fail("Operation should not succeed."); + } catch (UnsupportedOperationException e) {} + } + private int findUnusedPort() { for (int port = 0; port < 65535; port++) { Socket s = new Socket(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java new file mode 100644 index 00000000000..ee2764f4cdf --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.impl; + +import static org.junit.Assert.assertEquals; + +import org.apache.http.auth.AuthScope; +import org.apache.http.client.HttpClient; +import org.apache.http.client.params.ClientPNames; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.params.HttpConnectionParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Test; + +public class HttpClientUtilTest { + + @Test + public void testNoParamsSucceeds() { + HttpClient clien = HttpClientUtil.createClient(null); + clien.getConnectionManager().shutdown(); + } + + @Test + public void testSetParams() { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, true); + params.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, "pass"); + params.set(HttpClientUtil.PROP_BASIC_AUTH_USER, "user"); + params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 12345); + params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, true); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 22345); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32345); + params.set(HttpClientUtil.PROP_SO_TIMEOUT, 42345); + params.set(HttpClientUtil.PROP_USE_RETRY, false); + DefaultHttpClient client = (DefaultHttpClient) HttpClientUtil.createClient(params); + assertEquals(12345, HttpConnectionParams.getConnectionTimeout(client.getParams())); + assertEquals(ThreadSafeClientConnManager.class, client.getConnectionManager().getClass()); + assertEquals(22345, ((ThreadSafeClientConnManager)client.getConnectionManager()).getMaxTotal()); + assertEquals(32345, ((ThreadSafeClientConnManager)client.getConnectionManager()).getDefaultMaxPerRoute()); + assertEquals(42345, HttpConnectionParams.getSoTimeout(client.getParams())); + assertEquals(HttpClientUtil.NO_RETRY, client.getHttpRequestRetryHandler()); + assertEquals("pass", client.getCredentialsProvider().getCredentials(new AuthScope("127.0.0.1", 1234)).getPassword()); + assertEquals("user", client.getCredentialsProvider().getCredentials(new AuthScope("127.0.0.1", 1234)).getUserPrincipal().getName()); + assertEquals(true, client.getParams().getParameter(ClientPNames.HANDLE_REDIRECTS)); + client.getConnectionManager().shutdown(); + } + +}