SOLR-12314: Use http timeout's defined in solr.xml for creating ConcurrentUpdateSolrClient during indexing requests between leader and replica

This commit is contained in:
Varun Thacker 2018-05-28 11:21:49 -07:00
parent 11cfb86489
commit 071df6e114
4 changed files with 34 additions and 1 deletions

View File

@ -260,6 +260,9 @@ Bug Fixes
* SOLR-12294: update processors loaded from runtime jars fail to load if they are specified * SOLR-12294: update processors loaded from runtime jars fail to load if they are specified
in an update processor chain (noble) in an update processor chain (noble)
* SOLR-12314: Use http timeout's defined in solr.xml for creating ConcurrentUpdateSolrClient during
indexing requests between leader and replica ( Mark Miller, Varun Thacker)
Optimizations Optimizations
---------------------- ----------------------

View File

@ -51,10 +51,15 @@ public class StreamingSolrClients {
private ExecutorService updateExecutor; private ExecutorService updateExecutor;
private int socketTimeout;
private int connectionTimeout;
public StreamingSolrClients(UpdateShardHandler updateShardHandler) { public StreamingSolrClients(UpdateShardHandler updateShardHandler) {
this.updateExecutor = updateShardHandler.getUpdateExecutor(); this.updateExecutor = updateShardHandler.getUpdateExecutor();
httpClient = updateShardHandler.getUpdateOnlyHttpClient(); httpClient = updateShardHandler.getUpdateOnlyHttpClient();
socketTimeout = updateShardHandler.getSocketTimeout();
connectionTimeout = updateShardHandler.getConnectionTimeout();
} }
public List<Error> getErrors() { public List<Error> getErrors() {
@ -78,6 +83,8 @@ public class StreamingSolrClients {
.withThreadCount(runnerCount) .withThreadCount(runnerCount)
.withExecutorService(updateExecutor) .withExecutorService(updateExecutor)
.alwaysStreamDeletes() .alwaysStreamDeletes()
.withSocketTimeout(socketTimeout)
.withConnectionTimeout(connectionTimeout)
.build(); .build();
client.setPollQueueTime(Integer.MAX_VALUE); // minimize connections created client.setPollQueueTime(Integer.MAX_VALUE); // minimize connections created
client.setParser(new BinaryResponseParser()); client.setParser(new BinaryResponseParser());

View File

@ -78,6 +78,9 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
private final Set<String> metricNames = ConcurrentHashMap.newKeySet(); private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; private MetricRegistry registry;
private int socketTimeout = UpdateShardHandlerConfig.DEFAULT_DISTRIBUPDATESOTIMEOUT;
private int connectionTimeout = UpdateShardHandlerConfig.DEFAULT_DISTRIBUPDATECONNTIMEOUT;
public UpdateShardHandler(UpdateShardHandlerConfig cfg) { public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
updateOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry()); updateOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry()); defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
@ -92,6 +95,8 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
if (cfg != null) { if (cfg != null) {
clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout()); clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout()); clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
socketTimeout = cfg.getDistributedSocketTimeout();
connectionTimeout = cfg.getDistributedConnectionTimeout();
} }
HttpClientMetricNameStrategy metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY); HttpClientMetricNameStrategy metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY);
if (cfg != null) { if (cfg != null) {
@ -210,4 +215,12 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
} }
} }
public int getSocketTimeout() {
return socketTimeout;
}
public int getConnectionTimeout() {
return connectionTimeout;
}
} }

View File

@ -43,6 +43,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.update.DirectUpdateHandler2; import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.UpdateLog; import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.UpdateShardHandler;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
@ -68,6 +69,15 @@ public class TestCloudRecovery extends SolrCloudTestCase {
.process(cluster.getSolrClient()); .process(cluster.getSolrClient());
AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
false, true, 30); false, true, 30);
//SOLR-12314 : assert that these values are from the solr.xml file and not UpdateShardHandlerConfig#DEFAULT
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
UpdateShardHandler shardHandler = jettySolrRunner.getCoreContainer().getUpdateShardHandler();
int socketTimeout = shardHandler.getSocketTimeout();
int connectionTimeout = shardHandler.getConnectionTimeout();
assertEquals(340000, socketTimeout);
assertEquals(45000, connectionTimeout);
}
} }
@Before @Before