Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-7280: refactored to incorporate Mike's suggestions. Default thread count for cloud is limited to 8 now. In our internal teting 8 has given us the best stability during restarts

This commit is contained in:
Noble Paul 2016-07-19 18:21:23 +05:30
parent 5c4b7173a8
commit 2d1496c83d
4 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -16,19 +16,6 @@
*/
package org.apache.solr.core;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.EMPTY_MAP;
import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.core.NodeConfig.NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS;
import static org.apache.solr.core.NodeConfig.NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS_IN_CLOUD;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
@ -46,6 +33,8 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.config.Lookup;
@ -88,8 +77,16 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.EMPTY_MAP;
import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
/**
@ -489,7 +486,7 @@ public class CoreContainer {
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(
cfg.getCoreLoadThreadCount(isZooKeeperAware() ? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD : DEFAULT_CORE_LOAD_THREADS),
cfg.getCoreLoadThreadCount(isZooKeeperAware()),
new DefaultSolrThreadFactory("coreLoadExecutor") );
final List<Future<SolrCore>> futures = new ArrayList<>();
try {

12
solr/core/src/java/org/apache/solr/core/NodeConfig.java
View File

@ -87,7 +87,7 @@ public class NodeConfig {
this.solrProperties = solrProperties;
this.backupRepositoryPlugins = backupRepositoryPlugins;
if (this.cloudConfig != null && this.getCoreLoadThreadCount(NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS) < 2) {
if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"SolrCloud requires a value of at least 2 for coreLoadThreads (configured value = " + this.coreLoadThreads + ")");
}
@ -109,8 +109,10 @@ public class NodeConfig {
return updateShardHandlerConfig;
}
public int getCoreLoadThreadCount(int def) {
return coreLoadThreads == null ? def : coreLoadThreads;
public int getCoreLoadThreadCount(boolean zkAware) {
return coreLoadThreads == null ?
(zkAware ? NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS_IN_CLOUD : NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS)
: coreLoadThreads;
}
public String getSharedLibDirectory() {
@ -196,8 +198,8 @@ public class NodeConfig {
private final String nodeName;
public static final int DEFAULT_CORE_LOAD_THREADS = 3;
//No:of core load threads in cloud mode is set to a default of 24
public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 24;
//No:of core load threads in cloud mode is set to a default of 8
public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 8;
private static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;

16
solr/core/src/test/org/apache/solr/core/CoreSorterTest.java
View File

@ -39,7 +39,7 @@ import org.apache.solr.util.MockCoreContainer;
import static java.util.stream.Collectors.toList;
import static org.apache.solr.core.CoreSorter.getShardName;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
@ -49,7 +49,7 @@ public class CoreSorterTest extends SolrTestCaseJ4 {
public void testComparator() {
List<CountsForEachShard> l = new ArrayList<>();
// DN LIV MY
// DOWN LIVE MY
l.add(new CountsForEachShard(1, 3, 1));
l.add(new CountsForEachShard(0, 3, 2));
l.add(new CountsForEachShard(0, 3, 3));
@ -108,14 +108,10 @@ public class CoreSorterTest extends SolrTestCaseJ4 {
ZkController mockZKC = createMock(ZkController.class);
ClusterState mockClusterState = createMock(ClusterState.class);
reset(mockCC, mockZKC, mockClusterState);
mockCC.isZooKeeperAware();
expectLastCall().andAnswer(() -> Boolean.TRUE).anyTimes();
mockCC.getZkController();
expectLastCall().andAnswer(() -> mockZKC).anyTimes();
mockClusterState.getLiveNodes();
expectLastCall().andAnswer(() -> liveNodes).anyTimes();
mockZKC.getClusterState();
expectLastCall().andAnswer(() -> mockClusterState).anyTimes();
expect(mockCC.isZooKeeperAware()).andReturn(Boolean.TRUE).anyTimes();
expect(mockCC.getZkController()).andReturn(mockZKC).anyTimes();
expect(mockClusterState.getLiveNodes()).andReturn(liveNodes).anyTimes();
expect(mockZKC.getClusterState()).andReturn(mockClusterState).anyTimes();
replay(mockCC, mockZKC, mockClusterState);
return mockCC;
}

2
solr/core/src/test/org/apache/solr/core/TestSolrXml.java
View File

@ -73,7 +73,7 @@ public class TestSolrXml extends SolrTestCaseJ4 {
assertEquals("collection handler class", "testCollectionsHandler", cfg.getCollectionsHandlerClass());
assertEquals("info handler class", "testInfoHandler", cfg.getInfoHandlerClass());
assertEquals("config set handler class", "testConfigSetsHandler", cfg.getConfigSetsHandlerClass());
assertEquals("core load threads", 11, cfg.getCoreLoadThreadCount(0));
assertEquals("core load threads", 11, cfg.getCoreLoadThreadCount(false));
assertThat("core root dir", cfg.getCoreRootDirectory().toString(), containsString("testCoreRootDirectory"));
assertEquals("distrib conn timeout", 22, cfg.getUpdateShardHandlerConfig().getDistributedConnectionTimeout());
assertEquals("distrib socket timeout", 33, cfg.getUpdateShardHandlerConfig().getDistributedSocketTimeout());