Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-7280: refactored to incorporate Mike's suggestions. Default thread count for cloud is limited to 8 now. In our internal teting 8 has given us the best stability during restarts

This commit is contained in:
Noble Paul 2016-07-19 18:21:23 +05:30
parent 5c4b7173a8
commit 2d1496c83d
4 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -16,19 +16,6 @@
*/ */
package org.apache.solr.core; package org.apache.solr.core;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.EMPTY_MAP;
import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.core.NodeConfig.NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS;
import static org.apache.solr.core.NodeConfig.NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS_IN_CLOUD;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.nio.file.Path; import java.nio.file.Path;
@ -46,6 +33,8 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.http.auth.AuthSchemeProvider; import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.client.CredentialsProvider; import org.apache.http.client.CredentialsProvider;
import org.apache.http.config.Lookup; import org.apache.http.config.Lookup;
@ -88,8 +77,16 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap; import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.Maps; import static java.util.Collections.EMPTY_MAP;
import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
/** /**
@ -489,7 +486,7 @@ public class CoreContainer {
// setup executor to load cores in parallel // setup executor to load cores in parallel
ExecutorService coreLoadExecutor = ExecutorUtil.newMDCAwareFixedThreadPool( ExecutorService coreLoadExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(
cfg.getCoreLoadThreadCount(isZooKeeperAware() ? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD : DEFAULT_CORE_LOAD_THREADS), cfg.getCoreLoadThreadCount(isZooKeeperAware()),
new DefaultSolrThreadFactory("coreLoadExecutor") ); new DefaultSolrThreadFactory("coreLoadExecutor") );
final List<Future<SolrCore>> futures = new ArrayList<>(); final List<Future<SolrCore>> futures = new ArrayList<>();
try { try {

12
solr/core/src/java/org/apache/solr/core/NodeConfig.java
View File

@ -87,7 +87,7 @@ public class NodeConfig {
this.solrProperties = solrProperties; this.solrProperties = solrProperties;
this.backupRepositoryPlugins = backupRepositoryPlugins; this.backupRepositoryPlugins = backupRepositoryPlugins;
if (this.cloudConfig != null && this.getCoreLoadThreadCount(NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS) < 2) { if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"SolrCloud requires a value of at least 2 for coreLoadThreads (configured value = " + this.coreLoadThreads + ")"); "SolrCloud requires a value of at least 2 for coreLoadThreads (configured value = " + this.coreLoadThreads + ")");
} }
@ -109,8 +109,10 @@ public class NodeConfig {
return updateShardHandlerConfig; return updateShardHandlerConfig;
} }
public int getCoreLoadThreadCount(int def) { public int getCoreLoadThreadCount(boolean zkAware) {
return coreLoadThreads == null ? def : coreLoadThreads; return coreLoadThreads == null ?
(zkAware ? NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS_IN_CLOUD : NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS)
: coreLoadThreads;
} }
public String getSharedLibDirectory() { public String getSharedLibDirectory() {
@ -196,8 +198,8 @@ public class NodeConfig {
private final String nodeName; private final String nodeName;
public static final int DEFAULT_CORE_LOAD_THREADS = 3; public static final int DEFAULT_CORE_LOAD_THREADS = 3;
//No:of core load threads in cloud mode is set to a default of 24 //No:of core load threads in cloud mode is set to a default of 8
public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 24; public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 8;
private static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE; private static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;

34
solr/core/src/test/org/apache/solr/core/CoreSorterTest.java
View File

@ -39,7 +39,7 @@ import org.apache.solr.util.MockCoreContainer;
import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toList;
import static org.apache.solr.core.CoreSorter.getShardName; import static org.apache.solr.core.CoreSorter.getShardName;
import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.reset;
@ -49,16 +49,16 @@ public class CoreSorterTest extends SolrTestCaseJ4 {
public void testComparator() { public void testComparator() {
List<CountsForEachShard> l = new ArrayList<>(); List<CountsForEachShard> l = new ArrayList<>();
// DN LIV MY // DOWN LIVE MY
l.add(new CountsForEachShard(1, 3, 1)); l.add(new CountsForEachShard(1, 3, 1));
l.add(new CountsForEachShard(0, 3, 2)); l.add(new CountsForEachShard(0, 3, 2));
l.add(new CountsForEachShard(0, 3, 3)); l.add(new CountsForEachShard(0, 3, 3));
l.add(new CountsForEachShard(0, 3, 4)); l.add(new CountsForEachShard(0, 3, 4));
l.add(new CountsForEachShard(1, 0, 2)); l.add(new CountsForEachShard(1, 0, 2));
l.add(new CountsForEachShard(1, 0, 1)); l.add(new CountsForEachShard(1, 0, 1));
l.add(new CountsForEachShard(2, 5, 1)); l.add(new CountsForEachShard(2, 5, 1));
l.add(new CountsForEachShard(2, 4, 2)); l.add(new CountsForEachShard(2, 4, 2));
l.add(new CountsForEachShard(2, 3, 3)); l.add(new CountsForEachShard(2, 3, 3));
List<CountsForEachShard> expected = Arrays.asList( List<CountsForEachShard> expected = Arrays.asList(
new CountsForEachShard(0, 3, 2), new CountsForEachShard(0, 3, 2),
@ -108,14 +108,10 @@ public class CoreSorterTest extends SolrTestCaseJ4 {
ZkController mockZKC = createMock(ZkController.class); ZkController mockZKC = createMock(ZkController.class);
ClusterState mockClusterState = createMock(ClusterState.class); ClusterState mockClusterState = createMock(ClusterState.class);
reset(mockCC, mockZKC, mockClusterState); reset(mockCC, mockZKC, mockClusterState);
mockCC.isZooKeeperAware(); expect(mockCC.isZooKeeperAware()).andReturn(Boolean.TRUE).anyTimes();
expectLastCall().andAnswer(() -> Boolean.TRUE).anyTimes(); expect(mockCC.getZkController()).andReturn(mockZKC).anyTimes();
mockCC.getZkController(); expect(mockClusterState.getLiveNodes()).andReturn(liveNodes).anyTimes();
expectLastCall().andAnswer(() -> mockZKC).anyTimes(); expect(mockZKC.getClusterState()).andReturn(mockClusterState).anyTimes();
mockClusterState.getLiveNodes();
expectLastCall().andAnswer(() -> liveNodes).anyTimes();
mockZKC.getClusterState();
expectLastCall().andAnswer(() -> mockClusterState).anyTimes();
replay(mockCC, mockZKC, mockClusterState); replay(mockCC, mockZKC, mockClusterState);
return mockCC; return mockCC;
} }

2
solr/core/src/test/org/apache/solr/core/TestSolrXml.java
View File

@ -73,7 +73,7 @@ public class TestSolrXml extends SolrTestCaseJ4 {
assertEquals("collection handler class", "testCollectionsHandler", cfg.getCollectionsHandlerClass()); assertEquals("collection handler class", "testCollectionsHandler", cfg.getCollectionsHandlerClass());
assertEquals("info handler class", "testInfoHandler", cfg.getInfoHandlerClass()); assertEquals("info handler class", "testInfoHandler", cfg.getInfoHandlerClass());
assertEquals("config set handler class", "testConfigSetsHandler", cfg.getConfigSetsHandlerClass()); assertEquals("config set handler class", "testConfigSetsHandler", cfg.getConfigSetsHandlerClass());
assertEquals("core load threads", 11, cfg.getCoreLoadThreadCount(0)); assertEquals("core load threads", 11, cfg.getCoreLoadThreadCount(false));
assertThat("core root dir", cfg.getCoreRootDirectory().toString(), containsString("testCoreRootDirectory")); assertThat("core root dir", cfg.getCoreRootDirectory().toString(), containsString("testCoreRootDirectory"));
assertEquals("distrib conn timeout", 22, cfg.getUpdateShardHandlerConfig().getDistributedConnectionTimeout()); assertEquals("distrib conn timeout", 22, cfg.getUpdateShardHandlerConfig().getDistributedConnectionTimeout());
assertEquals("distrib socket timeout", 33, cfg.getUpdateShardHandlerConfig().getDistributedSocketTimeout()); assertEquals("distrib socket timeout", 33, cfg.getUpdateShardHandlerConfig().getDistributedSocketTimeout());