Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages

This commit is contained in:
Cao Manh Dat 2017-05-08 16:21:19 +07:00
parent 37df64450e
commit 20c4886816
3 changed files with 6 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -254,6 +254,8 @@ Optimizations
* SOLR-10548: JSON Facet API now uses hyper-log-log++ for determining the number of buckets
when merging requests from a multi-shard distributed request. (yonik)
* SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages (Cao Manh Dat, Noble Paul, shalin, Scott Blum)
Bug Fixes
----------------------
* SOLR-10281: ADMIN_PATHS is duplicated in two places and inconsistent. This can cause automatic

3
solr/core/src/java/org/apache/solr/cloud/Overseer.java
View File

@ -64,7 +64,8 @@ import static org.apache.solr.common.params.CommonParams.ID;
public class Overseer implements Closeable {
public static final String QUEUE_OPERATION = "operation";
public static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
public static final int STATE_UPDATE_DELAY = 2000; // delay between cloud state updates
public static final int STATE_UPDATE_BATCH_SIZE = 10000;
public static final int NUM_RESPONSES_TO_STORE = 10000;
public static final String OVERSEER_ELECT = "/overseer_elect";

14
solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
View File

@ -166,17 +166,7 @@ public class ZkStateWriter {
* @return true if a flush is required, false otherwise
*/
protected boolean maybeFlushBefore(ZkWriteCommand cmd) {
if (lastUpdatedTime == 0) {
// first update, make sure we go through
return false;
}
if (cmd.collection == null) {
return false;
}
if (cmd.collection.getStateFormat() != lastStateFormat) {
return true;
}
return cmd.collection.getStateFormat() > 1 && !cmd.name.equals(lastCollectionName);
return cmd.collection.getStateFormat() != lastStateFormat;
}
/**
@ -190,7 +180,7 @@ public class ZkStateWriter {
return false;
lastCollectionName = cmd.name;
lastStateFormat = cmd.collection.getStateFormat();
return System.nanoTime() - lastUpdatedTime > MAX_FLUSH_INTERVAL;
return System.nanoTime() - lastUpdatedTime > MAX_FLUSH_INTERVAL || updates.size() > Overseer.STATE_UPDATE_BATCH_SIZE;
}
public boolean hasPendingUpdates() {