Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14641: PeerSync, remove canHandleVersionRanges check (#1663)

This commit is contained in:
Cao Manh Dat 2020-08-10 15:04:34 +07:00 committed by GitHub
parent 35771c3cfe
commit 57b0160659
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 13 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -233,8 +233,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
validateMemoryBreakerThreshold();
useRangeVersionsForPeerSync = getBool("peerSync/useRangeVersions", true);
filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
@ -533,9 +531,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
// Circuit Breaker Configuration
public final boolean useCircuitBreakers;
public final int memoryCircuitBreakerThresholdPct;
public final boolean useRangeVersionsForPeerSync;
// IndexConfig settings
public final SolrIndexConfig indexConfig;
@ -932,10 +928,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
"addHttpRequestToContext", addHttpRequestToContext));
if (indexConfig != null) result.put("indexConfig", indexConfig);
m = new LinkedHashMap();
result.put("peerSync", m);
m.put("useRangeVersions", useRangeVersionsForPeerSync);
//TODO there is more to add
return result;

10
solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
View File

@ -136,15 +136,17 @@ public class RealTimeGetComponent extends SearchComponent
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
// This seems rather kludgey, may there is better way to indicate
// that replica can support handling version ranges
//TODO remove this at Solr 10
//After SOLR-14641 other nodes won't call RTG with this param.
//Just keeping here for backward-compatibility, if we remove this, nodes with older versions will
//assume that this node can't handle version ranges.
String val = params.get("checkCanHandleVersionRanges");
if(val != null) {
rb.rsp.add("canHandleVersionRanges", true);
return;
}
val = params.get("getFingerprint");
if(val != null) {
processGetFingeprint(rb);

68
solr/core/src/java/org/apache/solr/update/PeerSync.java
View File

@ -22,17 +22,13 @@ import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.client.solrj.SolrServerException;
@ -41,7 +37,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.handler.component.ShardHandler;
@ -84,7 +79,6 @@ public class PeerSync implements SolrMetricProducer {
private final boolean cantReachIsSuccess;
private final boolean doFingerprint;
private final HttpClient client;
private final boolean onlyIfActive;
private SolrCore core;
private Updater updater;
@ -117,7 +111,6 @@ public class PeerSync implements SolrMetricProducer {
this.nUpdates = nUpdates;
this.cantReachIsSuccess = cantReachIsSuccess;
this.doFingerprint = doFingerprint && !("true".equals(System.getProperty("solr.disableFingerprint")));
this.client = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
this.onlyIfActive = onlyIfActive;
uhandler = core.getUpdateHandler();
@ -406,31 +399,6 @@ public class PeerSync implements SolrMetricProducer {
}
}
private boolean canHandleVersionRanges(String replica) {
SyncShardRequest sreq = new SyncShardRequest();
requests.add(sreq);
// determine if leader can handle version ranges
sreq.shards = new String[] {replica};
sreq.actualShards = sreq.shards;
sreq.params = new ModifiableSolrParams();
sreq.params.set("qt", "/get");
sreq.params.set(DISTRIB, false);
sreq.params.set("checkCanHandleVersionRanges", false);
ShardHandler sh = shardHandlerFactory.getShardHandler();
sh.submit(sreq, replica, sreq.params);
ShardResponse srsp = sh.takeCompletedIncludingErrors();
Boolean canHandleVersionRanges = srsp.getSolrResponse().getResponse().getBooleanArg("canHandleVersionRanges");
if (canHandleVersionRanges == null || canHandleVersionRanges.booleanValue() == false) {
return false;
}
return true;
}
private boolean handleVersions(ShardResponse srsp) {
// we retrieved the last N updates from the replica
@SuppressWarnings({"unchecked"})
@ -453,8 +421,7 @@ public class PeerSync implements SolrMetricProducer {
}
MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(
otherVersions, sreq.shards[0],
() -> core.getSolrConfig().useRangeVersionsForPeerSync && canHandleVersionRanges(sreq.shards[0]));
otherVersions, sreq.shards[0]);
if (updatesRequest == MissedUpdatesRequest.ALREADY_IN_SYNC) {
return true;
@ -717,16 +684,12 @@ public class PeerSync implements SolrMetricProducer {
}
static abstract class MissedUpdatesFinderBase {
private Set<Long> ourUpdateSet;
private Set<Long> requestedUpdateSet = new HashSet<>();
long ourLowThreshold; // 20th percentile
List<Long> ourUpdates;
MissedUpdatesFinderBase(List<Long> ourUpdates, long ourLowThreshold) {
assert sorted(ourUpdates);
this.ourUpdates = ourUpdates;
this.ourUpdateSet = new HashSet<>(ourUpdates);
this.ourLowThreshold = ourLowThreshold;
}
@ -783,26 +746,6 @@ public class PeerSync implements SolrMetricProducer {
String rangesToRequestStr = rangesToRequest.stream().collect(Collectors.joining(","));
return MissedUpdatesRequest.of(rangesToRequestStr, totalRequestedVersions);
}
MissedUpdatesRequest handleIndividualVersions(List<Long> otherVersions, boolean completeList) {
List<Long> toRequest = new ArrayList<>();
for (Long otherVersion : otherVersions) {
// stop when the entries get old enough that reorders may lead us to see updates we don't need
if (!completeList && Math.abs(otherVersion) < ourLowThreshold) break;
if (ourUpdateSet.contains(otherVersion) || requestedUpdateSet.contains(otherVersion)) {
// we either have this update, or already requested it
// TODO: what if the shard we previously requested this from returns failure (because it goes
// down)
continue;
}
toRequest.add(otherVersion);
requestedUpdateSet.add(otherVersion);
}
return MissedUpdatesRequest.of(StrUtils.join(toRequest, ','), toRequest.size());
}
}
/**
@ -824,7 +767,7 @@ public class PeerSync implements SolrMetricProducer {
this.nUpdates = nUpdates;
}
public MissedUpdatesRequest find(List<Long> otherVersions, Object updateFrom, Supplier<Boolean> canHandleVersionRanges) {
public MissedUpdatesRequest find(List<Long> otherVersions, Object updateFrom) {
otherVersions.sort(absComparator);
if (debug) {
log.debug("{} sorted versions from {} = {}", logPrefix, otherVersions, updateFrom);
@ -858,12 +801,7 @@ public class PeerSync implements SolrMetricProducer {
boolean completeList = otherVersions.size() < nUpdates;
MissedUpdatesRequest updatesRequest;
if (canHandleVersionRanges.get()) {
updatesRequest = handleVersionsWithRanges(otherVersions, completeList);
} else {
updatesRequest = handleIndividualVersions(otherVersions, completeList);
}
MissedUpdatesRequest updatesRequest = handleVersionsWithRanges(otherVersions, completeList);
if (updatesRequest.totalRequestedUpdates > nUpdates) {
log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);

25
solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
View File

@ -21,7 +21,6 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Set;
import java.util.function.Supplier;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@ -238,7 +237,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
return MissedUpdatesRequest.UNABLE_TO_SYNC;
}
MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(otherVersions, leaderUrl, () -> core.getSolrConfig().useRangeVersionsForPeerSync && canHandleVersionRanges());
MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(otherVersions, leaderUrl);
if (updatesRequest == MissedUpdatesRequest.EMPTY) {
if (doFingerprint) return MissedUpdatesRequest.UNABLE_TO_SYNC;
return MissedUpdatesRequest.ALREADY_IN_SYNC;
@ -313,19 +312,6 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
return true;
}
// determine if leader can handle version ranges
private boolean canHandleVersionRanges() {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/get");
params.set(DISTRIB, false);
params.set("checkCanHandleVersionRanges", false);
NamedList<Object> rsp = request(params, "Failed on determine if leader can handle version ranges");
Boolean canHandleVersionRanges = rsp.getBooleanArg("canHandleVersionRanges");
return canHandleVersionRanges != null && canHandleVersionRanges;
}
private NamedList<Object> request(ModifiableSolrParams params, String onFail) {
try {
QueryResponse rsp = new QueryRequest(params, SolrRequest.METHOD.POST).process(clientToLeader);
@ -404,7 +390,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
this.nUpdates = nUpdates;
}
public MissedUpdatesRequest find(List<Long> leaderVersions, Object updateFrom, Supplier<Boolean> canHandleVersionRanges) {
public MissedUpdatesRequest find(List<Long> leaderVersions, Object updateFrom) {
leaderVersions.sort(absComparator);
log.debug("{} sorted versions from {} = {}", logPrefix, leaderVersions, updateFrom);
@ -418,12 +404,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
// In that case, we will fail on compute fingerprint with the current leader and start segments replication
boolean completeList = leaderVersions.size() < nUpdates;
MissedUpdatesRequest updatesRequest;
if (canHandleVersionRanges.get()) {
updatesRequest = handleVersionsWithRanges(leaderVersions, completeList);
} else {
updatesRequest = handleIndividualVersions(leaderVersions, completeList);
}
MissedUpdatesRequest updatesRequest = handleVersionsWithRanges(leaderVersions, completeList);
if (updatesRequest.totalRequestedUpdates > nUpdates) {
log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);

3
solr/core/src/resources/EditableSolrConfigAttributes.json
View File

@ -70,7 +70,4 @@
"enableRemoteStreaming":10,
"enableStreamBody":10,
"addHttpRequestToContext":0}},
"peerSync":{
"useRangeVersions":11
}
}

4
solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
View File

@ -49,10 +49,6 @@
<requestHandler name="/select" class="solr.SearchHandler" />
<peerSync>
<useRangeVersions>${solr.peerSync.useRangeVersions:true}</useRangeVersions>
</peerSync>
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met -->
<autoCommit>

2
solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
View File

@ -284,7 +284,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.setProperty("enable.update.log", usually() ? "true" : "false");
System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
System.setProperty("solr.clustering.enabled", "false");
System.setProperty("solr.peerSync.useRangeVersions", String.valueOf(random().nextBoolean()));
System.setProperty("solr.cloud.wait-for-updates-with-stale-state-pause", "500");
System.setProperty("pkiHandlerPrivateKeyPath", SolrTestCaseJ4.class.getClassLoader().getResource("cryptokeys/priv_key512_pkcs8.pem").toExternalForm());
@ -342,7 +341,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.clearProperty("enable.update.log");
System.clearProperty("useCompoundFile");
System.clearProperty("urlScheme");
System.clearProperty("solr.peerSync.useRangeVersions");
System.clearProperty("solr.cloud.wait-for-updates-with-stale-state-pause");
System.clearProperty("solr.zkclienttmeout");
System.clearProperty(ZK_WHITELIST_PROPERTY);