honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix from_range in search_after in changes snapshot (#33335) 

We can have multiple documents in Lucene with the same seq_no for
parent-child documents (or without rollback). In this case, the usage
"lastSeenSeqNo + 1" is an off-by-one error as it may miss some
documents. This error merely affects the `skippedOperations` contract.

See: https://github.com/elastic/elasticsearch/pull/33222#discussion_r213842257

Closes #33318
This commit is contained in:
Nhat Nguyen 2018-09-03 11:58:49 -04:00 committed by GitHub
parent 42424aff21
commit 24d60c7f4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java
View File

@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.join.ParentJoinPlugin;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase;
@ -59,8 +58,6 @@ public abstract class ParentChildTestCase extends ESIntegTestCase {
@Override @Override
public Settings indexSettings() { public Settings indexSettings() {
Settings.Builder builder = Settings.builder().put(super.indexSettings()) Settings.Builder builder = Settings.builder().put(super.indexSettings())
// AwaitsFix: https://github.com/elastic/elasticsearch/issues/33318
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
// aggressive filter caching so that we can assert on the filter cache size // aggressive filter caching so that we can assert on the filter cache size
.put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true)
.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true); .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true);

2
server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
View File

@ -212,7 +212,7 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
} }
private TopDocs searchOperations(ScoreDoc after) throws IOException { private TopDocs searchOperations(ScoreDoc after) throws IOException {
final Query rangeQuery = LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, lastSeenSeqNo + 1, toSeqNo); final Query rangeQuery = LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo);
final Sort sortedBySeqNoThenByTerm = new Sort( final Sort sortedBySeqNoThenByTerm = new Sort(
new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG), new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG),
new SortField(SeqNoFieldMapper.PRIMARY_TERM_NAME, SortField.Type.LONG, true) new SortField(SeqNoFieldMapper.PRIMARY_TERM_NAME, SortField.Type.LONG, true)

7
server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
View File

@ -182,12 +182,17 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
} }
} }
long maxSeqNo = engine.getLocalCheckpointTracker().getMaxSeqNo(); long maxSeqNo = engine.getLocalCheckpointTracker().getMaxSeqNo();
try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapperService, 0, maxSeqNo, false)) { engine.refresh("test");
Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
try (Translog.Snapshot snapshot = new LuceneChangesSnapshot(searcher, mapperService, between(1, 100), 0, maxSeqNo, false)) {
searcher = null;
Translog.Operation op; Translog.Operation op;
while ((op = snapshot.next()) != null) { while ((op = snapshot.next()) != null) {
assertThat(op.toString(), op.primaryTerm(), equalTo(latestOperations.get(op.seqNo()))); assertThat(op.toString(), op.primaryTerm(), equalTo(latestOperations.get(op.seqNo())));
} }
assertThat(snapshot.skippedOperations(), equalTo(totalOps - latestOperations.size())); assertThat(snapshot.skippedOperations(), equalTo(totalOps - latestOperations.size()));
} finally {
IOUtils.close(searcher);
} }
} }