honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow reader wrappers to have different live docs but the same cache key. 

Relates to #19856
This commit is contained in:
Adrien Grand 2017-06-15 13:40:48 +02:00
parent 0036f28a6a
commit 1b90c46a53
3 changed files with 32 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
View File

@ -52,12 +52,6 @@ final class PerThreadIDVersionAndSeqNoLookup {
// TODO: do we really need to store all this stuff? some if it might not speed up anything.
// we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff
/** The {@link LeafReaderContext} that needs to be looked up. */
private final LeafReaderContext context;
/** Live docs of the context, cached to avoid the cost of ensureOpen() on every
* segment for every index operation. */
private final Bits liveDocs;
/** terms enum for uid field */
final String uidField;
private final TermsEnum termsEnum;
@ -71,10 +65,7 @@ final class PerThreadIDVersionAndSeqNoLookup {
/**
* Initialize lookup for the provided segment
*/
PerThreadIDVersionAndSeqNoLookup(LeafReaderContext context, String uidField) throws IOException {
this.context = context;
final LeafReader reader = context.reader();
this.liveDocs = reader.getLiveDocs();
PerThreadIDVersionAndSeqNoLookup(LeafReader reader, String uidField) throws IOException {
this.uidField = uidField;
Fields fields = reader.fields();
Terms terms = fields.terms(uidField);
@ -91,12 +82,17 @@ final class PerThreadIDVersionAndSeqNoLookup {
this.readerKey = readerKey;
}
/** Return null if id is not found. */
public DocIdAndVersion lookupVersion(BytesRef id)
/** Return null if id is not found.
* We pass the {@link LeafReaderContext} as an argument so that things
* still work with reader wrappers that hide some documents while still
* using the same cache key. Otherwise we'd have to disable caching
* entirely for these readers.
*/
public DocIdAndVersion lookupVersion(BytesRef id, LeafReaderContext context)
throws IOException {
assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) :
"context's reader is not the same as the reader class was initialized on.";
int docID = getDocID(id);
int docID = getDocID(id, context.reader().getLiveDocs());
if (docID != DocIdSetIterator.NO_MORE_DOCS) {
final NumericDocValues versions = context.reader().getNumericDocValues(VersionFieldMapper.NAME);
@ -116,7 +112,7 @@ final class PerThreadIDVersionAndSeqNoLookup {
* returns the internal lucene doc id for the given id bytes.
* {@link DocIdSetIterator#NO_MORE_DOCS} is returned if not found
* */
private int getDocID(BytesRef id) throws IOException {
private int getDocID(BytesRef id, Bits liveDocs) throws IOException {
if (termsEnum.seekExact(id)) {
int docID = DocIdSetIterator.NO_MORE_DOCS;
// there may be more than one matching docID, in the case of nested docs, so we want the last one:
@ -134,8 +130,10 @@ final class PerThreadIDVersionAndSeqNoLookup {
}
/** Return null if id is not found. */
DocIdAndSeqNo lookupSeqNo(BytesRef id) throws IOException {
int docID = getDocID(id);
DocIdAndSeqNo lookupSeqNo(BytesRef id, LeafReaderContext context) throws IOException {
assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) :
"context's reader is not the same as the reader class was initialized on.";
int docID = getDocID(id, context.reader().getLiveDocs());
if (docID != DocIdSetIterator.NO_MORE_DOCS) {
NumericDocValues seqNos = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
long seqNo;

12
core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
View File

@ -73,7 +73,7 @@ public final class VersionsAndSeqNoResolver {
if (lookupState == null) {
lookupState = new PerThreadIDVersionAndSeqNoLookup[reader.leaves().size()];
for (LeafReaderContext leaf : reader.leaves()) {
lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf, uidField);
lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), uidField);
}
ctl.set(lookupState);
}
@ -132,8 +132,9 @@ public final class VersionsAndSeqNoResolver {
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaves.get(i).ord];
DocIdAndVersion result = lookup.lookupVersion(term.bytes());
final LeafReaderContext leaf = leaves.get(i);
PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord];
DocIdAndVersion result = lookup.lookupVersion(term.bytes(), leaf);
if (result != null) {
return result;
}
@ -153,8 +154,9 @@ public final class VersionsAndSeqNoResolver {
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaves.get(i).ord];
DocIdAndSeqNo result = lookup.lookupSeqNo(term.bytes());
final LeafReaderContext leaf = leaves.get(i);
PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord];
DocIdAndSeqNo result = lookup.lookupSeqNo(term.bytes(), leaf);
if (result != null) {
return result;
}

22
core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
View File

@ -56,21 +56,21 @@ public class VersionLookupTests extends ESTestCase {
writer.addDocument(new Document());
DirectoryReader reader = DirectoryReader.open(writer);
LeafReaderContext segment = reader.leaves().get(0);
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME);
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
// found doc
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"));
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), segment);
assertNotNull(result);
assertEquals(87, result.version);
assertEquals(0, result.docId);
// not found doc
assertNull(lookup.lookupVersion(new BytesRef("7")));
assertNull(lookup.lookupVersion(new BytesRef("7"), segment));
// deleted doc
writer.deleteDocuments(new Term(IdFieldMapper.NAME, "6"));
reader.close();
reader = DirectoryReader.open(writer);
segment = reader.leaves().get(0);
lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME);
assertNull(lookup.lookupVersion(new BytesRef("6")));
lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
assertNull(lookup.lookupVersion(new BytesRef("6"), segment));
reader.close();
writer.close();
dir.close();
@ -91,9 +91,9 @@ public class VersionLookupTests extends ESTestCase {
writer.addDocument(new Document());
DirectoryReader reader = DirectoryReader.open(writer);
LeafReaderContext segment = reader.leaves().get(0);
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME);
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
// return the last doc when there are duplicates
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"));
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), segment);
assertNotNull(result);
assertEquals(87, result.version);
assertEquals(1, result.docId);
@ -102,8 +102,8 @@ public class VersionLookupTests extends ESTestCase {
reader.close();
reader = DirectoryReader.open(writer);
segment = reader.leaves().get(0);
lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME);
result = lookup.lookupVersion(new BytesRef("6"));
lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
result = lookup.lookupVersion(new BytesRef("6"), segment);
assertNotNull(result);
assertEquals(87, result.version);
assertEquals(1, result.docId);
@ -112,8 +112,8 @@ public class VersionLookupTests extends ESTestCase {
reader.close();
reader = DirectoryReader.open(writer);
segment = reader.leaves().get(0);
lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME);
assertNull(lookup.lookupVersion(new BytesRef("6")));
lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
assertNull(lookup.lookupVersion(new BytesRef("6"), segment));
reader.close();
writer.close();
dir.close();