honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow reader wrappers to have different live docs but the same cache key. 

Relates to #19856
This commit is contained in:
Adrien Grand 2017-06-15 13:40:48 +02:00
parent 0036f28a6a
commit 1b90c46a53
3 changed files with 32 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
View File

@ -52,12 +52,6 @@ final class PerThreadIDVersionAndSeqNoLookup {
// TODO: do we really need to store all this stuff? some if it might not speed up anything. // TODO: do we really need to store all this stuff? some if it might not speed up anything.
// we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff // we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff
/** The {@link LeafReaderContext} that needs to be looked up. */
private final LeafReaderContext context;
/** Live docs of the context, cached to avoid the cost of ensureOpen() on every
* segment for every index operation. */
private final Bits liveDocs;
/** terms enum for uid field */ /** terms enum for uid field */
final String uidField; final String uidField;
private final TermsEnum termsEnum; private final TermsEnum termsEnum;
@ -71,10 +65,7 @@ final class PerThreadIDVersionAndSeqNoLookup {
/** /**
* Initialize lookup for the provided segment * Initialize lookup for the provided segment
*/ */
PerThreadIDVersionAndSeqNoLookup(LeafReaderContext context, String uidField) throws IOException { PerThreadIDVersionAndSeqNoLookup(LeafReader reader, String uidField) throws IOException {
this.context = context;
final LeafReader reader = context.reader();
this.liveDocs = reader.getLiveDocs();
this.uidField = uidField; this.uidField = uidField;
Fields fields = reader.fields(); Fields fields = reader.fields();
Terms terms = fields.terms(uidField); Terms terms = fields.terms(uidField);
@ -91,12 +82,17 @@ final class PerThreadIDVersionAndSeqNoLookup {
this.readerKey = readerKey; this.readerKey = readerKey;
} }
/** Return null if id is not found. */ /** Return null if id is not found.
public DocIdAndVersion lookupVersion(BytesRef id) * We pass the {@link LeafReaderContext} as an argument so that things
* still work with reader wrappers that hide some documents while still
* using the same cache key. Otherwise we'd have to disable caching
* entirely for these readers.
*/
public DocIdAndVersion lookupVersion(BytesRef id, LeafReaderContext context)
throws IOException { throws IOException {
assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) : assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) :
"context's reader is not the same as the reader class was initialized on."; "context's reader is not the same as the reader class was initialized on.";
int docID = getDocID(id); int docID = getDocID(id, context.reader().getLiveDocs());
if (docID != DocIdSetIterator.NO_MORE_DOCS) { if (docID != DocIdSetIterator.NO_MORE_DOCS) {
final NumericDocValues versions = context.reader().getNumericDocValues(VersionFieldMapper.NAME); final NumericDocValues versions = context.reader().getNumericDocValues(VersionFieldMapper.NAME);
@ -116,7 +112,7 @@ final class PerThreadIDVersionAndSeqNoLookup {
* returns the internal lucene doc id for the given id bytes. * returns the internal lucene doc id for the given id bytes.
* {@link DocIdSetIterator#NO_MORE_DOCS} is returned if not found * {@link DocIdSetIterator#NO_MORE_DOCS} is returned if not found
* */ * */
private int getDocID(BytesRef id) throws IOException { private int getDocID(BytesRef id, Bits liveDocs) throws IOException {
if (termsEnum.seekExact(id)) { if (termsEnum.seekExact(id)) {
int docID = DocIdSetIterator.NO_MORE_DOCS; int docID = DocIdSetIterator.NO_MORE_DOCS;
// there may be more than one matching docID, in the case of nested docs, so we want the last one: // there may be more than one matching docID, in the case of nested docs, so we want the last one:
@ -134,8 +130,10 @@ final class PerThreadIDVersionAndSeqNoLookup {
} }
/** Return null if id is not found. */ /** Return null if id is not found. */
DocIdAndSeqNo lookupSeqNo(BytesRef id) throws IOException { DocIdAndSeqNo lookupSeqNo(BytesRef id, LeafReaderContext context) throws IOException {
int docID = getDocID(id); assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) :
"context's reader is not the same as the reader class was initialized on.";
int docID = getDocID(id, context.reader().getLiveDocs());
if (docID != DocIdSetIterator.NO_MORE_DOCS) { if (docID != DocIdSetIterator.NO_MORE_DOCS) {
NumericDocValues seqNos = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME); NumericDocValues seqNos = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
long seqNo; long seqNo;

12
core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
View File

@ -73,7 +73,7 @@ public final class VersionsAndSeqNoResolver {
if (lookupState == null) { if (lookupState == null) {
lookupState = new PerThreadIDVersionAndSeqNoLookup[reader.leaves().size()]; lookupState = new PerThreadIDVersionAndSeqNoLookup[reader.leaves().size()];
for (LeafReaderContext leaf : reader.leaves()) { for (LeafReaderContext leaf : reader.leaves()) {
lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf, uidField); lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), uidField);
} }
ctl.set(lookupState); ctl.set(lookupState);
} }
@ -132,8 +132,9 @@ public final class VersionsAndSeqNoResolver {
// iterate backwards to optimize for the frequently updated documents // iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments // which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) { for (int i = leaves.size() - 1; i >= 0; i--) {
PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaves.get(i).ord]; final LeafReaderContext leaf = leaves.get(i);
DocIdAndVersion result = lookup.lookupVersion(term.bytes()); PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord];
DocIdAndVersion result = lookup.lookupVersion(term.bytes(), leaf);
if (result != null) { if (result != null) {
return result; return result;
} }
@ -153,8 +154,9 @@ public final class VersionsAndSeqNoResolver {
// iterate backwards to optimize for the frequently updated documents // iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments // which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) { for (int i = leaves.size() - 1; i >= 0; i--) {
PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaves.get(i).ord]; final LeafReaderContext leaf = leaves.get(i);
DocIdAndSeqNo result = lookup.lookupSeqNo(term.bytes()); PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord];
DocIdAndSeqNo result = lookup.lookupSeqNo(term.bytes(), leaf);
if (result != null) { if (result != null) {
return result; return result;
} }

22
core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
View File

@ -56,21 +56,21 @@ public class VersionLookupTests extends ESTestCase {
writer.addDocument(new Document()); writer.addDocument(new Document());
DirectoryReader reader = DirectoryReader.open(writer); DirectoryReader reader = DirectoryReader.open(writer);
LeafReaderContext segment = reader.leaves().get(0); LeafReaderContext segment = reader.leaves().get(0);
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME); PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
// found doc // found doc
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6")); DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), segment);
assertNotNull(result); assertNotNull(result);
assertEquals(87, result.version); assertEquals(87, result.version);
assertEquals(0, result.docId); assertEquals(0, result.docId);
// not found doc // not found doc
assertNull(lookup.lookupVersion(new BytesRef("7"))); assertNull(lookup.lookupVersion(new BytesRef("7"), segment));
// deleted doc // deleted doc
writer.deleteDocuments(new Term(IdFieldMapper.NAME, "6")); writer.deleteDocuments(new Term(IdFieldMapper.NAME, "6"));
reader.close(); reader.close();
reader = DirectoryReader.open(writer); reader = DirectoryReader.open(writer);
segment = reader.leaves().get(0); segment = reader.leaves().get(0);
lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME); lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
assertNull(lookup.lookupVersion(new BytesRef("6"))); assertNull(lookup.lookupVersion(new BytesRef("6"), segment));
reader.close(); reader.close();
writer.close(); writer.close();
dir.close(); dir.close();
@ -91,9 +91,9 @@ public class VersionLookupTests extends ESTestCase {
writer.addDocument(new Document()); writer.addDocument(new Document());
DirectoryReader reader = DirectoryReader.open(writer); DirectoryReader reader = DirectoryReader.open(writer);
LeafReaderContext segment = reader.leaves().get(0); LeafReaderContext segment = reader.leaves().get(0);
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME); PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
// return the last doc when there are duplicates // return the last doc when there are duplicates
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6")); DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), segment);
assertNotNull(result); assertNotNull(result);
assertEquals(87, result.version); assertEquals(87, result.version);
assertEquals(1, result.docId); assertEquals(1, result.docId);
@ -102,8 +102,8 @@ public class VersionLookupTests extends ESTestCase {
reader.close(); reader.close();
reader = DirectoryReader.open(writer); reader = DirectoryReader.open(writer);
segment = reader.leaves().get(0); segment = reader.leaves().get(0);
lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME); lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
result = lookup.lookupVersion(new BytesRef("6")); result = lookup.lookupVersion(new BytesRef("6"), segment);
assertNotNull(result); assertNotNull(result);
assertEquals(87, result.version); assertEquals(87, result.version);
assertEquals(1, result.docId); assertEquals(1, result.docId);
@ -112,8 +112,8 @@ public class VersionLookupTests extends ESTestCase {
reader.close(); reader.close();
reader = DirectoryReader.open(writer); reader = DirectoryReader.open(writer);
segment = reader.leaves().get(0); segment = reader.leaves().get(0);
lookup = new PerThreadIDVersionAndSeqNoLookup(segment, IdFieldMapper.NAME); lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
assertNull(lookup.lookupVersion(new BytesRef("6"))); assertNull(lookup.lookupVersion(new BytesRef("6"), segment));
reader.close(); reader.close();
writer.close(); writer.close();
dir.close(); dir.close();