Security for _field_names field should not override field statistics (#33261)
In Lucene 8 the statistics for a field (doc_count, sum_doc_count, ...) are checked and invalid values (v < 0) are rejected. Though for the _field_names field we hide the statistics of the field if security is enabled since some terms (field names) may be filtered. However this statistics are never used, this field is not used for ranking and cannot be used to generate term vectors. For these reasons this commit restores the original statistics for the field in order to be compliant with Lucene 8.
This commit is contained in:
Jim Ferenczi
GitHub
parent
713c07e14d
commit
f0a61b6dec
|
|
@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
|
|||
import org.elasticsearch.index.mapper.SourceFieldMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -70,7 +71,11 @@ public final class FieldSubsetReader extends FilterLeafReader {
|
|||
super(in, new FilterDirectoryReader.SubReaderWrapper() {
|
||||
@Override
|
||||
public LeafReader wrap(LeafReader reader) {
|
||||
return new FieldSubsetReader(reader, filter);
|
||||
try {
|
||||
return new FieldSubsetReader(reader, filter);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
this.filter = filter;
|
||||
|
|
@ -109,11 +114,13 @@ public final class FieldSubsetReader extends FilterLeafReader {
|
|||
private final FieldInfos fieldInfos;
|
||||
/** An automaton that only accepts authorized fields. */
|
||||
private final CharacterRunAutomaton filter;
|
||||
/** {@link Terms} cache with filtered stats for the {@link FieldNamesFieldMapper} field. */
|
||||
private final Terms fieldNamesFilterTerms;
|
||||
|
||||
/**
|
||||
* Wrap a single segment, exposing a subset of its fields.
|
||||
*/
|
||||
FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter) {
|
||||
FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter) throws IOException {
|
||||
super(in);
|
||||
ArrayList<FieldInfo> filteredInfos = new ArrayList<>();
|
||||
for (FieldInfo fi : in.getFieldInfos()) {
|
||||
|
|
@ -123,6 +130,8 @@ public final class FieldSubsetReader extends FilterLeafReader {
|
|||
}
|
||||
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
|
||||
this.filter = filter;
|
||||
final Terms fieldNameTerms = super.terms(FieldNamesFieldMapper.NAME);
|
||||
this.fieldNamesFilterTerms = fieldNameTerms == null ? null : new FieldNamesTerms(fieldNameTerms);
|
||||
}
|
||||
|
||||
/** returns true if this field is allowed. */
|
||||
|
|
@ -346,21 +355,14 @@ public final class FieldSubsetReader extends FilterLeafReader {
|
|||
}
|
||||
}
|
||||
|
||||
private Terms wrapTerms(Terms terms, String field) {
|
||||
private Terms wrapTerms(Terms terms, String field) throws IOException {
|
||||
if (!hasField(field)) {
|
||||
return null;
|
||||
} else if (FieldNamesFieldMapper.NAME.equals(field)) {
|
||||
// for the _field_names field, fields for the document
|
||||
// are encoded as postings, where term is the field.
|
||||
// so we hide terms for fields we filter out.
|
||||
if (terms != null) {
|
||||
// check for null, in case term dictionary is not a ghostbuster
|
||||
// So just because its in fieldinfos and "indexed=true" doesn't mean you can go grab a Terms for it.
|
||||
// It just means at one point there was a document with that field indexed...
|
||||
// The fields infos isn't updates/removed even if no docs refer to it
|
||||
terms = new FieldNamesTerms(terms);
|
||||
}
|
||||
return terms;
|
||||
return fieldNamesFilterTerms;
|
||||
} else {
|
||||
return terms;
|
||||
}
|
||||
|
|
@ -371,9 +373,22 @@ public final class FieldSubsetReader extends FilterLeafReader {
|
|||
* representing fields that should not be visible in this reader.
|
||||
*/
|
||||
class FieldNamesTerms extends FilterTerms {
|
||||
final long size;
|
||||
final long sumDocFreq;
|
||||
|
||||
FieldNamesTerms(Terms in) {
|
||||
FieldNamesTerms(Terms in) throws IOException {
|
||||
super(in);
|
||||
assert in.hasFreqs() == false;
|
||||
// re-compute the stats for the field to take
|
||||
// into account the filtered terms.
|
||||
final TermsEnum e = iterator();
|
||||
long size = 0, sumDocFreq = 0;
|
||||
while (e.next() != null) {
|
||||
size ++;
|
||||
sumDocFreq += e.docFreq();
|
||||
}
|
||||
this.size = size;
|
||||
this.sumDocFreq = sumDocFreq;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -381,27 +396,20 @@ public final class FieldSubsetReader extends FilterLeafReader {
|
|||
return new FieldNamesTermsEnum(in.iterator());
|
||||
}
|
||||
|
||||
// we don't support field statistics (since we filter out terms)
|
||||
// but this isn't really a big deal: _field_names is not used for ranking.
|
||||
|
||||
@Override
|
||||
public int getDocCount() throws IOException {
|
||||
return -1;
|
||||
public long size() throws IOException {
|
||||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSumDocFreq() throws IOException {
|
||||
return -1;
|
||||
return sumDocFreq;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSumTotalTermFreq() throws IOException {
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size() throws IOException {
|
||||
return -1;
|
||||
public int getDocCount() throws IOException {
|
||||
// it is costly to recompute this value so we assume that docCount == maxDoc.
|
||||
return maxDoc();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ import static org.hamcrest.Matchers.equalTo;
|
|||
|
||||
/** Simple tests for this filterreader */
|
||||
public class FieldSubsetReaderTests extends ESTestCase {
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two string fields
|
||||
*/
|
||||
|
|
@ -86,16 +86,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "test", Field.Store.NO));
|
||||
doc.add(new StringField("fieldB", "test", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
Set<String> seenFields = new HashSet<>();
|
||||
|
|
@ -105,11 +105,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
assertEquals(Collections.singleton("fieldA"), seenFields);
|
||||
assertNotNull(segmentReader.terms("fieldA"));
|
||||
assertNull(segmentReader.terms("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two int points
|
||||
*/
|
||||
|
|
@ -181,25 +181,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("fieldA", "testA"));
|
||||
doc.add(new StoredField("fieldB", "testB"));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals("testA", d2.get("fieldA"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two stored fields (binary)
|
||||
*/
|
||||
|
|
@ -207,25 +207,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("fieldA", new BytesRef("testA")));
|
||||
doc.add(new StoredField("fieldB", new BytesRef("testB")));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two stored fields (int)
|
||||
*/
|
||||
|
|
@ -233,25 +233,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("fieldA", 1));
|
||||
doc.add(new StoredField("fieldB", 2));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals(1, d2.getField("fieldA").numericValue());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two stored fields (long)
|
||||
*/
|
||||
|
|
@ -259,25 +259,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("fieldA", 1L));
|
||||
doc.add(new StoredField("fieldB", 2L));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals(1L, d2.getField("fieldA").numericValue());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two stored fields (float)
|
||||
*/
|
||||
|
|
@ -285,25 +285,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("fieldA", 1F));
|
||||
doc.add(new StoredField("fieldB", 2F));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals(1F, d2.getField("fieldA").numericValue());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two stored fields (double)
|
||||
*/
|
||||
|
|
@ -311,25 +311,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("fieldA", 1D));
|
||||
doc.add(new StoredField("fieldB", 2D));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals(1D, d2.getField("fieldA").numericValue());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two vector fields
|
||||
*/
|
||||
|
|
@ -337,7 +337,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);
|
||||
|
|
@ -345,10 +345,10 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
doc.add(new Field("fieldA", "testA", ft));
|
||||
doc.add(new Field("fieldB", "testB", ft));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
Fields vectors = ir.getTermVectors(0);
|
||||
Set<String> seenFields = new HashSet<>();
|
||||
|
|
@ -356,11 +356,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
seenFields.add(field);
|
||||
}
|
||||
assertEquals(Collections.singleton("fieldA"), seenFields);
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two text fields
|
||||
*/
|
||||
|
|
@ -368,25 +368,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new TextField("fieldA", "test", Field.Store.NO));
|
||||
doc.add(new TextField("fieldB", "test", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
assertNotNull(segmentReader.getNormValues("fieldA"));
|
||||
assertNull(segmentReader.getNormValues("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two numeric dv fields
|
||||
*/
|
||||
|
|
@ -394,16 +394,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new NumericDocValuesField("fieldA", 1));
|
||||
doc.add(new NumericDocValuesField("fieldB", 2));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
NumericDocValues values = segmentReader.getNumericDocValues("fieldA");
|
||||
|
|
@ -411,11 +411,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
assertTrue(values.advanceExact(0));
|
||||
assertEquals(1, values.longValue());
|
||||
assertNull(segmentReader.getNumericDocValues("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two binary dv fields
|
||||
*/
|
||||
|
|
@ -423,16 +423,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new BinaryDocValuesField("fieldA", new BytesRef("testA")));
|
||||
doc.add(new BinaryDocValuesField("fieldB", new BytesRef("testB")));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
BinaryDocValues values = segmentReader.getBinaryDocValues("fieldA");
|
||||
|
|
@ -444,7 +444,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two sorted dv fields
|
||||
*/
|
||||
|
|
@ -452,16 +452,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedDocValuesField("fieldA", new BytesRef("testA")));
|
||||
doc.add(new SortedDocValuesField("fieldB", new BytesRef("testB")));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
SortedDocValues values = segmentReader.getSortedDocValues("fieldA");
|
||||
|
|
@ -469,11 +469,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
assertTrue(values.advanceExact(0));
|
||||
assertEquals(new BytesRef("testA"), values.binaryValue());
|
||||
assertNull(segmentReader.getSortedDocValues("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two sortedset dv fields
|
||||
*/
|
||||
|
|
@ -481,16 +481,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesField("fieldA", new BytesRef("testA")));
|
||||
doc.add(new SortedSetDocValuesField("fieldB", new BytesRef("testB")));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
SortedSetDocValues dv = segmentReader.getSortedSetDocValues("fieldA");
|
||||
|
|
@ -500,11 +500,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
|
||||
assertEquals(new BytesRef("testA"), dv.lookupOrd(0));
|
||||
assertNull(segmentReader.getSortedSetDocValues("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering two sortednumeric dv fields
|
||||
*/
|
||||
|
|
@ -512,16 +512,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedNumericDocValuesField("fieldA", 1));
|
||||
doc.add(new SortedNumericDocValuesField("fieldB", 2));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
SortedNumericDocValues dv = segmentReader.getSortedNumericDocValues("fieldA");
|
||||
|
|
@ -530,11 +530,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
assertEquals(1, dv.docValueCount());
|
||||
assertEquals(1, dv.nextValue());
|
||||
assertNull(segmentReader.getSortedNumericDocValues("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test we have correct fieldinfos metadata
|
||||
*/
|
||||
|
|
@ -542,27 +542,27 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "test", Field.Store.NO));
|
||||
doc.add(new StringField("fieldB", "test", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
FieldInfos infos = segmentReader.getFieldInfos();
|
||||
assertEquals(1, infos.size());
|
||||
assertNotNull(infos.fieldInfo("fieldA"));
|
||||
assertNull(infos.fieldInfo("fieldB"));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test special handling for _source field.
|
||||
*/
|
||||
|
|
@ -570,7 +570,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "testA", Field.Store.NO));
|
||||
|
|
@ -578,16 +578,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
byte bytes[] = "{\"fieldA\":\"testA\", \"fieldB\":\"testB\"}".getBytes(StandardCharsets.UTF_8);
|
||||
doc.add(new StoredField(SourceFieldMapper.NAME, bytes, 0, bytes.length));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", SourceFieldMapper.NAME));
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
|
||||
|
||||
|
||||
// see only one field
|
||||
Document d2 = ir.document(0);
|
||||
assertEquals(1, d2.getFields().size());
|
||||
assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
|
@ -741,7 +741,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "test", Field.Store.NO));
|
||||
|
|
@ -749,37 +749,37 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
doc.add(new StringField(FieldNamesFieldMapper.NAME, "fieldA", Field.Store.NO));
|
||||
doc.add(new StringField(FieldNamesFieldMapper.NAME, "fieldB", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
Set<String> fields = new HashSet<>();
|
||||
fields.add("fieldA");
|
||||
Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", FieldNamesFieldMapper.NAME));
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
Terms terms = segmentReader.terms(FieldNamesFieldMapper.NAME);
|
||||
TermsEnum termsEnum = terms.iterator();
|
||||
assertEquals(new BytesRef("fieldA"), termsEnum.next());
|
||||
assertNull(termsEnum.next());
|
||||
|
||||
// seekExact
|
||||
|
||||
// seekExact
|
||||
termsEnum = terms.iterator();
|
||||
assertTrue(termsEnum.seekExact(new BytesRef("fieldA")));
|
||||
assertFalse(termsEnum.seekExact(new BytesRef("fieldB")));
|
||||
|
||||
// seekCeil
|
||||
|
||||
// seekCeil
|
||||
termsEnum = terms.iterator();
|
||||
assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("fieldA")));
|
||||
assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(new BytesRef("field0000")));
|
||||
assertEquals(new BytesRef("fieldA"), termsEnum.term());
|
||||
assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("fieldAAA")));
|
||||
assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("fieldB")));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test special handling for _field_names field (three fields, to exercise termsenum better)
|
||||
*/
|
||||
|
|
@ -787,7 +787,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "test", Field.Store.NO));
|
||||
|
|
@ -797,11 +797,11 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
doc.add(new StringField(FieldNamesFieldMapper.NAME, "fieldB", Field.Store.NO));
|
||||
doc.add(new StringField(FieldNamesFieldMapper.NAME, "fieldC", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", "fieldC", FieldNamesFieldMapper.NAME));
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
|
||||
|
||||
|
||||
// see only two fields
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
Terms terms = segmentReader.terms(FieldNamesFieldMapper.NAME);
|
||||
|
|
@ -809,24 +809,24 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
assertEquals(new BytesRef("fieldA"), termsEnum.next());
|
||||
assertEquals(new BytesRef("fieldC"), termsEnum.next());
|
||||
assertNull(termsEnum.next());
|
||||
|
||||
// seekExact
|
||||
|
||||
// seekExact
|
||||
termsEnum = terms.iterator();
|
||||
assertTrue(termsEnum.seekExact(new BytesRef("fieldA")));
|
||||
assertFalse(termsEnum.seekExact(new BytesRef("fieldB")));
|
||||
assertTrue(termsEnum.seekExact(new BytesRef("fieldC")));
|
||||
|
||||
// seekCeil
|
||||
|
||||
// seekCeil
|
||||
termsEnum = terms.iterator();
|
||||
assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("fieldA")));
|
||||
assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(new BytesRef("fieldB")));
|
||||
assertEquals(new BytesRef("fieldC"), termsEnum.term());
|
||||
assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("fieldD")));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test _field_names where a field is permitted, but doesn't exist in the segment.
|
||||
*/
|
||||
|
|
@ -834,7 +834,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "test", Field.Store.NO));
|
||||
|
|
@ -842,27 +842,27 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
doc.add(new StringField(FieldNamesFieldMapper.NAME, "fieldA", Field.Store.NO));
|
||||
doc.add(new StringField(FieldNamesFieldMapper.NAME, "fieldB", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", "fieldC", FieldNamesFieldMapper.NAME));
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
Terms terms = segmentReader.terms(FieldNamesFieldMapper.NAME);
|
||||
|
||||
// seekExact
|
||||
|
||||
// seekExact
|
||||
TermsEnum termsEnum = terms.iterator();
|
||||
assertFalse(termsEnum.seekExact(new BytesRef("fieldC")));
|
||||
|
||||
// seekCeil
|
||||
|
||||
// seekCeil
|
||||
termsEnum = terms.iterator();
|
||||
assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("fieldC")));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test where _field_names does not exist
|
||||
*/
|
||||
|
|
@ -870,25 +870,25 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("fieldA", "test", Field.Store.NO));
|
||||
doc.add(new StringField("fieldB", "test", Field.Store.NO));
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", SourceFieldMapper.NAME));
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
|
||||
|
||||
|
||||
// see only one field
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
assertNull(segmentReader.terms(FieldNamesFieldMapper.NAME));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/** test that core cache key (needed for NRT) is working */
|
||||
public void testCoreCacheKey() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
|
|
@ -896,7 +896,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
iwc.setMaxBufferedDocs(100);
|
||||
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add two docs, id:0 and id:1
|
||||
Document doc = new Document();
|
||||
Field idField = new StringField("id", "", Field.Store.NO);
|
||||
|
|
@ -905,7 +905,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
iw.addDocument(doc);
|
||||
idField.setStringValue("1");
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("id")));
|
||||
assertEquals(2, ir.numDocs());
|
||||
|
|
@ -914,17 +914,17 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
// delete id:0 and reopen
|
||||
iw.deleteDocuments(new Term("id", "0"));
|
||||
DirectoryReader ir2 = DirectoryReader.openIfChanged(ir);
|
||||
|
||||
|
||||
// we should have the same cache key as before
|
||||
assertEquals(1, ir2.numDocs());
|
||||
assertEquals(1, ir2.leaves().size());
|
||||
assertSame(ir.leaves().get(0).reader().getCoreCacheHelper().getKey(),
|
||||
ir2.leaves().get(0).reader().getCoreCacheHelper().getKey());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, ir2, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering the only vector fields
|
||||
*/
|
||||
|
|
@ -932,7 +932,7 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
|
||||
|
||||
// add document with 2 fields
|
||||
Document doc = new Document();
|
||||
FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);
|
||||
|
|
@ -940,17 +940,17 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
doc.add(new Field("fieldA", "testA", ft));
|
||||
doc.add(new StringField("fieldB", "testB", Field.Store.NO)); // no vectors
|
||||
iw.addDocument(doc);
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldB")));
|
||||
|
||||
|
||||
// sees no fields
|
||||
assertNull(ir.getTermVectors(0));
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* test filtering an index with no fields
|
||||
*/
|
||||
|
|
@ -959,10 +959,10 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
IndexWriterConfig iwc = new IndexWriterConfig(null);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
iw.addDocument(new Document());
|
||||
|
||||
|
||||
// open reader
|
||||
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
|
||||
|
||||
|
||||
// see no fields
|
||||
LeafReader segmentReader = ir.leaves().get(0).reader();
|
||||
Set<String> seenFields = new HashSet<>();
|
||||
|
|
@ -971,14 +971,14 @@ public class FieldSubsetReaderTests extends ESTestCase {
|
|||
}
|
||||
assertEquals(0, seenFields.size());
|
||||
assertNull(segmentReader.terms("foo"));
|
||||
|
||||
|
||||
// see no vectors
|
||||
assertNull(segmentReader.getTermVectors(0));
|
||||
|
||||
|
||||
// see no stored fields
|
||||
Document document = segmentReader.document(0);
|
||||
assertEquals(0, document.getFields().size());
|
||||
|
||||
|
||||
TestUtil.checkReader(ir);
|
||||
IOUtils.close(ir, iw, dir);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue