Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5706: remove the ability to unset a DocValues field through DV updates 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1597600 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shai Erera 2014-05-26 14:54:10 +00:00
parent a912f7f8f1
commit d8ce025471
9 changed files with 63 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -165,6 +165,9 @@ API Changes
* LUCENE-5701: Core closed listeners are now available in the AtomicReader API,
they used to sit only in SegmentReader. (Adrien Grand, Robert Muir)
* LUCENE-5706: Removed the option to unset a DocValues field through DocValues
updates. (Shai Erera)
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes

62
lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
View File

@ -1,11 +1,9 @@
package org.apache.lucene.index;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
@ -41,31 +39,25 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
private final int size;
private final PagedGrowableWriter lengths;
private final PagedMutable docs;
private final FixedBitSet docsWithField;
private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
private int doc = -1;
private final BytesRef value;
private int offset, length;
Iterator(int size, PagedGrowableWriter offsets, PagedGrowableWriter lengths,
PagedMutable docs, BytesRef values, FixedBitSet docsWithField) {
PagedMutable docs, BytesRef values) {
this.offsets = offsets;
this.size = size;
this.lengths = lengths;
this.docs = docs;
this.docsWithField = docsWithField;
value = values.clone();
}
@Override
BytesRef value() {
if (offset == -1) {
return null;
} else {
value.offset = offset;
value.length = length;
return value;
}
value.offset = offset;
value.length = length;
return value;
}
@Override
@ -81,15 +73,11 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
// idx points to the "next" element
long prevIdx = idx - 1;
if (!docsWithField.get((int) prevIdx)) {
offset = -1;
} else {
// cannot change 'value' here because nextDoc is called before the
// value is used, and it's a waste to clone the BytesRef when we
// obtain the value
offset = (int) offsets.get(prevIdx);
length = (int) lengths.get(prevIdx);
}
// cannot change 'value' here because nextDoc is called before the
// value is used, and it's a waste to clone the BytesRef when we
// obtain the value
offset = (int) offsets.get(prevIdx);
length = (int) lengths.get(prevIdx);
return doc;
}
@ -106,7 +94,6 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
}
private FixedBitSet docsWithField;
private PagedMutable docs;
private PagedGrowableWriter offsets, lengths;
private BytesRef values;
@ -115,7 +102,6 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
public BinaryDocValuesFieldUpdates(String field, int maxDoc) {
super(field, Type.BINARY);
docsWithField = new FixedBitSet(64);
bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
offsets = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
@ -132,21 +118,12 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
BytesRef val = (BytesRef) value;
if (val == null) {
val = BinaryDocValuesUpdate.MISSING;
}
// grow the structures to have room for more elements
if (docs.size() == size) {
docs = docs.grow(size + 1);
offsets = offsets.grow(size + 1);
lengths = lengths.grow(size + 1);
docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
}
if (val != BinaryDocValuesUpdate.MISSING) {
// only mark the document as having a value in that field if the value wasn't set to null (MISSING)
docsWithField.set(size);
}
docs.set(size, doc);
@ -162,7 +139,6 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
final PagedGrowableWriter offsets = this.offsets;
final PagedGrowableWriter lengths = this.lengths;
final BytesRef values = this.values;
final FixedBitSet docsWithField = this.docsWithField;
new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
@ -177,18 +153,6 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
long tmpLength = lengths.get(j);
lengths.set(j, lengths.get(i));
lengths.set(i, tmpLength);
boolean tmpBool = docsWithField.get(j);
if (docsWithField.get(i)) {
docsWithField.set(j);
} else {
docsWithField.clear(j);
}
if (tmpBool) {
docsWithField.set(i);
} else {
docsWithField.clear(i);
}
}
@Override
@ -199,7 +163,7 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
}.sort(0, size);
return new Iterator(size, offsets, lengths, docs, values, docsWithField);
return new Iterator(size, offsets, lengths, docs, values);
}
@Override
@ -214,12 +178,8 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
docs = docs.grow(newSize);
offsets = offsets.grow(newSize);
lengths = lengths.grow(newSize);
docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
for (int i = 0; i < otherUpdates.size; i++) {
int doc = (int) otherUpdates.docs.get(i);
if (otherUpdates.docsWithField.get(i)) {
docsWithField.set(size);
}
docs.set(size, doc);
offsets.set(size, values.length + otherUpdates.offsets.get(i)); // correct relative offset
lengths.set(size, otherUpdates.lengths.get(i));
@ -240,7 +200,7 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
@Override
public long ramBytesPerDoc() {
long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue + 1 /* docsWithField */) / 8); // docs
long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue) / 8); // docs
final int capacity = estimateCapacity(size);
bytesPerDoc += (long) Math.ceil((double) offsets.ramBytesUsed() / capacity); // offsets
bytesPerDoc += (long) Math.ceil((double) lengths.ramBytesUsed() / capacity); // lengths

8
lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
View File

@ -81,10 +81,8 @@ abstract class DocValuesUpdate {
/* Size of BytesRef: 2*INT + ARRAY_HEADER + PTR */
private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*NUM_BYTES_INT + NUM_BYTES_OBJECT_REF;
static final BytesRef MISSING = new BytesRef();
BinaryDocValuesUpdate(Term term, String field, BytesRef value) {
super(DocValuesFieldUpdates.Type.BINARY, term, field, value == null ? MISSING : value);
super(DocValuesFieldUpdates.Type.BINARY, term, field, value);
}
@Override
@ -97,10 +95,8 @@ abstract class DocValuesUpdate {
/** An in-place update to a numeric DocValues field */
static final class NumericDocValuesUpdate extends DocValuesUpdate {
static final Long MISSING = new Long(0);
NumericDocValuesUpdate(Term term, String field, Long value) {
super(DocValuesFieldUpdates.Type.NUMERIC, term, field, value == null ? MISSING : value);
super(DocValuesFieldUpdates.Type.NUMERIC, term, field, value);
}
@Override

4
lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
View File

@ -156,9 +156,9 @@ final class DocumentsWriter implements Closeable {
return applyAllDeletes( deleteQueue);
}
synchronized boolean updateNumericDocValue(Term term, String field, Long value) throws IOException {
synchronized boolean updateNumericDocValue(Term term, String field, long value) throws IOException {
final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.addNumericUpdate(new NumericDocValuesUpdate(term, field, value));
deleteQueue.addNumericUpdate(new NumericDocValuesUpdate(term, field, Long.valueOf(value)));
flushControl.doOnDelete();
return applyAllDeletes(deleteQueue);
}

17
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -1404,10 +1404,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{
/**
* Updates a document's {@link NumericDocValues} for <code>field</code> to the
* given <code>value</code>. This method can be used to 'unset' a document's
* value by passing {@code null} as the new value. Also, you can only update
* fields that already exist in the index, not add new fields through this
* method.
* given <code>value</code>. You can only update fields that already exist in
* the index, not add new fields through this method.
*
* <p>
* <b>NOTE</b>: if this method hits an OutOfMemoryError you should immediately
@ -1425,7 +1423,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{
* @throws IOException
* if there is a low-level IO error
*/
public void updateNumericDocValue(Term term, String field, Long value) throws IOException {
public void updateNumericDocValue(Term term, String field, long value) throws IOException {
ensureOpen();
if (!globalFieldNumberMap.contains(field, DocValuesType.NUMERIC)) {
throw new IllegalArgumentException("can only update existing numeric-docvalues fields!");
@ -1441,10 +1439,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{
/**
* Updates a document's {@link BinaryDocValues} for <code>field</code> to the
* given <code>value</code>. This method can be used to 'unset' a document's
* value by passing {@code null} as the new value. Also, you can only update
* fields that already exist in the index, not add new fields through this
* method.
* given <code>value</code>. You can only update fields that already exist in
* the index, not add new fields through this method.
*
* <p>
* <b>NOTE:</b> this method currently replaces the existing value of all
@ -1468,6 +1464,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{
*/
public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
ensureOpen();
if (value == null) {
throw new IllegalArgumentException("cannot update a field to a null value: " + field);
}
if (!globalFieldNumberMap.contains(field, DocValuesType.BINARY)) {
throw new IllegalArgumentException("can only update existing binary-docvalues fields!");
}

46
lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
View File

@ -1,9 +1,7 @@
package org.apache.lucene.index;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
@ -37,16 +35,14 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
final static class Iterator extends DocValuesFieldUpdates.Iterator {
private final int size;
private final PagedGrowableWriter values;
private final FixedBitSet docsWithField;
private final PagedMutable docs;
private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
private int doc = -1;
private Long value = null;
Iterator(int size, PagedGrowableWriter values, FixedBitSet docsWithField, PagedMutable docs) {
Iterator(int size, PagedGrowableWriter values, PagedMutable docs) {
this.size = size;
this.values = values;
this.docsWithField = docsWithField;
this.docs = docs;
}
@ -66,12 +62,8 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
while (idx < size && docs.get(idx) == doc) {
++idx;
}
if (!docsWithField.get((int) (idx - 1))) {
value = null;
} else {
// idx points to the "next" element
value = Long.valueOf(values.get(idx - 1));
}
// idx points to the "next" element
value = Long.valueOf(values.get(idx - 1));
return doc;
}
@ -89,14 +81,12 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
private final int bitsPerValue;
private FixedBitSet docsWithField;
private PagedMutable docs;
private PagedGrowableWriter values;
private int size;
public NumericDocValuesFieldUpdates(String field, int maxDoc) {
super(field, Type.NUMERIC);
docsWithField = new FixedBitSet(64);
bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
@ -111,20 +101,11 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
Long val = (Long) value;
if (val == null) {
val = NumericDocValuesUpdate.MISSING;
}
// grow the structures to have room for more elements
if (docs.size() == size) {
docs = docs.grow(size + 1);
values = values.grow(size + 1);
docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
}
if (val != NumericDocValuesUpdate.MISSING) {
// only mark the document as having a value in that field if the value wasn't set to null (MISSING)
docsWithField.set(size);
}
docs.set(size, doc);
@ -136,7 +117,6 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
public Iterator iterator() {
final PagedMutable docs = this.docs;
final PagedGrowableWriter values = this.values;
final FixedBitSet docsWithField = this.docsWithField;
new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
@ -147,18 +127,6 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
long tmpVal = values.get(j);
values.set(j, values.get(i));
values.set(i, tmpVal);
boolean tmpBool = docsWithField.get(j);
if (docsWithField.get(i)) {
docsWithField.set(j);
} else {
docsWithField.clear(j);
}
if (tmpBool) {
docsWithField.set(i);
} else {
docsWithField.clear(i);
}
}
@Override
@ -169,7 +137,7 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
}.sort(0, size);
return new Iterator(size, values, docsWithField, docs);
return new Iterator(size, values, docs);
}
@Override
@ -183,12 +151,8 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
docs = docs.grow(size + otherUpdates.size);
values = values.grow(size + otherUpdates.size);
docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
for (int i = 0; i < otherUpdates.size; i++) {
int doc = (int) otherUpdates.docs.get(i);
if (otherUpdates.docsWithField.get(i)) {
docsWithField.set(size);
}
docs.set(size, doc);
values.set(size, otherUpdates.values.get(i));
++size;
@ -202,7 +166,7 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
@Override
public long ramBytesPerDoc() {
long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue + 1 /* docsWithField */) / 8);
long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue) / 8);
final int capacity = estimateCapacity(size);
bytesPerDoc += (long) Math.ceil((double) values.ramBytesUsed() / capacity); // values
return bytesPerDoc;

121
lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
View File

@ -1,7 +1,6 @@
package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@ -466,80 +465,6 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
dir.close();
}
public void testUnsetValue() throws Exception {
assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
for (int i = 0; i < 2; i++) {
Document doc = new Document();
doc.add(new StringField("id", "doc" + i, Store.NO));
doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
writer.addDocument(doc);
}
writer.commit();
// unset the value of 'doc0'
writer.updateBinaryDocValue(new Term("id", "doc0"), "bdv", null);
writer.shutdown();
final DirectoryReader reader = DirectoryReader.open(dir);
AtomicReader r = reader.leaves().get(0).reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
if (i == 0) {
bdv.get(i, scratch);
assertEquals(0, scratch.length);
} else {
assertEquals(5, getValue(bdv, i, scratch));
}
}
Bits docsWithField = r.getDocsWithField("bdv");
assertFalse(docsWithField.get(0));
assertTrue(docsWithField.get(1));
reader.close();
dir.close();
}
public void testUnsetAllValues() throws Exception {
assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
for (int i = 0; i < 2; i++) {
Document doc = new Document();
doc.add(new StringField("id", "doc", Store.NO));
doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
writer.addDocument(doc);
}
writer.commit();
// unset the value of 'doc'
writer.updateBinaryDocValue(new Term("id", "doc"), "bdv", null);
writer.shutdown();
final DirectoryReader reader = DirectoryReader.open(dir);
AtomicReader r = reader.leaves().get(0).reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
bdv.get(i, scratch);
assertEquals(0, scratch.length);
}
Bits docsWithField = r.getDocsWithField("bdv");
assertFalse(docsWithField.get(0));
assertFalse(docsWithField.get(1));
reader.close();
dir.close();
}
public void testUpdateNonBinaryDocValuesField() throws Exception {
// we don't support adding new fields or updating existing non-binary-dv
// fields through binary updates
@ -757,8 +682,6 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
final int numFields = random.nextInt(4) + 3; // 3-7
final long[] fieldValues = new long[numFields];
final boolean[] fieldHasValue = new boolean[numFields];
Arrays.fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.length; i++) {
fieldValues[i] = 1;
}
@ -780,24 +703,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
++docID;
}
// if field's value was unset before, unset it from all new added documents too
for (int field = 0; field < fieldHasValue.length; field++) {
if (!fieldHasValue[field]) {
writer.updateBinaryDocValue(new Term("key", "all"), "f" + field, null);
}
}
int fieldIdx = random.nextInt(fieldValues.length);
String updateField = "f" + fieldIdx;
if (random.nextBoolean()) {
// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
fieldHasValue[fieldIdx] = false;
writer.updateBinaryDocValue(new Term("key", "all"), updateField, null);
} else {
fieldHasValue[fieldIdx] = true;
writer.updateBinaryDocValue(new Term("key", "all"), updateField, toBytes(++fieldValues[fieldIdx]));
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
}
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
writer.updateBinaryDocValue(new Term("key", "all"), updateField, toBytes(++fieldValues[fieldIdx]));
if (random.nextDouble() < 0.2) {
int deleteDoc = random.nextInt(docID); // might also delete an already deleted document, ok!
@ -831,12 +740,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
if (fieldHasValue[field]) {
assertTrue(docsWithField.get(doc));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc, scratch));
} else {
assertFalse(docsWithField.get(doc));
}
assertTrue(docsWithField.get(doc));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc, scratch));
}
}
}
@ -1049,14 +954,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
else if (group < 0.8) t = new Term("updKey", "g2");
else t = new Term("updKey", "g3");
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
if (random.nextBoolean()) { // sometimes unset a value
writer.updateBinaryDocValue(t, f, null);
writer.updateBinaryDocValue(t, cf, null);
} else {
long updValue = random.nextInt();
writer.updateBinaryDocValue(t, f, toBytes(updValue));
writer.updateBinaryDocValue(t, cf, toBytes(updValue * 2));
}
long updValue = random.nextInt();
writer.updateBinaryDocValue(t, f, toBytes(updValue));
writer.updateBinaryDocValue(t, cf, toBytes(updValue * 2));
if (random.nextDouble() < 0.2) {
// delete a random document
@ -1120,10 +1020,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
assertEquals(docsWithBdv.get(j), docsWithControl.get(j));
if (docsWithBdv.get(j)) {
assertEquals(getValue(control, j, scratch), getValue(bdv, j, scratch) * 2);
}
assertTrue(docsWithBdv.get(j));
assertTrue(docsWithControl.get(j));
assertEquals(getValue(control, j, scratch), getValue(bdv, j, scratch) * 2);
}
}
}

59
lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
View File

@ -1,7 +1,6 @@
package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@ -65,8 +64,6 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
final int numFields = random.nextInt(4) + 3; // 3-7
final int numNDVFields = random.nextInt(numFields/2) + 1; // 1-3
final long[] fieldValues = new long[numFields];
final boolean[] fieldHasValue = new boolean[numFields];
Arrays.fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.length; i++) {
fieldValues[i] = 1;
}
@ -92,36 +89,14 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
++docID;
}
// if field's value was unset before, unset it from all new added documents too
for (int field = 0; field < fieldHasValue.length; field++) {
if (!fieldHasValue[field]) {
if (field < numNDVFields) {
writer.updateNumericDocValue(new Term("key", "all"), "f" + field, null);
} else {
writer.updateBinaryDocValue(new Term("key", "all"), "f" + field, null);
}
}
}
int fieldIdx = random.nextInt(fieldValues.length);
String updateField = "f" + fieldIdx;
if (random.nextBoolean()) {
// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
fieldHasValue[fieldIdx] = false;
if (fieldIdx < numNDVFields) {
writer.updateNumericDocValue(new Term("key", "all"), updateField, null);
} else {
writer.updateBinaryDocValue(new Term("key", "all"), updateField, null);
}
if (fieldIdx < numNDVFields) {
writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
} else {
fieldHasValue[fieldIdx] = true;
if (fieldIdx < numNDVFields) {
writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
} else {
writer.updateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.toBytes(++fieldValues[fieldIdx]));
}
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
writer.updateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.toBytes(++fieldValues[fieldIdx]));
}
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
if (random.nextDouble() < 0.2) {
int deleteDoc = random.nextInt(docID); // might also delete an already deleted document, ok!
@ -162,15 +137,11 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
if (fieldHasValue[field]) {
assertTrue(docsWithField.get(doc));
if (field < numNDVFields) {
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
} else {
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc, scratch));
}
assertTrue(docsWithField.get(doc));
if (field < numNDVFields) {
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
} else {
assertFalse(docsWithField.get(doc));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc, scratch));
}
}
}
@ -232,16 +203,10 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
else if (group < 0.8) t = new Term("updKey", "g2");
else t = new Term("updKey", "g3");
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
if (random.nextBoolean()) { // sometimes unset a value
// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=UNSET");
writer.updateBinaryDocValue(t, f, null);
writer.updateNumericDocValue(t, cf, null);
} else {
long updValue = random.nextInt();
// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
writer.updateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.toBytes(updValue));
writer.updateNumericDocValue(t, cf, updValue * 2);
}
long updValue = random.nextInt();
// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
writer.updateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.toBytes(updValue));
writer.updateNumericDocValue(t, cf, updValue * 2);
if (random.nextDouble() < 0.2) {
// delete a random document

118
lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
View File

@ -1,7 +1,6 @@
package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@ -444,77 +443,6 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
dir.close();
}
@Test
public void testUnsetValue() throws Exception {
assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
for (int i = 0; i < 2; i++) {
Document doc = new Document();
doc.add(new StringField("id", "doc" + i, Store.NO));
doc.add(new NumericDocValuesField("ndv", 5));
writer.addDocument(doc);
}
writer.commit();
// unset the value of 'doc0'
writer.updateNumericDocValue(new Term("id", "doc0"), "ndv", null);
writer.shutdown();
final DirectoryReader reader = DirectoryReader.open(dir);
AtomicReader r = reader.leaves().get(0).reader();
NumericDocValues ndv = r.getNumericDocValues("ndv");
for (int i = 0; i < r.maxDoc(); i++) {
if (i == 0) {
assertEquals(0, ndv.get(i));
} else {
assertEquals(5, ndv.get(i));
}
}
Bits docsWithField = r.getDocsWithField("ndv");
assertFalse(docsWithField.get(0));
assertTrue(docsWithField.get(1));
reader.close();
dir.close();
}
public void testUnsetAllValues() throws Exception {
assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
for (int i = 0; i < 2; i++) {
Document doc = new Document();
doc.add(new StringField("id", "doc", Store.NO));
doc.add(new NumericDocValuesField("ndv", 5));
writer.addDocument(doc);
}
writer.commit();
// unset the value of 'doc'
writer.updateNumericDocValue(new Term("id", "doc"), "ndv", null);
writer.shutdown();
final DirectoryReader reader = DirectoryReader.open(dir);
AtomicReader r = reader.leaves().get(0).reader();
NumericDocValues ndv = r.getNumericDocValues("ndv");
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(0, ndv.get(i));
}
Bits docsWithField = r.getDocsWithField("ndv");
assertFalse(docsWithField.get(0));
assertFalse(docsWithField.get(1));
reader.close();
dir.close();
}
@Test
public void testUpdateNonNumericDocValuesField() throws Exception {
// we don't support adding new fields or updating existing non-numeric-dv
@ -735,8 +663,6 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
final int numFields = random.nextInt(4) + 3; // 3-7
final long[] fieldValues = new long[numFields];
final boolean[] fieldHasValue = new boolean[numFields];
Arrays.fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.length; i++) {
fieldValues[i] = 1;
}
@ -758,24 +684,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
++docID;
}
// if field's value was unset before, unset it from all new added documents too
for (int field = 0; field < fieldHasValue.length; field++) {
if (!fieldHasValue[field]) {
writer.updateNumericDocValue(new Term("key", "all"), "f" + field, null);
}
}
int fieldIdx = random.nextInt(fieldValues.length);
String updateField = "f" + fieldIdx;
if (random.nextBoolean()) {
// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
fieldHasValue[fieldIdx] = false;
writer.updateNumericDocValue(new Term("key", "all"), updateField, null);
} else {
fieldHasValue[fieldIdx] = true;
writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
}
writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
if (random.nextDouble() < 0.2) {
int deleteDoc = random.nextInt(docID); // might also delete an already deleted document, ok!
@ -808,12 +720,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + ndv.get(doc));
if (fieldHasValue[field]) {
assertTrue(docsWithField.get(doc));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
} else {
assertFalse(docsWithField.get(doc));
}
assertTrue(docsWithField.get(doc));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
}
}
}
@ -1028,14 +936,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
else if (group < 0.8) t = new Term("updKey", "g2");
else t = new Term("updKey", "g3");
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
if (random.nextBoolean()) { // sometimes unset a value
writer.updateNumericDocValue(t, f, null);
writer.updateNumericDocValue(t, cf, null);
} else {
long updValue = random.nextInt();
writer.updateNumericDocValue(t, f, updValue);
writer.updateNumericDocValue(t, cf, updValue * 2);
}
long updValue = random.nextInt();
writer.updateNumericDocValue(t, f, updValue);
writer.updateNumericDocValue(t, cf, updValue * 2);
if (random.nextDouble() < 0.2) {
// delete a random document
@ -1098,10 +1001,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
assertEquals(docsWithNdv.get(j), docsWithControl.get(j));
if (docsWithNdv.get(j)) {
assertEquals(control.get(j), ndv.get(j) * 2);
}
assertTrue(docsWithNdv.get(j));
assertTrue(docsWithControl.get(j));
assertEquals(control.get(j), ndv.get(j) * 2);
}
}
}