Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix test fails 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1410964 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-11-18 19:32:32 +00:00
parent 92bb3aaf0d
commit ef58554820
4 changed files with 69 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
View File

@ -23,8 +23,10 @@ import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.ParseException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.DocValuesArraySource;
@ -140,15 +142,25 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
static class SimpleTextDocValuesWriter extends SimpleDVConsumer {
final IndexOutput data;
final BytesRef scratch = new BytesRef();
final int numDocs; // for asserting
private final Set<String> fieldsSeen = new HashSet<String>(); // for asserting
SimpleTextDocValuesWriter(Directory dir, SegmentInfo si, IOContext context) throws IOException {
data = dir.createOutput(IndexFileNames.segmentFileName(si.name, "", "dat"), context);
numDocs = si.getDocCount();
}
// for asserting
private boolean fieldSeen(String field) {
assert !fieldsSeen.contains(field): "field \"" + field + "\" was added more than once during flush";
fieldsSeen.add(field);
return true;
}
@Override
public NumericDocValuesConsumer addNumericField(FieldInfo field, final long minValue, long maxValue) throws IOException {
assert fieldSeen(field.name);
writeFieldEntry(field);
// write our minimum value to the .dat, all entries are deltas from that
@ -189,17 +201,19 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
SimpleTextUtil.write(data, s, scratch);
SimpleTextUtil.writeNewline(data);
numDocsWritten++;
assert numDocsWritten <= numDocs;
}
@Override
public void finish() throws IOException {
assert numDocs == numDocsWritten;
assert numDocs == numDocsWritten: "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten;
}
};
}
@Override
public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, final int maxLength) throws IOException {
assert fieldSeen(field.name);
writeFieldEntry(field);
// write maxLength
SimpleTextUtil.write(data, MAXLENGTH);
@ -249,6 +263,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
// nocommit
@Override
public SortedDocValuesConsumer addSortedField(FieldInfo field, final int valueCount, boolean fixedLength, final int maxLength) throws IOException {
assert fieldSeen(field.name);
writeFieldEntry(field);
// write numValues
SimpleTextUtil.write(data, NUMVALUES);

7
lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java
View File

@ -184,6 +184,13 @@ public abstract class SortedDocValuesConsumer {
for(int docID=0;docID<maxDoc;docID++) {
if (liveDocs == null || liveDocs.get(docID)) {
int segOrd = segState.values.getOrd(docID);
/*
if (segState.segOrdToMergedOrd == null) {
BytesRef scratch = new BytesRef();
segState.values.lookupOrd(segOrd, scratch);
System.out.println("docID=" + docID + " segOrd=" + segOrd + " segValue=" + scratch.utf8ToString());
}
*/
int mergedOrd = segState.segOrdToMergedOrd[segOrd];
consumer.addDoc(mergedOrd);
}

19
lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
View File

@ -17,6 +17,9 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
@ -43,6 +46,7 @@ final class DocFieldProcessorPerField {
int fieldCount;
IndexableField[] fields = new IndexableField[1];
private final Map<FieldInfo,String> dvFields = new HashMap<FieldInfo,String>();
public DocFieldProcessorPerField(final DocFieldProcessor docFieldProcessor, final FieldInfo fieldInfo) {
this.consumer = docFieldProcessor.consumer.addField(fieldInfo);
@ -53,6 +57,7 @@ final class DocFieldProcessorPerField {
// nocommit make this generic chain through consumer?
public void addBytesDVField(int docID, BytesRef value) {
if (bytesDVWriter == null) {
verifyField(fieldInfo, "binary");
bytesDVWriter = new BytesDVWriter(fieldInfo, bytesUsed);
}
bytesDVWriter.addValue(docID, value);
@ -61,6 +66,7 @@ final class DocFieldProcessorPerField {
// nocommit make this generic chain through consumer?
public void addSortedBytesDVField(int docID, BytesRef value) {
if (sortedBytesDVWriter == null) {
verifyField(fieldInfo, "sorted");
sortedBytesDVWriter = new SortedBytesDVWriter(fieldInfo, bytesUsed);
}
sortedBytesDVWriter.addValue(docID, value);
@ -69,6 +75,7 @@ final class DocFieldProcessorPerField {
// nocommit make this generic chain through consumer?
public void addNumberDVField(int docID, Number value) {
if (numberDVWriter == null) {
verifyField(fieldInfo, "numeric");
numberDVWriter = new NumberDVWriter(fieldInfo, bytesUsed);
}
numberDVWriter.addValue(docID, value.longValue());
@ -77,6 +84,7 @@ final class DocFieldProcessorPerField {
// nocommit make this generic chain through consumer?
public void addFloatDVField(int docID, Number value) {
if (numberDVWriter == null) {
verifyField(fieldInfo, "numeric");
numberDVWriter = new NumberDVWriter(fieldInfo, bytesUsed);
}
numberDVWriter.addValue(docID, Float.floatToRawIntBits(value.floatValue()));
@ -85,11 +93,19 @@ final class DocFieldProcessorPerField {
// nocommit make this generic chain through consumer?
public void addDoubleDVField(int docID, Number value) {
if (numberDVWriter == null) {
verifyField(fieldInfo, "numeric");
numberDVWriter = new NumberDVWriter(fieldInfo, bytesUsed);
}
numberDVWriter.addValue(docID, Double.doubleToRawLongBits(value.doubleValue()));
}
private void verifyField(FieldInfo field, String type) {
if (dvFields.containsKey(field)) {
throw new IllegalArgumentException("Incompatible DocValues type: field \"" + field.name + "\" changed from " + dvFields.get(field) + " to " + type);
}
dvFields.put(field, type);
}
public void addField(IndexableField field) {
if (fieldCount == fields.length) {
int newSize = ArrayUtil.oversize(fieldCount + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
@ -106,6 +122,9 @@ final class DocFieldProcessorPerField {
if (bytesDVWriter != null) {
bytesDVWriter.abort();
}
if (sortedBytesDVWriter != null) {
sortedBytesDVWriter.abort();
}
if (numberDVWriter != null) {
numberDVWriter.abort();
}

27
lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
View File

@ -808,6 +808,33 @@ public class TestDocValuesIndexing extends LuceneTestCase {
r.close();
d.close();
}
public void testDifferentTypedDocValuesField2() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
// Index doc values are single-valued so we should not
// be able to add same field more than once:
Field f;
doc.add(f = new PackedLongDocValuesField("field", 17));
doc.add(new SortedBytesDocValuesField("field", new BytesRef("hello")));
try {
w.addDocument(doc);
fail("didn't hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
}
doc = new Document();
doc.add(f);
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = w.getReader();
w.close();
assertEquals(17, getOnlySegmentReader(r).docValues("field").loadSource().getInt(0));
r.close();
d.close();
}
public void testSortedBytes() throws IOException {
Type[] types = new Type[] { Type.BYTES_FIXED_SORTED, Type.BYTES_VAR_SORTED };