Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users to select a fewer number of dimensions to be used for creating the index than the total number of dimensions used for field encoding. i.e., dimensions 0 to N may be used to determine how to split the inner nodes, and dimensions N+1 to D are ignored and stored as data dimensions at the leaves.

This commit is contained in:
Nicholas Knize 2018-10-08 18:51:03 -05:00
parent df07a43b56
commit 1118299c33
73 changed files with 893 additions and 609 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lucene/CHANGES.txt
View File

@ -192,6 +192,14 @@ Bug fixes:
* LUCENE-8479: QueryBuilder#analyzeGraphPhrase now throws TooManyClause exception
if the number of expanded path reaches the BooleanQuery#maxClause limit. (Jim Ferenczi)
New Features
* LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users
to select a fewer number of dimensions to be used for creating the index than
the total number of dimensions used for field encoding. i.e., dimensions 0 to N
may be used to determine how to split the inner nodes, and dimensions N+1 to D
are ignored and stored as data dimensions at the leaves. (Nick Knize)
======================= Lucene 7.5.1 =======================
Bug Fixes:

59
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
View File

@ -41,7 +41,8 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
final private byte[] splitPackedValues;
final long[] leafBlockFPs;
final private int leafNodeOffset;
final int numDims;
final int numDataDims;
final int numIndexDims;
final int bytesPerDim;
final int bytesPerIndexEntry;
final IndexInput in;
@ -52,16 +53,19 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
final int docCount;
final int version;
protected final int packedBytesLength;
protected final int packedIndexBytesLength;
public SimpleTextBKDReader(IndexInput in, int numDims, int maxPointsInLeafNode, int bytesPerDim, long[] leafBlockFPs, byte[] splitPackedValues,
public SimpleTextBKDReader(IndexInput in, int numDataDims, int numIndexDims, int maxPointsInLeafNode, int bytesPerDim, long[] leafBlockFPs, byte[] splitPackedValues,
byte[] minPackedValue, byte[] maxPackedValue, long pointCount, int docCount) throws IOException {
this.in = in;
this.numDims = numDims;
this.numDataDims = numDataDims;
this.numIndexDims = numIndexDims;
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.bytesPerDim = bytesPerDim;
// no version check here because callers of this API (SimpleText) have no back compat:
bytesPerIndexEntry = numDims == 1 ? bytesPerDim : bytesPerDim + 1;
packedBytesLength = numDims * bytesPerDim;
bytesPerIndexEntry = numIndexDims == 1 ? bytesPerDim : bytesPerDim + 1;
packedBytesLength = numDataDims * bytesPerDim;
packedIndexBytesLength = numIndexDims * bytesPerDim;
this.leafNodeOffset = leafBlockFPs.length;
this.leafBlockFPs = leafBlockFPs;
this.splitPackedValues = splitPackedValues;
@ -70,8 +74,8 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
this.pointCount = pointCount;
this.docCount = docCount;
this.version = SimpleTextBKDWriter.VERSION_CURRENT;
assert minPackedValue.length == packedBytesLength;
assert maxPackedValue.length == packedBytesLength;
assert minPackedValue.length == packedIndexBytesLength;
assert maxPackedValue.length == packedIndexBytesLength;
}
/** Used to track all state for a single call to {@link #intersect}. */
@ -115,7 +119,7 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
/** Create a new {@link IntersectState} */
public IntersectState getIntersectState(IntersectVisitor visitor) {
return new IntersectState(in.clone(), numDims,
return new IntersectState(in.clone(), numDataDims,
packedBytesLength,
maxPointsInLeafNode,
visitor);
@ -181,7 +185,7 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
scratchPackedValue[compressedByteOffset] = in.readByte();
final int runLen = Byte.toUnsignedInt(in.readByte());
for (int j = 0; j < runLen; ++j) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
}
@ -196,14 +200,14 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
private int readCompressedDim(IndexInput in) throws IOException {
int compressedDim = in.readByte();
if (compressedDim < -1 || compressedDim >= numDims) {
if (compressedDim < -1 || compressedDim >= numIndexDims) {
throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
}
return compressedDim;
}
private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = in.readVInt();
commonPrefixLengths[dim] = prefix;
if (prefix > 0) {
@ -258,27 +262,27 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
int address = nodeID * bytesPerIndexEntry;
int splitDim;
if (numDims == 1) {
if (numIndexDims == 1) {
splitDim = 0;
} else {
splitDim = splitPackedValues[address++] & 0xff;
}
assert splitDim < numDims;
assert splitDim < numIndexDims;
// TODO: can we alloc & reuse this up front?
byte[] splitPackedValue = new byte[packedBytesLength];
byte[] splitPackedValue = new byte[packedIndexBytesLength];
// Recurse on left sub-tree:
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
intersect(state,
2*nodeID,
cellMinPacked, splitPackedValue);
// Recurse on right sub-tree:
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
intersect(state,
2*nodeID+1,
@ -307,27 +311,27 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
int address = nodeID * bytesPerIndexEntry;
int splitDim;
if (numDims == 1) {
if (numIndexDims == 1) {
splitDim = 0;
} else {
splitDim = splitPackedValues[address++] & 0xff;
}
assert splitDim < numDims;
assert splitDim < numIndexDims;
// TODO: can we alloc & reuse this up front?
byte[] splitPackedValue = new byte[packedBytesLength];
byte[] splitPackedValue = new byte[packedIndexBytesLength];
// Recurse on left sub-tree:
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
final long leftCost = estimatePointCount(state,
2*nodeID,
cellMinPacked, splitPackedValue);
// Recurse on right sub-tree:
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
final long rightCost = estimatePointCount(state,
2*nodeID+1,
@ -340,13 +344,13 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
public void copySplitValue(int nodeID, byte[] splitPackedValue) {
int address = nodeID * bytesPerIndexEntry;
int splitDim;
if (numDims == 1) {
if (numIndexDims == 1) {
splitDim = 0;
} else {
splitDim = splitPackedValues[address++] & 0xff;
}
assert splitDim < numDims;
assert splitDim < numIndexDims;
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
}
@ -367,8 +371,13 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
}
@Override
public int getNumDimensions() {
return numDims;
public int getNumDataDimensions() {
return numDataDims;
}
@Override
public int getNumIndexDimensions() {
return numIndexDims;
}
@Override

144
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDWriter.java
View File

@ -62,7 +62,8 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.INDEX_C
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_LEAF_POINTS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MIN_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DATA_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_INDEX_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.POINT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_DIM;
@ -102,15 +103,21 @@ final class SimpleTextBKDWriter implements Closeable {
/** Maximum number of dimensions */
public static final int MAX_DIMS = 8;
/** How many dimensions we are indexing */
protected final int numDims;
/** How many dimensions we are storing at the leaf (data) nodes */
protected final int numDataDims;
/** How many dimensions we are indexing in the internal nodes */
protected final int numIndexDims;
/** How many bytes each value in each dimension takes. */
protected final int bytesPerDim;
/** numDims * bytesPerDim */
/** numDataDims * bytesPerDim */
protected final int packedBytesLength;
/** numIndexDims * bytesPerDim */
protected final int packedIndexBytesLength;
final BytesRefBuilder scratch = new BytesRefBuilder();
final TrackingDirectoryWrapper tempDir;
@ -158,37 +165,39 @@ final class SimpleTextBKDWriter implements Closeable {
private final int maxDoc;
public SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
public SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException {
this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
this(maxDoc, tempDir, tempFileNamePrefix, numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
totalPointCount > Integer.MAX_VALUE, Math.max(1, (long) maxMBSortInHeap), OfflineSorter.MAX_TEMPFILES);
}
private SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
private SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount,
boolean singleValuePerDoc, boolean longOrds, long offlineSorterBufferMB, int offlineSorterMaxTempFiles) throws IOException {
verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
verifyParams(numDataDims, numIndexDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
this.tempDir = new TrackingDirectoryWrapper(tempDir);
this.tempFileNamePrefix = tempFileNamePrefix;
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.numDims = numDims;
this.numDataDims = numDataDims;
this.numIndexDims = numIndexDims;
this.bytesPerDim = bytesPerDim;
this.totalPointCount = totalPointCount;
this.maxDoc = maxDoc;
this.offlineSorterBufferMB = OfflineSorter.BufferSize.megabytes(offlineSorterBufferMB);
this.offlineSorterMaxTempFiles = offlineSorterMaxTempFiles;
docsSeen = new FixedBitSet(maxDoc);
packedBytesLength = numDims * bytesPerDim;
packedBytesLength = numDataDims * bytesPerDim;
packedIndexBytesLength = numIndexDims * bytesPerDim;
scratchDiff = new byte[bytesPerDim];
scratch1 = new byte[packedBytesLength];
scratch2 = new byte[packedBytesLength];
commonPrefixLengths = new int[numDims];
commonPrefixLengths = new int[numDataDims];
minPackedValue = new byte[packedBytesLength];
maxPackedValue = new byte[packedBytesLength];
minPackedValue = new byte[packedIndexBytesLength];
maxPackedValue = new byte[packedIndexBytesLength];
// If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes).
this.longOrds = longOrds;
@ -215,7 +224,7 @@ final class SimpleTextBKDWriter implements Closeable {
// bytes to points here. Each dimension has its own sorted partition, so
// we must divide by numDims as wel.
maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims));
maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDataDims));
// Finally, we must be able to hold at least the leaf node in heap during build:
if (maxPointsSortInHeap < maxPointsInLeafNode) {
@ -228,11 +237,14 @@ final class SimpleTextBKDWriter implements Closeable {
this.maxMBSortInHeap = maxMBSortInHeap;
}
public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
public static void verifyParams(int numDataDims, int numIndexDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
// We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
// remaining 4 bits for another purpose later
if (numDims < 1 || numDims > MAX_DIMS) {
throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
if (numDataDims < 1 || numDataDims > MAX_DIMS) {
throw new IllegalArgumentException("numDataDims must be 1 .. " + MAX_DIMS + " (got: " + numDataDims + ")");
}
if (numIndexDims < 1 || numIndexDims > numDataDims) {
throw new IllegalArgumentException("numIndexDims must be 1 .. " + numDataDims + " (got: " + numIndexDims + ")");
}
if (maxPointsInLeafNode <= 0) {
throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
@ -281,10 +293,10 @@ final class SimpleTextBKDWriter implements Closeable {
// TODO: we could specialize for the 1D case:
if (pointCount == 0) {
System.arraycopy(packedValue, 0, minPackedValue, 0, packedBytesLength);
System.arraycopy(packedValue, 0, maxPackedValue, 0, packedBytesLength);
System.arraycopy(packedValue, 0, minPackedValue, 0, packedIndexBytesLength);
System.arraycopy(packedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
} else {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(packedValue, offset, minPackedValue, offset, bytesPerDim);
@ -313,7 +325,7 @@ final class SimpleTextBKDWriter implements Closeable {
* disk. This method does not use transient disk in order to reorder points.
*/
public long writeField(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
if (numDims == 1) {
if (numIndexDims == 1) {
return writeField1Dim(out, fieldName, reader);
} else {
return writeFieldNDims(out, fieldName, reader);
@ -356,7 +368,7 @@ final class SimpleTextBKDWriter implements Closeable {
Arrays.fill(maxPackedValue, (byte) 0);
for (int i = 0; i < Math.toIntExact(pointCount); ++i) {
values.getValue(i, scratchBytesRef1);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, scratchBytesRef1.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, minPackedValue, offset, bytesPerDim);
@ -382,7 +394,7 @@ final class SimpleTextBKDWriter implements Closeable {
/* In the 1D case, we can simply sort points in ascending order and use the
* same writing logic as we use at merge time. */
private long writeField1Dim(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
MutablePointsReaderUtils.sort(maxDoc, packedBytesLength, reader, 0, Math.toIntExact(reader.size()));
MutablePointsReaderUtils.sort(maxDoc, packedIndexBytesLength, reader, 0, Math.toIntExact(reader.size()));
final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
@ -418,8 +430,8 @@ final class SimpleTextBKDWriter implements Closeable {
int leafCount;
OneDimensionBKDWriter(IndexOutput out) {
if (numDims != 1) {
throw new UnsupportedOperationException("numDims must be 1 but got " + numDims);
if (numIndexDims != 1) {
throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
}
if (pointCount != 0) {
throw new IllegalStateException("cannot mix add and merge");
@ -497,9 +509,9 @@ final class SimpleTextBKDWriter implements Closeable {
private void writeLeafBlock() throws IOException {
assert leafCount != 0;
if (valueCount == 0) {
System.arraycopy(leafValues, 0, minPackedValue, 0, packedBytesLength);
System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);
}
System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedBytesLength);
System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedIndexBytesLength);
valueCount += leafCount;
@ -512,7 +524,7 @@ final class SimpleTextBKDWriter implements Closeable {
Arrays.fill(commonPrefixLengths, bytesPerDim);
// Find per-dim common prefix:
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int offset1 = dim * bytesPerDim;
int offset2 = (leafCount - 1) * packedBytesLength + offset1;
for(int j=0;j<commonPrefixLengths[dim];j++) {
@ -658,7 +670,7 @@ final class SimpleTextBKDWriter implements Closeable {
}
private PointWriter sort(int dim) throws IOException {
assert dim >= 0 && dim < numDims;
assert dim >= 0 && dim < numDataDims;
if (heapPointWriter != null) {
@ -691,7 +703,7 @@ final class SimpleTextBKDWriter implements Closeable {
final int offset = bytesPerDim * dim;
Comparator<BytesRef> cmp;
if (dim == numDims - 1) {
if (dim == numDataDims - 1) {
// in that case the bytes for the dimension and for the doc id are contiguous,
// so we don't need a branch
cmp = new BytesRefComparator(bytesPerDim + Integer.BYTES) {
@ -776,7 +788,7 @@ final class SimpleTextBKDWriter implements Closeable {
}
LongBitSet ordBitSet;
if (numDims > 1) {
if (numDataDims > 1) {
if (singleValuePerDoc) {
ordBitSet = new LongBitSet(maxDoc);
} else {
@ -811,7 +823,7 @@ final class SimpleTextBKDWriter implements Closeable {
assert pointCount / numLeaves <= maxPointsInLeafNode: "pointCount=" + pointCount + " numLeaves=" + numLeaves + " maxPointsInLeafNode=" + maxPointsInLeafNode;
// Sort all docs once by each dimension:
PathSlice[] sortedPointWriters = new PathSlice[numDims];
PathSlice[] sortedPointWriters = new PathSlice[numDataDims];
// This is only used on exception; on normal code paths we close all files we opened:
List<Closeable> toCloseHeroically = new ArrayList<>();
@ -819,7 +831,7 @@ final class SimpleTextBKDWriter implements Closeable {
boolean success = false;
try {
//long t0 = System.nanoTime();
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
sortedPointWriters[dim] = new PathSlice(sort(dim), 0, pointCount);
}
//long t1 = System.nanoTime();
@ -867,8 +879,12 @@ final class SimpleTextBKDWriter implements Closeable {
/** Subclass can change how it writes the index. */
private void writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
write(out, NUM_DIMS);
writeInt(out, numDims);
write(out, NUM_DATA_DIMS);
writeInt(out, numDataDims);
newline(out);
write(out, NUM_INDEX_DIMS);
writeInt(out, numIndexDims);
newline(out);
write(out, BYTES_PER_DIM);
@ -952,7 +968,7 @@ final class SimpleTextBKDWriter implements Closeable {
BytesRef ref = packedValues.apply(i);
assert ref.length == packedBytesLength;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
out.writeBytes(ref.bytes, ref.offset + dim*bytesPerDim + prefix, bytesPerDim-prefix);
}
@ -1036,7 +1052,7 @@ final class SimpleTextBKDWriter implements Closeable {
boolean result = reader.next();
assert result;
System.arraycopy(reader.packedValue(), splitDim*bytesPerDim, scratch1, 0, bytesPerDim);
if (numDims > 1) {
if (numDataDims > 1) {
assert ordBitSet.get(reader.ord()) == false;
ordBitSet.set(reader.ord());
// Subtract 1 from rightCount because we already did the first value above (so we could record the split value):
@ -1051,7 +1067,7 @@ final class SimpleTextBKDWriter implements Closeable {
/** Called only in assert */
private boolean valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim*dim;
if (FutureArrays.compareUnsigned(packedValue.bytes, packedValue.offset + offset, packedValue.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
return false;
@ -1067,7 +1083,7 @@ final class SimpleTextBKDWriter implements Closeable {
protected int split(byte[] minPackedValue, byte[] maxPackedValue) {
// Find which dim has the largest span so we can split on it:
int splitDim = -1;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
if (splitDim == -1 || FutureArrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
@ -1115,7 +1131,7 @@ final class SimpleTextBKDWriter implements Closeable {
reader.getValue(from, scratchBytesRef1);
for (int i = from + 1; i < to; ++i) {
reader.getValue(i, scratchBytesRef2);
for (int dim=0;dim<numDims;dim++) {
for (int dim=0;dim<numDataDims;dim++) {
final int offset = dim * bytesPerDim;
for(int j=0;j<commonPrefixLengths[dim];j++) {
if (scratchBytesRef1.bytes[scratchBytesRef1.offset+offset+j] != scratchBytesRef2.bytes[scratchBytesRef2.offset+offset+j]) {
@ -1127,14 +1143,14 @@ final class SimpleTextBKDWriter implements Closeable {
}
// Find the dimension that has the least number of unique bytes at commonPrefixLengths[dim]
FixedBitSet[] usedBytes = new FixedBitSet[numDims];
for (int dim = 0; dim < numDims; ++dim) {
FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];
for (int dim = 0; dim < numDataDims; ++dim) {
if (commonPrefixLengths[dim] < bytesPerDim) {
usedBytes[dim] = new FixedBitSet(256);
}
}
for (int i = from + 1; i < to; ++i) {
for (int dim=0;dim<numDims;dim++) {
for (int dim=0;dim<numDataDims;dim++) {
if (usedBytes[dim] != null) {
byte b = reader.getByteAt(i, dim * bytesPerDim + commonPrefixLengths[dim]);
usedBytes[dim].set(Byte.toUnsignedInt(b));
@ -1143,7 +1159,7 @@ final class SimpleTextBKDWriter implements Closeable {
}
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
for (int dim = 0; dim < numDims; ++dim) {
for (int dim = 0; dim < numDataDims; ++dim) {
if (usedBytes[dim] != null) {
final int cardinality = usedBytes[dim].cardinality();
if (cardinality < sortedDimCardinality) {
@ -1206,8 +1222,8 @@ final class SimpleTextBKDWriter implements Closeable {
reader.getValue(mid, scratchBytesRef1);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim, splitPackedValues, address + 1, bytesPerDim);
byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedBytesLength);
byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedBytesLength);
byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);
byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
minSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
@ -1235,7 +1251,7 @@ final class SimpleTextBKDWriter implements Closeable {
assert slice.count == slices[0].count;
}
if (numDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
if (numDataDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
// Special case for 1D, to cutover to heap once we recurse deeply enough:
slices[0] = switchToHeap(slices[0], toCloseHeroically);
}
@ -1248,7 +1264,7 @@ final class SimpleTextBKDWriter implements Closeable {
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
for (int dim=0;dim<numDims;dim++) {
for (int dim=0;dim<numDataDims;dim++) {
if (slices[dim].writer instanceof HeapPointWriter == false) {
// Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started
// offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer
@ -1333,7 +1349,7 @@ final class SimpleTextBKDWriter implements Closeable {
// Inner node: partition/recurse
int splitDim;
if (numDims > 1) {
if (numIndexDims > 1) {
splitDim = split(minPackedValue, maxPackedValue);
} else {
splitDim = 0;
@ -1354,24 +1370,24 @@ final class SimpleTextBKDWriter implements Closeable {
// Partition all PathSlice that are not the split dim into sorted left and right sets, so we can recurse:
PathSlice[] leftSlices = new PathSlice[numDims];
PathSlice[] rightSlices = new PathSlice[numDims];
PathSlice[] leftSlices = new PathSlice[numDataDims];
PathSlice[] rightSlices = new PathSlice[numDataDims];
byte[] minSplitPackedValue = new byte[packedBytesLength];
System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedBytesLength);
byte[] minSplitPackedValue = new byte[packedIndexBytesLength];
System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);
byte[] maxSplitPackedValue = new byte[packedBytesLength];
System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedBytesLength);
byte[] maxSplitPackedValue = new byte[packedIndexBytesLength];
System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedIndexBytesLength);
// When we are on this dim, below, we clear the ordBitSet:
int dimToClear;
if (numDims - 1 == splitDim) {
dimToClear = numDims - 2;
if (numDataDims - 1 == splitDim) {
dimToClear = numDataDims - 2;
} else {
dimToClear = numDims - 1;
dimToClear = numDataDims - 1;
}
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
if (dim == splitDim) {
// No need to partition on this dim since it's a simple slice of the incoming already sorted slice, and we
@ -1392,7 +1408,7 @@ final class SimpleTextBKDWriter implements Closeable {
long nextRightCount = reader.split(source.count, ordBitSet, leftPointWriter, rightPointWriter, dim == dimToClear);
if (rightCount != nextRightCount) {
throw new IllegalStateException("wrong number of points in split: expected=" + rightCount + " but actual=" + nextRightCount);
throw new IllegalStateException("wrong number of points in split: expected=" + rightCount + " but actual=" + nextRightCount + " in dim " + dim);
}
leftSlices[dim] = new PathSlice(leftPointWriter, 0, leftCount);
@ -1407,7 +1423,7 @@ final class SimpleTextBKDWriter implements Closeable {
ordBitSet, out,
minPackedValue, maxSplitPackedValue,
splitPackedValues, leafBlockFPs, toCloseHeroically);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
leftSlices[dim].writer.destroy();
@ -1420,7 +1436,7 @@ final class SimpleTextBKDWriter implements Closeable {
ordBitSet, out,
minSplitPackedValue, maxPackedValue,
splitPackedValues, leafBlockFPs, toCloseHeroically);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
rightSlices[dim].writer.destroy();
@ -1454,10 +1470,10 @@ final class SimpleTextBKDWriter implements Closeable {
if (ord > 0) {
int cmp = FutureArrays.compareUnsigned(lastPackedValue, dimOffset, dimOffset + bytesPerDim, packedValue, packedValueOffset + dimOffset, packedValueOffset + dimOffset + bytesPerDim);
if (cmp > 0) {
throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord);
throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord + " sortedDim=" + sortedDim);
}
if (cmp == 0 && doc < lastDoc) {
throw new AssertionError("docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord);
throw new AssertionError("docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord + " sortedDim=" + sortedDim);
}
}
System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, packedBytesLength);

21
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
View File

@ -64,7 +64,8 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
static final BytesRef NUM_ATTS = new BytesRef(" attributes ");
static final BytesRef ATT_KEY = new BytesRef(" key ");
static final BytesRef ATT_VALUE = new BytesRef(" value ");
static final BytesRef DIM_COUNT = new BytesRef(" dimensional count ");
static final BytesRef DATA_DIM_COUNT = new BytesRef(" data dimensional count ");
static final BytesRef INDEX_DIM_COUNT = new BytesRef(" index dimensional count ");
static final BytesRef DIM_NUM_BYTES = new BytesRef(" dimensional num bytes ");
static final BytesRef SOFT_DELETES = new BytesRef(" soft-deletes ");
@ -134,8 +135,12 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
}
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DIM_COUNT);
int dimensionalCount = Integer.parseInt(readString(DIM_COUNT.length, scratch));
assert StringHelper.startsWith(scratch.get(), DATA_DIM_COUNT);
int dataDimensionalCount = Integer.parseInt(readString(DATA_DIM_COUNT.length, scratch));
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), INDEX_DIM_COUNT);
int indexDimensionalCount = Integer.parseInt(readString(INDEX_DIM_COUNT.length, scratch));
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
@ -147,7 +152,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts),
dimensionalCount, dimensionalNumBytes, isSoftDeletesField);
dataDimensionalCount, indexDimensionalCount, dimensionalNumBytes, isSoftDeletesField);
}
SimpleTextUtil.checkFooter(input);
@ -236,8 +241,12 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
}
}
SimpleTextUtil.write(out, DIM_COUNT);
SimpleTextUtil.write(out, Integer.toString(fi.getPointDimensionCount()), scratch);
SimpleTextUtil.write(out, DATA_DIM_COUNT);
SimpleTextUtil.write(out, Integer.toString(fi.getPointDataDimensionCount()), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, INDEX_DIM_COUNT);
SimpleTextUtil.write(out, Integer.toString(fi.getPointIndexDimensionCount()), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, DIM_NUM_BYTES);

20
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsReader.java
View File

@ -47,7 +47,8 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.INDEX_C
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_LEAF_POINTS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MIN_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DATA_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_INDEX_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.POINT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_DIM;
@ -101,7 +102,10 @@ class SimpleTextPointsReader extends PointsReader {
// NOTE: matches what writeIndex does in SimpleTextPointsWriter
dataIn.seek(fp);
readLine(dataIn);
int numDims = parseInt(NUM_DIMS);
int numDataDims = parseInt(NUM_DATA_DIMS);
readLine(dataIn);
int numIndexDims = parseInt(NUM_INDEX_DIMS);
readLine(dataIn);
int bytesPerDim = parseInt(BYTES_PER_DIM);
@ -115,12 +119,12 @@ class SimpleTextPointsReader extends PointsReader {
readLine(dataIn);
assert startsWith(MIN_VALUE);
BytesRef minValue = SimpleTextUtil.fromBytesRefString(stripPrefix(MIN_VALUE));
assert minValue.length == numDims*bytesPerDim;
assert minValue.length == numIndexDims*bytesPerDim;
readLine(dataIn);
assert startsWith(MAX_VALUE);
BytesRef maxValue = SimpleTextUtil.fromBytesRefString(stripPrefix(MAX_VALUE));
assert maxValue.length == numDims*bytesPerDim;
assert maxValue.length == numIndexDims*bytesPerDim;
readLine(dataIn);
assert startsWith(POINT_COUNT);
@ -140,7 +144,7 @@ class SimpleTextPointsReader extends PointsReader {
byte[] splitPackedValues;
int bytesPerIndexEntry;
if (numDims == 1) {
if (numIndexDims == 1) {
bytesPerIndexEntry = bytesPerDim;
} else {
bytesPerIndexEntry = 1 + bytesPerDim;
@ -150,7 +154,7 @@ class SimpleTextPointsReader extends PointsReader {
readLine(dataIn);
int address = bytesPerIndexEntry * i;
int splitDim = parseInt(SPLIT_DIM);
if (numDims != 1) {
if (numIndexDims != 1) {
splitPackedValues[address++] = (byte) splitDim;
}
readLine(dataIn);
@ -160,7 +164,7 @@ class SimpleTextPointsReader extends PointsReader {
System.arraycopy(br.bytes, br.offset, splitPackedValues, address, bytesPerDim);
}
return new SimpleTextBKDReader(dataIn, numDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minValue.bytes, maxValue.bytes, pointCount, docCount);
return new SimpleTextBKDReader(dataIn, numDataDims, numIndexDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minValue.bytes, maxValue.bytes, pointCount, docCount);
}
private void readLine(IndexInput in) throws IOException {
@ -191,7 +195,7 @@ class SimpleTextPointsReader extends PointsReader {
if (fieldInfo == null) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
}
if (fieldInfo.getPointDimensionCount() == 0) {
if (fieldInfo.getPointDataDimensionCount() == 0) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index points");
}
return readers.get(fieldName);

6
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
View File

@ -35,7 +35,8 @@ import org.apache.lucene.util.BytesRefBuilder;
class SimpleTextPointsWriter extends PointsWriter {
public final static BytesRef NUM_DIMS = new BytesRef("num dims ");
public final static BytesRef NUM_DATA_DIMS = new BytesRef("num data dims ");
public final static BytesRef NUM_INDEX_DIMS = new BytesRef("num index dims ");
public final static BytesRef BYTES_PER_DIM = new BytesRef("bytes per dim ");
public final static BytesRef MAX_LEAF_POINTS = new BytesRef("max leaf points ");
public final static BytesRef INDEX_COUNT = new BytesRef("index count ");
@ -76,7 +77,8 @@ class SimpleTextPointsWriter extends PointsWriter {
try (SimpleTextBKDWriter writer = new SimpleTextBKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointDataDimensionCount(),
fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
SimpleTextBKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
SimpleTextBKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,

13
lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
View File

@ -48,7 +48,7 @@ public abstract class PointsWriter implements Closeable {
PointsReader pointsReader = mergeState.pointsReaders[i];
if (pointsReader != null) {
FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
if (readerFieldInfo != null && readerFieldInfo.getPointDataDimensionCount() > 0) {
PointValues values = pointsReader.getValues(fieldInfo.name);
if (values != null) {
maxPointCount += values.size();
@ -92,7 +92,7 @@ public abstract class PointsWriter implements Closeable {
continue;
}
if (readerFieldInfo.getPointDimensionCount() == 0) {
if (readerFieldInfo.getPointDataDimensionCount() == 0) {
// This segment saw this field, but the field did not index points in it:
continue;
}
@ -143,7 +143,12 @@ public abstract class PointsWriter implements Closeable {
}
@Override
public int getNumDimensions() {
public int getNumDataDimensions() {
throw new UnsupportedOperationException();
}
@Override
public int getNumIndexDimensions() {
throw new UnsupportedOperationException();
}
@ -182,7 +187,7 @@ public abstract class PointsWriter implements Closeable {
}
// merge field at a time
for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
if (fieldInfo.getPointDimensionCount() != 0) {
if (fieldInfo.getPointDataDimensionCount() != 0) {
mergeOneField(mergeState, fieldInfo);
}
}

2
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
View File

@ -148,7 +148,7 @@ public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
lastAttributes = attributes;
try {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValuesType, dvGen, attributes, 0, 0, false);
indexOptions, docValuesType, dvGen, attributes, 0, 0, 0, false);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);

21
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
View File

@ -114,7 +114,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
Throwable priorE = null;
FieldInfo infos[] = null;
try {
CodecUtil.checkIndexHeader(input,
int version = CodecUtil.checkIndexHeader(input,
Lucene60FieldInfosFormat.CODEC_NAME,
Lucene60FieldInfosFormat.FORMAT_START,
Lucene60FieldInfosFormat.FORMAT_CURRENT,
@ -149,9 +149,13 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
attributes = lastAttributes;
}
lastAttributes = attributes;
int pointDimensionCount = input.readVInt();
int pointDataDimensionCount = input.readVInt();
int pointNumBytes;
if (pointDimensionCount != 0) {
int pointIndexDimensionCount = pointDataDimensionCount;
if (pointDataDimensionCount != 0) {
if (version >= Lucene60FieldInfosFormat.FORMAT_SELECTIVE_INDEXING) {
pointIndexDimensionCount = input.readVInt();
}
pointNumBytes = input.readVInt();
} else {
pointNumBytes = 0;
@ -160,7 +164,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
try {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValuesType, dvGen, attributes,
pointDimensionCount, pointNumBytes, isSoftDeletesField);
pointDataDimensionCount, pointIndexDimensionCount, pointNumBytes, isSoftDeletesField);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@ -287,9 +291,9 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
output.writeByte(docValuesByte(fi.getDocValuesType()));
output.writeLong(fi.getDocValuesGen());
output.writeMapOfStrings(fi.attributes());
int pointDimensionCount = fi.getPointDimensionCount();
output.writeVInt(pointDimensionCount);
if (pointDimensionCount != 0) {
output.writeVInt(fi.getPointDataDimensionCount());
if (fi.getPointDataDimensionCount() != 0) {
output.writeVInt(fi.getPointIndexDimensionCount());
output.writeVInt(fi.getPointNumBytes());
}
}
@ -304,7 +308,8 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
static final String CODEC_NAME = "Lucene60FieldInfos";
static final int FORMAT_START = 0;
static final int FORMAT_SOFT_DELETES = 1;
static final int FORMAT_CURRENT = FORMAT_SOFT_DELETES;
static final int FORMAT_SELECTIVE_INDEXING = 2;
static final int FORMAT_CURRENT = FORMAT_SELECTIVE_INDEXING;
// Field flags
static final byte STORE_TERMVECTOR = 0x1;

2
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
View File

@ -124,7 +124,7 @@ public class Lucene60PointsReader extends PointsReader implements Closeable {
if (fieldInfo == null) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
}
if (fieldInfo.getPointDimensionCount() == 0) {
if (fieldInfo.getPointDataDimensionCount() == 0) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");
}

14
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
View File

@ -94,7 +94,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointDataDimensionCount(),
fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
maxMBSortInHeap,
@ -152,8 +153,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
}
for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
if (fieldInfo.getPointDimensionCount() != 0) {
if (fieldInfo.getPointDimensionCount() == 1) {
if (fieldInfo.getPointDataDimensionCount() != 0) {
if (fieldInfo.getPointDataDimensionCount() == 1) {
boolean singleValuePerDoc = true;
@ -164,7 +165,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
if (reader != null) {
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
if (readerFieldInfo != null && readerFieldInfo.getPointDataDimensionCount() > 0) {
PointValues values = reader.getValues(fieldInfo.name);
if (values != null) {
totMaxSize += values.size();
@ -181,7 +182,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointDataDimensionCount(),
fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
maxMBSortInHeap,
@ -204,7 +206,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
if (readerFieldInfo != null && readerFieldInfo.getPointDataDimensionCount() > 0) {
BKDReader bkdReader = reader60.readers.get(readerFieldInfo.number);
if (bkdReader != null) {
bkdReaders.add(bkdReader);

2
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
View File

@ -134,7 +134,7 @@ final class PerFieldMergeState {
hasNorms |= fi.hasNorms();
hasDocValues |= fi.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= fi.hasPayloads();
hasPointValues |= (fi.getPointDimensionCount() != 0);
hasPointValues |= (fi.getPointDataDimensionCount() != 0);
}
}

4
lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java
View File

@ -124,8 +124,8 @@ public final class BinaryPoint extends Field {
/** Expert API */
public BinaryPoint(String name, byte[] packedPoint, IndexableFieldType type) {
super(name, packedPoint, type);
if (packedPoint.length != type.pointDimensionCount() * type.pointNumBytes()) {
throw new IllegalArgumentException("packedPoint is length=" + packedPoint.length + " but type.pointDimensionCount()=" + type.pointDimensionCount() + " and type.pointNumBytes()=" + type.pointNumBytes());
if (packedPoint.length != type.pointDataDimensionCount() * type.pointNumBytes()) {
throw new IllegalArgumentException("packedPoint is length=" + packedPoint.length + " but type.pointDimensionCount()=" + type.pointDataDimensionCount() + " and type.pointNumBytes()=" + type.pointNumBytes());
}
}

10
lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
View File

@ -85,8 +85,8 @@ public final class DoublePoint extends Field {
/** Change the values of this field */
public void setDoubleValues(double... point) {
if (type.pointDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
if (type.pointDataDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@ -98,8 +98,8 @@ public final class DoublePoint extends Field {
@Override
public Number numericValue() {
if (type.pointDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
if (type.pointDataDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Double.BYTES;
@ -142,7 +142,7 @@ public final class DoublePoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}

10
lucene/core/src/java/org/apache/lucene/document/DoubleRange.java
View File

@ -77,8 +77,8 @@ public class DoubleRange extends Field {
*/
public void setRangeValues(double[] min, double[] max) {
checkArgs(min, max);
if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@ -148,7 +148,7 @@ public class DoubleRange extends Field {
* @return the decoded min value
*/
public double getMin(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@ -158,7 +158,7 @@ public class DoubleRange extends Field {
* @return the decoded max value
*/
public double getMax(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@ -244,7 +244,7 @@ public class DoubleRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}

68
lucene/core/src/java/org/apache/lucene/document/FieldType.java
View File

@ -38,7 +38,8 @@ public class FieldType implements IndexableFieldType {
private IndexOptions indexOptions = IndexOptions.NONE;
private boolean frozen;
private DocValuesType docValuesType = DocValuesType.NONE;
private int dimensionCount;
private int dataDimensionCount;
private int indexDimensionCount;
private int dimensionNumBytes;
/**
@ -54,7 +55,8 @@ public class FieldType implements IndexableFieldType {
this.omitNorms = ref.omitNorms();
this.indexOptions = ref.indexOptions();
this.docValuesType = ref.docValuesType();
this.dimensionCount = ref.pointDimensionCount();
this.dataDimensionCount = ref.pointDataDimensionCount();
this.indexDimensionCount = ref.pointIndexDimensionCount();
this.dimensionNumBytes = ref.pointNumBytes();
// Do not copy frozen!
}
@ -279,11 +281,24 @@ public class FieldType implements IndexableFieldType {
* Enables points indexing.
*/
public void setDimensions(int dimensionCount, int dimensionNumBytes) {
if (dimensionCount < 0) {
throw new IllegalArgumentException("dimensionCount must be >= 0; got " + dimensionCount);
this.setDimensions(dimensionCount, dimensionCount, dimensionNumBytes);
}
/**
* Enables points indexing with selectable dimension indexing.
*/
public void setDimensions(int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (dataDimensionCount < 0) {
throw new IllegalArgumentException("dataDimensionCount must be >= 0; got " + dataDimensionCount);
}
if (dimensionCount > PointValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException("dimensionCount must be <= " + PointValues.MAX_DIMENSIONS + "; got " + dimensionCount);
if (dataDimensionCount > PointValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException("dataDimensionCount must be <= " + PointValues.MAX_DIMENSIONS + "; got " + dataDimensionCount);
}
if (indexDimensionCount < 0) {
throw new IllegalArgumentException("indexDimensionCount must be >= 0; got " + indexDimensionCount);
}
if (indexDimensionCount > dataDimensionCount) {
throw new IllegalArgumentException("indexDimensionCount must be <= dataDimensionCount: " + dataDimensionCount + "; got " + indexDimensionCount);
}
if (dimensionNumBytes < 0) {
throw new IllegalArgumentException("dimensionNumBytes must be >= 0; got " + dimensionNumBytes);
@ -291,23 +306,34 @@ public class FieldType implements IndexableFieldType {
if (dimensionNumBytes > PointValues.MAX_NUM_BYTES) {
throw new IllegalArgumentException("dimensionNumBytes must be <= " + PointValues.MAX_NUM_BYTES + "; got " + dimensionNumBytes);
}
if (dimensionCount == 0) {
if (dimensionNumBytes != 0) {
throw new IllegalArgumentException("when dimensionCount is 0, dimensionNumBytes must 0; got " + dimensionNumBytes);
if (dataDimensionCount == 0) {
if (indexDimensionCount != 0) {
throw new IllegalArgumentException("when dataDimensionCount is 0, indexDimensionCount must be 0; got " + indexDimensionCount);
}
if (dimensionNumBytes != 0) {
throw new IllegalArgumentException("when dataDimensionCount is 0, dimensionNumBytes must be 0; got " + dimensionNumBytes);
}
} else if (indexDimensionCount == 0) {
throw new IllegalArgumentException("when dataDimensionCount is > 0, indexDimensionCount must be > 0; got " + indexDimensionCount);
} else if (dimensionNumBytes == 0) {
if (dimensionCount != 0) {
throw new IllegalArgumentException("when dimensionNumBytes is 0, dimensionCount must 0; got " + dimensionCount);
if (dataDimensionCount != 0) {
throw new IllegalArgumentException("when dimensionNumBytes is 0, dataDimensionCount must be 0; got " + dataDimensionCount);
}
}
this.dimensionCount = dimensionCount;
this.dataDimensionCount = dataDimensionCount;
this.indexDimensionCount = indexDimensionCount;
this.dimensionNumBytes = dimensionNumBytes;
}
@Override
public int pointDimensionCount() {
return dimensionCount;
public int pointDataDimensionCount() {
return dataDimensionCount;
}
@Override
public int pointIndexDimensionCount() {
return indexDimensionCount;
}
@Override
@ -349,12 +375,14 @@ public class FieldType implements IndexableFieldType {
result.append(indexOptions);
}
}
if (dimensionCount != 0) {
if (dataDimensionCount != 0) {
if (result.length() > 0) {
result.append(",");
}
result.append("pointDimensionCount=");
result.append(dimensionCount);
result.append("pointDataDimensionCount=");
result.append(dataDimensionCount);
result.append(",pointIndexDimensionCount=");
result.append(indexDimensionCount);
result.append(",pointNumBytes=");
result.append(dimensionNumBytes);
}
@ -399,7 +427,8 @@ public class FieldType implements IndexableFieldType {
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + dimensionCount;
result = prime * result + dataDimensionCount;
result = prime * result + indexDimensionCount;
result = prime * result + dimensionNumBytes;
result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode());
result = prime * result + indexOptions.hashCode();
@ -419,7 +448,8 @@ public class FieldType implements IndexableFieldType {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
FieldType other = (FieldType) obj;
if (dimensionCount != other.dimensionCount) return false;
if (dataDimensionCount != other.dataDimensionCount) return false;
if (indexDimensionCount != other.indexDimensionCount) return false;
if (dimensionNumBytes != other.dimensionNumBytes) return false;
if (docValuesType != other.docValuesType) return false;
if (indexOptions != other.indexOptions) return false;

10
lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
View File

@ -85,8 +85,8 @@ public final class FloatPoint extends Field {
/** Change the values of this field */
public void setFloatValues(float... point) {
if (type.pointDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
if (type.pointDataDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@ -98,8 +98,8 @@ public final class FloatPoint extends Field {
@Override
public Number numericValue() {
if (type.pointDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
if (type.pointDataDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Float.BYTES;
@ -142,7 +142,7 @@ public final class FloatPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}

10
lucene/core/src/java/org/apache/lucene/document/FloatRange.java
View File

@ -77,8 +77,8 @@ public class FloatRange extends Field {
*/
public void setRangeValues(float[] min, float[] max) {
checkArgs(min, max);
if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@ -148,7 +148,7 @@ public class FloatRange extends Field {
* @return the decoded min value
*/
public float getMin(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@ -158,7 +158,7 @@ public class FloatRange extends Field {
* @return the decoded max value
*/
public float getMax(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@ -244,7 +244,7 @@ public class FloatRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}

10
lucene/core/src/java/org/apache/lucene/document/IntPoint.java
View File

@ -59,8 +59,8 @@ public final class IntPoint extends Field {
/** Change the values of this field */
public void setIntValues(int... point) {
if (type.pointDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
if (type.pointDataDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@ -72,8 +72,8 @@ public final class IntPoint extends Field {
@Override
public Number numericValue() {
if (type.pointDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
if (type.pointDataDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Integer.BYTES;
@ -116,7 +116,7 @@ public final class IntPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}

10
lucene/core/src/java/org/apache/lucene/document/IntRange.java
View File

@ -77,8 +77,8 @@ public class IntRange extends Field {
*/
public void setRangeValues(int[] min, int[] max) {
checkArgs(min, max);
if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@ -148,7 +148,7 @@ public class IntRange extends Field {
* @return the decoded min value
*/
public int getMin(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@ -158,7 +158,7 @@ public class IntRange extends Field {
* @return the decoded max value
*/
public int getMax(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@ -244,7 +244,7 @@ public class IntRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}

6
lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
View File

@ -147,9 +147,9 @@ public class LatLonPoint extends Field {
/** helper: checks a fieldinfo and throws exception if its definitely not a LatLonPoint */
static void checkCompatible(FieldInfo fieldInfo) {
// point/dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
if (fieldInfo.getPointDimensionCount() != 0 && fieldInfo.getPointDimensionCount() != TYPE.pointDimensionCount()) {
throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with numDims=" + fieldInfo.getPointDimensionCount() +
" but this point type has numDims=" + TYPE.pointDimensionCount() +
if (fieldInfo.getPointDataDimensionCount() != 0 && fieldInfo.getPointDataDimensionCount() != TYPE.pointDataDimensionCount()) {
throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with numDims=" + fieldInfo.getPointDataDimensionCount() +
" but this point type has numDims=" + TYPE.pointDataDimensionCount() +
", is the field really a LatLonPoint?");
}
if (fieldInfo.getPointNumBytes() != 0 && fieldInfo.getPointNumBytes() != TYPE.pointNumBytes()) {

10
lucene/core/src/java/org/apache/lucene/document/LongPoint.java
View File

@ -62,8 +62,8 @@ public final class LongPoint extends Field {
/** Change the values of this field */
public void setLongValues(long... point) {
if (type.pointDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
if (type.pointDataDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@ -75,8 +75,8 @@ public final class LongPoint extends Field {
@Override
public Number numericValue() {
if (type.pointDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
if (type.pointDataDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Long.BYTES;
@ -119,7 +119,7 @@ public final class LongPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}

10
lucene/core/src/java/org/apache/lucene/document/LongRange.java
View File

@ -77,8 +77,8 @@ public class LongRange extends Field {
*/
public void setRangeValues(long[] min, long[] max) {
checkArgs(min, max);
if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@ -146,7 +146,7 @@ public class LongRange extends Field {
* @return the decoded min value
*/
public long getMin(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@ -156,7 +156,7 @@ public class LongRange extends Field {
* @return the decoded max value
*/
public long getMax(int dimension) {
FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@ -242,7 +242,7 @@ public class LongRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}

4
lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
View File

@ -255,9 +255,9 @@ abstract class RangeFieldQuery extends Query {
/** Check indexed field info against the provided query data. */
private void checkFieldInfo(FieldInfo fieldInfo) {
if (fieldInfo.getPointDimensionCount()/2 != numDims) {
if (fieldInfo.getPointDataDimensionCount()/2 != numDims) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims="
+ fieldInfo.getPointDimensionCount()/2 + " but this query has numDims=" + numDims);
+ fieldInfo.getPointDataDimensionCount()/2 + " but this query has numDims=" + numDims);
}
}

37
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -1900,7 +1900,7 @@ public final class CheckIndex implements Closeable {
throw new RuntimeException("there are fields with points, but reader.getPointsReader() is null");
}
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.getPointDimensionCount() > 0) {
if (fieldInfo.getPointDataDimensionCount() > 0) {
PointValues values = pointsReader.getValues(fieldInfo.name);
if (values == null) {
continue;
@ -1970,7 +1970,9 @@ public final class CheckIndex implements Closeable {
private final byte[] globalMinPackedValue;
private final byte[] globalMaxPackedValue;
private final int packedBytesCount;
private final int numDims;
private final int packedIndexBytesCount;
private final int numDataDims;
private final int numIndexDims;
private final int bytesPerDim;
private final String fieldName;
@ -1978,14 +1980,16 @@ public final class CheckIndex implements Closeable {
public VerifyPointsVisitor(String fieldName, int maxDoc, PointValues values) throws IOException {
this.maxDoc = maxDoc;
this.fieldName = fieldName;
numDims = values.getNumDimensions();
numDataDims = values.getNumDataDimensions();
numIndexDims = values.getNumIndexDimensions();
bytesPerDim = values.getBytesPerDimension();
packedBytesCount = numDims * bytesPerDim;
packedBytesCount = numDataDims * bytesPerDim;
packedIndexBytesCount = numIndexDims * bytesPerDim;
globalMinPackedValue = values.getMinPackedValue();
globalMaxPackedValue = values.getMaxPackedValue();
docsSeen = new FixedBitSet(maxDoc);
lastMinPackedValue = new byte[packedBytesCount];
lastMaxPackedValue = new byte[packedBytesCount];
lastMinPackedValue = new byte[packedIndexBytesCount];
lastMaxPackedValue = new byte[packedIndexBytesCount];
lastPackedValue = new byte[packedBytesCount];
if (values.getDocCount() > values.size()) {
@ -2000,14 +2004,14 @@ public final class CheckIndex implements Closeable {
if (values.size() != 0) {
throw new RuntimeException("getMinPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size());
}
} else if (globalMinPackedValue.length != packedBytesCount) {
} else if (globalMinPackedValue.length != packedIndexBytesCount) {
throw new RuntimeException("getMinPackedValue for field \"" + fieldName + "\" return length=" + globalMinPackedValue.length + " array, but should be " + packedBytesCount);
}
if (globalMaxPackedValue == null) {
if (values.size() != 0) {
throw new RuntimeException("getMaxPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size());
}
} else if (globalMaxPackedValue.length != packedBytesCount) {
} else if (globalMaxPackedValue.length != packedIndexBytesCount) {
throw new RuntimeException("getMaxPackedValue for field \"" + fieldName + "\" return length=" + globalMaxPackedValue.length + " array, but should be " + packedBytesCount);
}
}
@ -2033,7 +2037,7 @@ public final class CheckIndex implements Closeable {
pointCountSeen++;
docsSeen.set(docID);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim * dim;
// Compare to last cell:
@ -2048,9 +2052,10 @@ public final class CheckIndex implements Closeable {
}
}
// In the 1D case, PointValues must make a single in-order sweep through all values, and tie-break by
// In the 1D data case, PointValues must make a single in-order sweep through all values, and tie-break by
// increasing docID:
if (numDims == 1) {
// for data dimension > 1, leaves are sorted by the dimension with the lowest cardinality to improve block compression
if (numDataDims == 1) {
int cmp = FutureArrays.compareUnsigned(lastPackedValue, 0, bytesPerDim, packedValue, 0, bytesPerDim);
if (cmp > 0) {
throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for field=\"" + fieldName + "\", for docID=" + docID + " is out-of-order vs the previous document's value " + Arrays.toString(lastPackedValue));
@ -2067,11 +2072,11 @@ public final class CheckIndex implements Closeable {
@Override
public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
checkPackedValue("min packed value", minPackedValue, -1);
System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, packedBytesCount);
System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, packedIndexBytesCount);
checkPackedValue("max packed value", maxPackedValue, -1);
System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, packedBytesCount);
System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, packedIndexBytesCount);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim * dim;
if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) > 0) {
@ -2110,8 +2115,8 @@ public final class CheckIndex implements Closeable {
throw new RuntimeException(desc + " is null for docID=" + docID + " field=\"" + fieldName + "\"");
}
if (packedValue.length != packedBytesCount) {
throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + packedBytesCount + " for docID=" + docID + " field=\"" + fieldName + "\"");
if (packedValue.length != (docID < 0 ? packedIndexBytesCount : packedBytesCount)) {
throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + packedIndexBytesCount + " for docID=" + docID + " field=\"" + fieldName + "\"");
}
}
}

2
lucene/core/src/java/org/apache/lucene/index/CodecReader.java
View File

@ -194,7 +194,7 @@ public abstract class CodecReader extends LeafReader implements Accountable {
public final PointValues getPointValues(String field) throws IOException {
ensureOpen();
FieldInfo fi = getFieldInfos().fieldInfo(field);
if (fi == null || fi.getPointDimensionCount() == 0) {
if (fi == null || fi.getPointDataDimensionCount() == 0) {
// Field does not exist or does not index points
return null;
}

15
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -201,7 +201,7 @@ final class DefaultIndexingChain extends DocConsumer {
PerField perField = fieldHash[i];
while (perField != null) {
if (perField.pointValuesWriter != null) {
if (perField.fieldInfo.getPointDimensionCount() == 0) {
if (perField.fieldInfo.getPointDataDimensionCount() == 0) {
// BUG
throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has no points but wrote them");
}
@ -216,7 +216,7 @@ final class DefaultIndexingChain extends DocConsumer {
perField.pointValuesWriter.flush(state, sortMap, pointsWriter);
perField.pointValuesWriter = null;
} else if (perField.fieldInfo.getPointDimensionCount() != 0) {
} else if (perField.fieldInfo.getPointDataDimensionCount() != 0) {
// BUG
throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has points but did not write them");
}
@ -477,7 +477,7 @@ final class DefaultIndexingChain extends DocConsumer {
}
indexDocValue(fp, dvType, field);
}
if (fieldType.pointDimensionCount() != 0) {
if (fieldType.pointDataDimensionCount() != 0) {
if (fp == null) {
fp = getOrAddField(fieldName, fieldType, false);
}
@ -508,17 +508,18 @@ final class DefaultIndexingChain extends DocConsumer {
/** Called from processDocument to index one field's point */
private void indexPoint(PerField fp, IndexableField field) throws IOException {
int pointDimensionCount = field.fieldType().pointDimensionCount();
int pointDataDimensionCount = field.fieldType().pointDataDimensionCount();
int pointIndexDimensionCount = field.fieldType().pointIndexDimensionCount();
int dimensionNumBytes = field.fieldType().pointNumBytes();
// Record dimensions for this field; this setter will throw IllegalArgExc if
// the dimensions were already set to something different:
if (fp.fieldInfo.getPointDimensionCount() == 0) {
fieldInfos.globalFieldNumbers.setDimensions(fp.fieldInfo.number, fp.fieldInfo.name, pointDimensionCount, dimensionNumBytes);
if (fp.fieldInfo.getPointDataDimensionCount() == 0) {
fieldInfos.globalFieldNumbers.setDimensions(fp.fieldInfo.number, fp.fieldInfo.name, pointDataDimensionCount, pointIndexDimensionCount, dimensionNumBytes);
}
fp.fieldInfo.setPointDimensions(pointDimensionCount, dimensionNumBytes);
fp.fieldInfo.setPointDimensions(pointDataDimensionCount, pointIndexDimensionCount, dimensionNumBytes);
if (fp.pointValuesWriter == null) {
fp.pointValuesWriter = new PointValuesWriter(docWriter, fp.fieldInfo);

73
lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -50,7 +50,8 @@ public final class FieldInfo {
/** If both of these are positive it means this field indexed points
* (see {@link org.apache.lucene.codecs.PointsFormat}). */
private int pointDimensionCount;
private int pointDataDimensionCount;
private int pointIndexDimensionCount;
private int pointNumBytes;
// whether this field is used as the soft-deletes field
@ -63,7 +64,7 @@ public final class FieldInfo {
*/
public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms, boolean storePayloads,
IndexOptions indexOptions, DocValuesType docValues, long dvGen, Map<String,String> attributes,
int pointDimensionCount, int pointNumBytes, boolean softDeletesField) {
int pointDataDimensionCount, int pointIndexDimensionCount, int pointNumBytes, boolean softDeletesField) {
this.name = Objects.requireNonNull(name);
this.number = number;
this.docValuesType = Objects.requireNonNull(docValues, "DocValuesType must not be null (field: \"" + name + "\")");
@ -79,7 +80,8 @@ public final class FieldInfo {
}
this.dvGen = dvGen;
this.attributes = Objects.requireNonNull(attributes);
this.pointDimensionCount = pointDimensionCount;
this.pointDataDimensionCount = pointDataDimensionCount;
this.pointIndexDimensionCount = pointIndexDimensionCount;
this.pointNumBytes = pointNumBytes;
this.softDeletesField = softDeletesField;
assert checkConsistency();
@ -107,20 +109,28 @@ public final class FieldInfo {
}
}
if (pointDimensionCount < 0) {
throw new IllegalStateException("pointDimensionCount must be >= 0; got " + pointDimensionCount);
if (pointDataDimensionCount < 0) {
throw new IllegalStateException("pointDataDimensionCount must be >= 0; got " + pointDataDimensionCount);
}
if (pointIndexDimensionCount < 0) {
throw new IllegalStateException("pointIndexDimensionCount must be >= 0; got " + pointIndexDimensionCount);
}
if (pointNumBytes < 0) {
throw new IllegalStateException("pointNumBytes must be >= 0; got " + pointNumBytes);
}
if (pointDimensionCount != 0 && pointNumBytes == 0) {
throw new IllegalStateException("pointNumBytes must be > 0 when pointDimensionCount=" + pointDimensionCount);
if (pointDataDimensionCount != 0 && pointNumBytes == 0) {
throw new IllegalStateException("pointNumBytes must be > 0 when pointDataDimensionCount=" + pointDataDimensionCount);
}
if (pointNumBytes != 0 && pointDimensionCount == 0) {
throw new IllegalStateException("pointDimensionCount must be > 0 when pointNumBytes=" + pointNumBytes);
if (pointIndexDimensionCount != 0 && pointDataDimensionCount == 0) {
throw new IllegalStateException("pointIndexDimensionCount must be 0 when pointDataDimensionCount=0");
}
if (pointNumBytes != 0 && pointDataDimensionCount == 0) {
throw new IllegalStateException("pointDataDimensionCount must be > 0 when pointNumBytes=" + pointNumBytes);
}
if (dvGen != -1 && docValuesType == DocValuesType.NONE) {
@ -132,7 +142,7 @@ public final class FieldInfo {
// should only be called by FieldInfos#addOrUpdate
void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
int dimensionCount, int dimensionNumBytes) {
int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (indexOptions == null) {
throw new NullPointerException("IndexOptions must not be null (field: \"" + name + "\")");
}
@ -145,11 +155,12 @@ public final class FieldInfo {
}
}
if (this.pointDimensionCount == 0 && dimensionCount != 0) {
this.pointDimensionCount = dimensionCount;
if (this.pointDataDimensionCount == 0 && dataDimensionCount != 0) {
this.pointDataDimensionCount = dataDimensionCount;
this.pointIndexDimensionCount = indexDimensionCount;
this.pointNumBytes = dimensionNumBytes;
} else if (dimensionCount != 0 && (this.pointDimensionCount != dimensionCount || this.pointNumBytes != dimensionNumBytes)) {
throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dimensionCount=" + this.pointDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dimensionCount=" + dimensionCount + ", numBytes=" + dimensionNumBytes);
} else if (dataDimensionCount != 0 && (this.pointDataDimensionCount != dataDimensionCount || this.pointIndexDimensionCount != indexDimensionCount || this.pointNumBytes != dimensionNumBytes)) {
throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dataDimensionCount=" + this.pointDataDimensionCount + ", indexDimensionCount=" + this.pointIndexDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dataDimensionCount=" + dataDimensionCount +", indexDimensionCount=" + indexDimensionCount + ", numBytes=" + dimensionNumBytes);
}
if (this.indexOptions != IndexOptions.NONE) { // if updated field data is not for indexing, leave the updates out
@ -170,12 +181,15 @@ public final class FieldInfo {
/** Record that this field is indexed with points, with the
* specified number of dimensions and bytes per dimension. */
public void setPointDimensions(int count, int numBytes) {
if (count <= 0) {
throw new IllegalArgumentException("point dimension count must be >= 0; got " + count + " for field=\"" + name + "\"");
public void setPointDimensions(int dataDimensionCount, int indexDimensionCount, int numBytes) {
if (dataDimensionCount <= 0) {
throw new IllegalArgumentException("point data dimension count must be >= 0; got " + dataDimensionCount + " for field=\"" + name + "\"");
}
if (count > PointValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException("point dimension count must be < PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + count + " for field=\"" + name + "\"");
if (dataDimensionCount > PointValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException("point data dimension count must be < PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + dataDimensionCount + " for field=\"" + name + "\"");
}
if (indexDimensionCount > dataDimensionCount) {
throw new IllegalArgumentException("point index dimension count must be <= point data dimension count (= " + dataDimensionCount + "); got " + indexDimensionCount + " for field=\"" + name + "\"");
}
if (numBytes <= 0) {
throw new IllegalArgumentException("point numBytes must be >= 0; got " + numBytes + " for field=\"" + name + "\"");
@ -183,22 +197,31 @@ public final class FieldInfo {
if (numBytes > PointValues.MAX_NUM_BYTES) {
throw new IllegalArgumentException("point numBytes must be <= PointValues.MAX_NUM_BYTES (= " + PointValues.MAX_NUM_BYTES + "); got " + numBytes + " for field=\"" + name + "\"");
}
if (pointDimensionCount != 0 && pointDimensionCount != count) {
throw new IllegalArgumentException("cannot change point dimension count from " + pointDimensionCount + " to " + count + " for field=\"" + name + "\"");
if (pointDataDimensionCount != 0 && pointDataDimensionCount != dataDimensionCount) {
throw new IllegalArgumentException("cannot change point data dimension count from " + pointDataDimensionCount + " to " + dataDimensionCount + " for field=\"" + name + "\"");
}
if (pointIndexDimensionCount != 0 && pointIndexDimensionCount != indexDimensionCount) {
throw new IllegalArgumentException("cannot change point index dimension count from " + pointIndexDimensionCount + " to " + indexDimensionCount + " for field=\"" + name + "\"");
}
if (pointNumBytes != 0 && pointNumBytes != numBytes) {
throw new IllegalArgumentException("cannot change point numBytes from " + pointNumBytes + " to " + numBytes + " for field=\"" + name + "\"");
}
pointDimensionCount = count;
pointDataDimensionCount = dataDimensionCount;
pointIndexDimensionCount = indexDimensionCount;
pointNumBytes = numBytes;
assert checkConsistency();
}
/** Return point dimension count */
public int getPointDimensionCount() {
return pointDimensionCount;
/** Return point data dimension count */
public int getPointDataDimensionCount() {
return pointDataDimensionCount;
}
/** Return point data dimension count */
public int getPointIndexDimensionCount() {
return pointIndexDimensionCount;
}
/** Return number of bytes per dimension */

61
lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -93,7 +93,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
hasNorms |= info.hasNorms();
hasDocValues |= info.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= info.hasPayloads();
hasPointValues |= (info.getPointDimensionCount() != 0);
hasPointValues |= (info.getPointDataDimensionCount() != 0);
if (info.isSoftDeletesField()) {
if (softDeletesField != null && softDeletesField.equals(info.name) == false) {
throw new IllegalArgumentException("multiple soft-deletes fields [" + info.name + ", " + softDeletesField + "]");
@ -210,11 +210,13 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
static final class FieldDimensions {
public final int dimensionCount;
public final int dataDimensionCount;
public final int indexDimensionCount;
public final int dimensionNumBytes;
public FieldDimensions(int dimensionCount, int dimensionNumBytes) {
this.dimensionCount = dimensionCount;
public FieldDimensions(int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
this.dataDimensionCount = dataDimensionCount;
this.indexDimensionCount = indexDimensionCount;
this.dimensionNumBytes = dimensionNumBytes;
}
}
@ -254,7 +256,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
* number assigned if possible otherwise the first unassigned field number
* is used as the field number.
*/
synchronized int addOrGet(String fieldName, int preferredFieldNumber, IndexOptions indexOptions, DocValuesType dvType, int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
synchronized int addOrGet(String fieldName, int preferredFieldNumber, IndexOptions indexOptions, DocValuesType dvType, int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
if (indexOptions != IndexOptions.NONE) {
IndexOptions currentOpts = this.indexOptions.get(fieldName);
if (currentOpts == null) {
@ -271,17 +273,20 @@ public class FieldInfos implements Iterable<FieldInfo> {
throw new IllegalArgumentException("cannot change DocValues type from " + currentDVType + " to " + dvType + " for field \"" + fieldName + "\"");
}
}
if (dimensionCount != 0) {
if (dataDimensionCount != 0) {
FieldDimensions dims = dimensions.get(fieldName);
if (dims != null) {
if (dims.dimensionCount != dimensionCount) {
throw new IllegalArgumentException("cannot change point dimension count from " + dims.dimensionCount + " to " + dimensionCount + " for field=\"" + fieldName + "\"");
if (dims.dataDimensionCount != dataDimensionCount) {
throw new IllegalArgumentException("cannot change point data dimension count from " + dims.dataDimensionCount + " to " + dataDimensionCount + " for field=\"" + fieldName + "\"");
}
if (dims.indexDimensionCount != indexDimensionCount) {
throw new IllegalArgumentException("cannot change point index dimension count from " + dims.indexDimensionCount + " to " + indexDimensionCount + " for field=\"" + fieldName + "\"");
}
if (dims.dimensionNumBytes != dimensionNumBytes) {
throw new IllegalArgumentException("cannot change point numBytes from " + dims.dimensionNumBytes + " to " + dimensionNumBytes + " for field=\"" + fieldName + "\"");
}
} else {
dimensions.put(fieldName, new FieldDimensions(dimensionCount, dimensionNumBytes));
dimensions.put(fieldName, new FieldDimensions(dataDimensionCount, indexDimensionCount, dimensionNumBytes));
}
}
Integer fieldNumber = nameToNumber.get(fieldName);
@ -341,7 +346,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
}
synchronized void verifyConsistentDimensions(Integer number, String name, int dimensionCount, int dimensionNumBytes) {
synchronized void verifyConsistentDimensions(Integer number, String name, int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (name.equals(numberToName.get(number)) == false) {
throw new IllegalArgumentException("field number " + number + " is already mapped to field name \"" + numberToName.get(number) + "\", not \"" + name + "\"");
}
@ -350,8 +355,11 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
FieldDimensions dim = dimensions.get(name);
if (dim != null) {
if (dim.dimensionCount != dimensionCount) {
throw new IllegalArgumentException("cannot change point dimension count from " + dim.dimensionCount + " to " + dimensionCount + " for field=\"" + name + "\"");
if (dim.dataDimensionCount != dataDimensionCount) {
throw new IllegalArgumentException("cannot change point data dimension count from " + dim.dataDimensionCount + " to " + dataDimensionCount + " for field=\"" + name + "\"");
}
if (dim.indexDimensionCount != indexDimensionCount) {
throw new IllegalArgumentException("cannot change point index dimension count from " + dim.indexDimensionCount + " to " + indexDimensionCount + " for field=\"" + name + "\"");
}
if (dim.dimensionNumBytes != dimensionNumBytes) {
throw new IllegalArgumentException("cannot change point numBytes from " + dim.dimensionNumBytes + " to " + dimensionNumBytes + " for field=\"" + name + "\"");
@ -395,15 +403,18 @@ public class FieldInfos implements Iterable<FieldInfo> {
docValuesType.put(name, dvType);
}
synchronized void setDimensions(int number, String name, int dimensionCount, int dimensionNumBytes) {
synchronized void setDimensions(int number, String name, int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (dimensionNumBytes > PointValues.MAX_NUM_BYTES) {
throw new IllegalArgumentException("dimension numBytes must be <= PointValues.MAX_NUM_BYTES (= " + PointValues.MAX_NUM_BYTES + "); got " + dimensionNumBytes + " for field=\"" + name + "\"");
}
if (dimensionCount > PointValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException("pointDimensionCount must be <= PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + dimensionCount + " for field=\"" + name + "\"");
if (dataDimensionCount > PointValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException("pointDataDimensionCount must be <= PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + dataDimensionCount + " for field=\"" + name + "\"");
}
verifyConsistentDimensions(number, name, dimensionCount, dimensionNumBytes);
dimensions.put(name, new FieldDimensions(dimensionCount, dimensionNumBytes));
if (indexDimensionCount > dataDimensionCount) {
throw new IllegalArgumentException("pointIndexDimensionCount must be <= pointDataDimensionCount (= " + dataDimensionCount + "); got " + indexDimensionCount + " for field=\"" + name + "\"");
}
verifyConsistentDimensions(number, name, dataDimensionCount, indexDimensionCount, dimensionNumBytes);
dimensions.put(name, new FieldDimensions(dataDimensionCount, indexDimensionCount, dimensionNumBytes));
}
}
@ -438,8 +449,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
// before then we'll get the same name and number,
// else we'll allocate a new one:
final boolean isSoftDeletesField = name.equals(globalFieldNumbers.softDeletesFieldName);
final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, IndexOptions.NONE, DocValuesType.NONE, 0, 0, isSoftDeletesField);
fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0, isSoftDeletesField);
final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, IndexOptions.NONE, DocValuesType.NONE, 0, 0, 0, isSoftDeletesField);
fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0, 0, isSoftDeletesField);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE);
byName.put(fi.name, fi);
@ -452,7 +463,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
DocValuesType docValues, long dvGen,
int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes,
boolean isSoftDeletesField) {
assert assertNotFinished();
if (docValues == null) {
throw new NullPointerException("DocValuesType must not be null");
@ -464,13 +476,13 @@ public class FieldInfos implements Iterable<FieldInfo> {
// number for this field. If the field was seen
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, indexOptions, docValues, dimensionCount, dimensionNumBytes, isSoftDeletesField);
fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes, isSoftDeletesField);
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, indexOptions, docValues, dataDimensionCount, indexDimensionCount, dimensionNumBytes, isSoftDeletesField);
fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dataDimensionCount, indexDimensionCount, dimensionNumBytes, isSoftDeletesField);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
} else {
fi.update(storeTermVector, omitNorms, storePayloads, indexOptions, dimensionCount, dimensionNumBytes);
fi.update(storeTermVector, omitNorms, storePayloads, indexOptions, dataDimensionCount, indexDimensionCount, dimensionNumBytes);
if (docValues != DocValuesType.NONE) {
// Only pay the synchronization cost if fi does not already have a DVType
@ -498,7 +510,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType(), dvGen,
fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(),
fi.isSoftDeletesField());
}
public FieldInfo fieldInfo(String fieldName) {

6
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -991,7 +991,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
for(SegmentCommitInfo info : segmentInfos) {
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
map.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
map.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
}
@ -1813,7 +1813,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
if (globalFieldNumberMap.contains(f.name(), dvType) == false) {
// if this field doesn't exists we try to add it. if it exists and the DV type doesn't match we
// get a consistent error message as if you try to do that during an indexing operation.
globalFieldNumberMap.addOrGet(f.name(), -1, IndexOptions.NONE, dvType, 0, 0, f.name().equals(config.softDeletesField));
globalFieldNumberMap.addOrGet(f.name(), -1, IndexOptions.NONE, dvType, 0, 0, 0, f.name().equals(config.softDeletesField));
assert globalFieldNumberMap.contains(f.name(), dvType);
}
if (config.getIndexSortFields().contains(f.name())) {
@ -2849,7 +2849,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
// This will throw exceptions if any of the incoming fields have an illegal schema change:
globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
infos.add(copySegmentAsIs(info, newSegName, context));
}

9
lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
View File

@ -98,9 +98,14 @@ public interface IndexableFieldType {
public DocValuesType docValuesType();
/**
* If this is positive, the field is indexed as a point.
* If this is positive (representing the number of point data dimensions), the field is indexed as a point.
*/
public int pointDimensionCount();
public int pointDataDimensionCount();
/**
* The number of dimensions used for the index key
*/
public int pointIndexDimensionCount();
/**
* The number of bytes in each dimension's values.

11
lucene/core/src/java/org/apache/lucene/index/PointValues.java
View File

@ -136,7 +136,7 @@ public abstract class PointValues {
if (minValue == null) {
minValue = leafMinValue.clone();
} else {
final int numDimensions = values.getNumDimensions();
final int numDimensions = values.getNumIndexDimensions();
final int numBytesPerDimension = values.getBytesPerDimension();
for (int i = 0; i < numDimensions; ++i) {
int offset = i * numBytesPerDimension;
@ -167,7 +167,7 @@ public abstract class PointValues {
if (maxValue == null) {
maxValue = leafMaxValue.clone();
} else {
final int numDimensions = values.getNumDimensions();
final int numDimensions = values.getNumIndexDimensions();
final int numBytesPerDimension = values.getBytesPerDimension();
for (int i = 0; i < numDimensions; ++i) {
int offset = i * numBytesPerDimension;
@ -233,8 +233,11 @@ public abstract class PointValues {
/** Returns maximum value for each dimension, packed, or null if {@link #size} is <code>0</code> */
public abstract byte[] getMaxPackedValue() throws IOException;
/** Returns how many dimensions were indexed */
public abstract int getNumDimensions() throws IOException;
/** Returns how many data dimensions are represented in the values */
public abstract int getNumDataDimensions() throws IOException;
/** Returns how many dimensions are used for the index */
public abstract int getNumIndexDimensions() throws IOException;
/** Returns the number of bytes per dimension */
public abstract int getBytesPerDimension() throws IOException;

20
lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
View File

@ -43,7 +43,7 @@ class PointValuesWriter {
this.bytes = new ByteBlockPool(docWriter.byteBlockAllocator);
docIDs = new int[16];
iwBytesUsed.addAndGet(16 * Integer.BYTES);
packedBytesLength = fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes();
packedBytesLength = fieldInfo.getPointDataDimensionCount() * fieldInfo.getPointNumBytes();
}
// TODO: if exactly the same value is added to exactly the same doc, should we dedup?
@ -52,7 +52,7 @@ class PointValuesWriter {
throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value must not be null");
}
if (value.length != packedBytesLength) {
throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()));
throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDataDimensionCount() * fieldInfo.getPointNumBytes()));
}
if (docIDs.length == numPoints) {
@ -106,7 +106,12 @@ class PointValuesWriter {
}
@Override
public int getNumDimensions() {
public int getNumDataDimensions() {
throw new UnsupportedOperationException();
}
@Override
public int getNumIndexDimensions() {
throw new UnsupportedOperationException();
}
@ -229,8 +234,13 @@ class PointValuesWriter {
}
@Override
public int getNumDimensions() throws IOException {
return in.getNumDimensions();
public int getNumDataDimensions() throws IOException {
return in.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
return in.getNumIndexDimensions();
}
@Override

9
lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
View File

@ -344,8 +344,13 @@ class SortingLeafReader extends FilterLeafReader {
}
@Override
public int getNumDimensions() throws IOException {
return in.getNumDimensions();
public int getNumDataDimensions() throws IOException {
return in.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
return in.getNumIndexDimensions();
}
@Override

4
lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java
View File

@ -123,8 +123,8 @@ public abstract class PointInSetQuery extends Query {
return null;
}
if (values.getNumDimensions() != numDims) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + values.getNumDimensions() + " but this query has numDims=" + numDims);
if (values.getNumIndexDimensions() != numDims) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numIndexDims=" + values.getNumIndexDimensions() + " but this query has numIndexDims=" + numDims);
}
if (values.getBytesPerDimension() != bytesPerDim) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + " but this query has bytesPerDim=" + bytesPerDim);

4
lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
View File

@ -234,8 +234,8 @@ public abstract class PointRangeQuery extends Query {
return null;
}
if (values.getNumDimensions() != numDims) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + values.getNumDimensions() + " but this query has numDims=" + numDims);
if (values.getNumIndexDimensions() != numDims) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numIndexDimensions=" + values.getNumIndexDimensions() + " but this query has numDims=" + numDims);
}
if (bytesPerDim != values.getBytesPerDimension()) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + " but this query has bytesPerDim=" + bytesPerDim);

140
lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
View File

@ -35,7 +35,8 @@ import org.apache.lucene.util.MathUtil;
public final class BKDReader extends PointValues implements Accountable {
// Packed array of byte[] holding all split values in the full binary tree:
final int leafNodeOffset;
final int numDims;
final int numDataDims;
final int numIndexDims;
final int bytesPerDim;
final int numLeaves;
final IndexInput in;
@ -46,29 +47,36 @@ public final class BKDReader extends PointValues implements Accountable {
final int docCount;
final int version;
protected final int packedBytesLength;
protected final int packedIndexBytesLength;
final byte[] packedIndex;
/** Caller must pre-seek the provided {@link IndexInput} to the index location that {@link BKDWriter#finish} returned */
public BKDReader(IndexInput in) throws IOException {
version = CodecUtil.checkHeader(in, BKDWriter.CODEC_NAME, BKDWriter.VERSION_START, BKDWriter.VERSION_CURRENT);
numDims = in.readVInt();
numDataDims = in.readVInt();
if (version >= BKDWriter.VERSION_SELECTIVE_INDEXING) {
numIndexDims = in.readVInt();
} else {
numIndexDims = numDataDims;
}
maxPointsInLeafNode = in.readVInt();
bytesPerDim = in.readVInt();
packedBytesLength = numDims * bytesPerDim;
packedBytesLength = numDataDims * bytesPerDim;
packedIndexBytesLength = numIndexDims * bytesPerDim;
// Read index:
numLeaves = in.readVInt();
assert numLeaves > 0;
leafNodeOffset = numLeaves;
minPackedValue = new byte[packedBytesLength];
maxPackedValue = new byte[packedBytesLength];
minPackedValue = new byte[packedIndexBytesLength];
maxPackedValue = new byte[packedIndexBytesLength];
in.readBytes(minPackedValue, 0, packedBytesLength);
in.readBytes(maxPackedValue, 0, packedBytesLength);
in.readBytes(minPackedValue, 0, packedIndexBytesLength);
in.readBytes(maxPackedValue, 0, packedIndexBytesLength);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(minPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, maxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) > 0) {
throw new CorruptIndexException("minPackedValue " + new BytesRef(minPackedValue) + " is > maxPackedValue " + new BytesRef(maxPackedValue) + " for dim=" + dim, in);
}
@ -122,16 +130,16 @@ public final class BKDReader extends PointValues implements Accountable {
splitPackedValueStack = new byte[treeDepth+1][];
nodeID = 1;
level = 1;
splitPackedValueStack[level] = new byte[packedBytesLength];
splitPackedValueStack[level] = new byte[packedIndexBytesLength];
leafBlockFPStack = new long[treeDepth+1];
leftNodePositions = new int[treeDepth+1];
rightNodePositions = new int[treeDepth+1];
splitValuesStack = new byte[treeDepth+1][];
splitDims = new int[treeDepth+1];
negativeDeltas = new boolean[numDims*(treeDepth+1)];
negativeDeltas = new boolean[numIndexDims*(treeDepth+1)];
in = new ByteArrayDataInput(packedIndex);
splitValuesStack[0] = new byte[packedBytesLength];
splitValuesStack[0] = new byte[packedIndexBytesLength];
readNodeData(false);
scratch = new BytesRef();
scratch.length = bytesPerDim;
@ -142,11 +150,11 @@ public final class BKDReader extends PointValues implements Accountable {
nodeID *= 2;
level++;
if (splitPackedValueStack[level] == null) {
splitPackedValueStack[level] = new byte[packedBytesLength];
splitPackedValueStack[level] = new byte[packedIndexBytesLength];
}
System.arraycopy(negativeDeltas, (level-1)*numDims, negativeDeltas, level*numDims, numDims);
System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
assert splitDim != -1;
negativeDeltas[level*numDims+splitDim] = true;
negativeDeltas[level*numIndexDims+splitDim] = true;
in.setPosition(nodePosition);
readNodeData(true);
}
@ -162,7 +170,7 @@ public final class BKDReader extends PointValues implements Accountable {
index.leftNodePositions[level] = leftNodePositions[level];
index.rightNodePositions[level] = rightNodePositions[level];
index.splitValuesStack[index.level] = splitValuesStack[index.level].clone();
System.arraycopy(negativeDeltas, level*numDims, index.negativeDeltas, level*numDims, numDims);
System.arraycopy(negativeDeltas, level*numIndexDims, index.negativeDeltas, level*numIndexDims, numIndexDims);
index.splitDims[level] = splitDims[level];
return index;
}
@ -172,11 +180,11 @@ public final class BKDReader extends PointValues implements Accountable {
nodeID = nodeID * 2 + 1;
level++;
if (splitPackedValueStack[level] == null) {
splitPackedValueStack[level] = new byte[packedBytesLength];
splitPackedValueStack[level] = new byte[packedIndexBytesLength];
}
System.arraycopy(negativeDeltas, (level-1)*numDims, negativeDeltas, level*numDims, numDims);
System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
assert splitDim != -1;
negativeDeltas[level*numDims+splitDim] = false;
negativeDeltas[level*numIndexDims+splitDim] = false;
in.setPosition(nodePosition);
readNodeData(false);
}
@ -276,19 +284,19 @@ public final class BKDReader extends PointValues implements Accountable {
// read split dim, prefix, firstDiffByteDelta encoded as int:
int code = in.readVInt();
splitDim = code % numDims;
splitDim = code % numIndexDims;
splitDims[level] = splitDim;
code /= numDims;
code /= numIndexDims;
int prefix = code % (1+bytesPerDim);
int suffix = bytesPerDim - prefix;
if (splitValuesStack[level] == null) {
splitValuesStack[level] = new byte[packedBytesLength];
splitValuesStack[level] = new byte[packedIndexBytesLength];
}
System.arraycopy(splitValuesStack[level-1], 0, splitValuesStack[level], 0, packedBytesLength);
System.arraycopy(splitValuesStack[level-1], 0, splitValuesStack[level], 0, packedIndexBytesLength);
if (suffix > 0) {
int firstDiffByteDelta = code / (1+bytesPerDim);
if (negativeDeltas[level*numDims + splitDim]) {
if (negativeDeltas[level*numIndexDims + splitDim]) {
firstDiffByteDelta = -firstDiffByteDelta;
}
int oldByte = splitValuesStack[level][splitDim*bytesPerDim+prefix] & 0xFF;
@ -325,7 +333,7 @@ public final class BKDReader extends PointValues implements Accountable {
public static final class IntersectState {
final IndexInput in;
final int[] scratchDocIDs;
final byte[] scratchPackedValue1, scratchPackedValue2;
final byte[] scratchDataPackedValue, scratchMinIndexPackedValue, scratchMaxIndexPackedValue;
final int[] commonPrefixLengths;
final IntersectVisitor visitor;
@ -333,6 +341,7 @@ public final class BKDReader extends PointValues implements Accountable {
public IntersectState(IndexInput in, int numDims,
int packedBytesLength,
int packedIndexBytesLength,
int maxPointsInLeafNode,
IntersectVisitor visitor,
IndexTree indexVisitor) {
@ -340,8 +349,9 @@ public final class BKDReader extends PointValues implements Accountable {
this.visitor = visitor;
this.commonPrefixLengths = new int[numDims];
this.scratchDocIDs = new int[maxPointsInLeafNode];
this.scratchPackedValue1 = new byte[packedBytesLength];
this.scratchPackedValue2 = new byte[packedBytesLength];
this.scratchDataPackedValue = new byte[packedBytesLength];
this.scratchMinIndexPackedValue = new byte[packedIndexBytesLength];
this.scratchMaxIndexPackedValue = new byte[packedIndexBytesLength];
this.index = indexVisitor;
}
}
@ -389,8 +399,9 @@ public final class BKDReader extends PointValues implements Accountable {
/** Create a new {@link IntersectState} */
public IntersectState getIntersectState(IntersectVisitor visitor) {
IndexTree index = new IndexTree();
return new IntersectState(in.clone(), numDims,
return new IntersectState(in.clone(), numDataDims,
packedBytesLength,
packedIndexBytesLength,
maxPointsInLeafNode,
visitor,
index);
@ -403,7 +414,7 @@ public final class BKDReader extends PointValues implements Accountable {
int count = readDocIDs(state.in, index.getLeafBlockFP(), state.scratchDocIDs);
// Again, this time reading values and checking with the visitor
visitDocValues(state.commonPrefixLengths, state.scratchPackedValue1, state.scratchPackedValue2, state.in, state.scratchDocIDs, count, state.visitor);
visitDocValues(state.commonPrefixLengths, state.scratchDataPackedValue, state.scratchMinIndexPackedValue, state.scratchMaxIndexPackedValue, state.in, state.scratchDocIDs, count, state.visitor);
}
private void visitDocIDs(IndexInput in, long blockFP, IntersectVisitor visitor) throws IOException {
@ -428,17 +439,19 @@ public final class BKDReader extends PointValues implements Accountable {
return count;
}
void visitDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue1, byte[] scratchPackedValue2, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
void visitDocValues(int[] commonPrefixLengths, byte[] scratchDataPackedValue, byte[] scratchMinIndexPackedValue, byte[] scratchMaxIndexPackedValue,
IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
readCommonPrefixes(commonPrefixLengths, scratchPackedValue1, in);
readCommonPrefixes(commonPrefixLengths, scratchDataPackedValue, in);
if (numDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
byte[] minPackedValue = scratchPackedValue1;
byte[] maxPackedValue = scratchPackedValue2;
if (numIndexDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
byte[] minPackedValue = scratchMinIndexPackedValue;
System.arraycopy(scratchDataPackedValue, 0, minPackedValue, 0, packedIndexBytesLength);
byte[] maxPackedValue = scratchMaxIndexPackedValue;
//Copy common prefixes before reading adjusted
// box
System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedBytesLength);
System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
readMinMax(commonPrefixLengths, minPackedValue, maxPackedValue, in);
// The index gives us range of values for each dimension, but the actual range of values
@ -467,24 +480,24 @@ public final class BKDReader extends PointValues implements Accountable {
int compressedDim = readCompressedDim(in);
if (compressedDim == -1) {
visitRawDocValues(commonPrefixLengths, scratchPackedValue1, in, docIDs, count, visitor);
visitRawDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor);
} else {
visitCompressedDocValues(commonPrefixLengths, scratchPackedValue1, in, docIDs, count, visitor, compressedDim);
visitCompressedDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor, compressedDim);
}
}
private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException {
for (int dim = 0; dim < numDims; dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(minPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
in.readBytes(maxPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
}
private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException {
for (int dim = 0; dim < numIndexDims; dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(minPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
in.readBytes(maxPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
}
}
// Just read suffixes for every dimension
private void visitRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
for (int i = 0; i < count; ++i) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
}
@ -502,7 +515,7 @@ public final class BKDReader extends PointValues implements Accountable {
scratchPackedValue[compressedByteOffset] = in.readByte();
final int runLen = Byte.toUnsignedInt(in.readByte());
for (int j = 0; j < runLen; ++j) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
}
@ -517,14 +530,14 @@ public final class BKDReader extends PointValues implements Accountable {
private int readCompressedDim(IndexInput in) throws IOException {
int compressedDim = in.readByte();
if (compressedDim < -1 || compressedDim >= numDims) {
if (compressedDim < -1 || compressedDim >= numDataDims) {
throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
}
return compressedDim;
}
private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = in.readVInt();
commonPrefixLengths[dim] = prefix;
if (prefix > 0) {
@ -561,15 +574,15 @@ public final class BKDReader extends PointValues implements Accountable {
int count = readDocIDs(state.in, state.index.getLeafBlockFP(), state.scratchDocIDs);
// Again, this time reading values and checking with the visitor
visitDocValues(state.commonPrefixLengths, state.scratchPackedValue1, state.scratchPackedValue2, state.in, state.scratchDocIDs, count, state.visitor);
visitDocValues(state.commonPrefixLengths, state.scratchDataPackedValue, state.scratchMinIndexPackedValue, state.scratchMaxIndexPackedValue, state.in, state.scratchDocIDs, count, state.visitor);
}
} else {
// Non-leaf node: recurse on the split left and right nodes
int splitDim = state.index.getSplitDim();
assert splitDim >= 0: "splitDim=" + splitDim;
assert splitDim < numDims;
assert splitDim >= 0: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
assert splitDim < numIndexDims: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
byte[] splitPackedValue = state.index.getSplitPackedValue();
BytesRef splitDimValue = state.index.getSplitDimValue();
@ -577,11 +590,11 @@ public final class BKDReader extends PointValues implements Accountable {
//System.out.println(" splitDimValue=" + splitDimValue + " splitDim=" + splitDim);
// make sure cellMin <= splitValue <= cellMax:
assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
// Recurse on left sub-tree:
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushLeft();
intersect(state, cellMinPacked, splitPackedValue);
@ -591,7 +604,7 @@ public final class BKDReader extends PointValues implements Accountable {
System.arraycopy(splitPackedValue, splitDim*bytesPerDim, splitDimValue.bytes, splitDimValue.offset, bytesPerDim);
// Recurse on right sub-tree:
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushRight();
intersect(state, splitPackedValue, cellMaxPacked);
@ -622,8 +635,8 @@ public final class BKDReader extends PointValues implements Accountable {
// Non-leaf node: recurse on the split left and right nodes
int splitDim = state.index.getSplitDim();
assert splitDim >= 0: "splitDim=" + splitDim;
assert splitDim < numDims;
assert splitDim >= 0: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
assert splitDim < numIndexDims: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
byte[] splitPackedValue = state.index.getSplitPackedValue();
BytesRef splitDimValue = state.index.getSplitDimValue();
@ -631,11 +644,11 @@ public final class BKDReader extends PointValues implements Accountable {
//System.out.println(" splitDimValue=" + splitDimValue + " splitDim=" + splitDim);
// make sure cellMin <= splitValue <= cellMax:
assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
// Recurse on left sub-tree:
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushLeft();
final long leftCost = estimatePointCount(state, cellMinPacked, splitPackedValue);
@ -645,7 +658,7 @@ public final class BKDReader extends PointValues implements Accountable {
System.arraycopy(splitPackedValue, splitDim*bytesPerDim, splitDimValue.bytes, splitDimValue.offset, bytesPerDim);
// Recurse on right sub-tree:
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushRight();
final long rightCost = estimatePointCount(state, splitPackedValue, cellMaxPacked);
@ -670,8 +683,13 @@ public final class BKDReader extends PointValues implements Accountable {
}
@Override
public int getNumDimensions() {
return numDims;
public int getNumDataDimensions() {
return numDataDims;
}
@Override
public int getNumIndexDimensions() {
return numIndexDims;
}
@Override

174
lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
View File

@ -86,7 +86,8 @@ public class BKDWriter implements Closeable {
public static final int VERSION_START = 4; // version used by Lucene 7.0
//public static final int VERSION_CURRENT = VERSION_START;
public static final int VERSION_LEAF_STORES_BOUNDS = 5;
public static final int VERSION_CURRENT = VERSION_LEAF_STORES_BOUNDS;
public static final int VERSION_SELECTIVE_INDEXING = 6;
public static final int VERSION_CURRENT = VERSION_SELECTIVE_INDEXING;
/** How many bytes each docs takes in the fixed-width offline format */
private final int bytesPerDoc;
@ -100,15 +101,21 @@ public class BKDWriter implements Closeable {
/** Maximum number of dimensions */
public static final int MAX_DIMS = 8;
/** How many dimensions we are indexing */
protected final int numDims;
/** How many dimensions we are storing at the leaf (data) nodes */
protected final int numDataDims;
/** How many dimensions we are indexing in the internal nodes */
protected final int numIndexDims;
/** How many bytes each value in each dimension takes. */
protected final int bytesPerDim;
/** numDims * bytesPerDim */
/** numDataDims * bytesPerDim */
protected final int packedBytesLength;
/** numIndexDims * bytesPerDim */
protected final int packedIndexBytesLength;
final TrackingDirectoryWrapper tempDir;
final String tempFileNamePrefix;
final double maxMBSortInHeap;
@ -154,37 +161,39 @@ public class BKDWriter implements Closeable {
private final int maxDoc;
public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException {
this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
this(maxDoc, tempDir, tempFileNamePrefix, numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
totalPointCount > Integer.MAX_VALUE, Math.max(1, (long) maxMBSortInHeap), OfflineSorter.MAX_TEMPFILES);
}
protected BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
protected BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount,
boolean singleValuePerDoc, boolean longOrds, long offlineSorterBufferMB, int offlineSorterMaxTempFiles) throws IOException {
verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
verifyParams(numDataDims, numIndexDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
this.tempDir = new TrackingDirectoryWrapper(tempDir);
this.tempFileNamePrefix = tempFileNamePrefix;
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.numDims = numDims;
this.numDataDims = numDataDims;
this.numIndexDims = numIndexDims;
this.bytesPerDim = bytesPerDim;
this.totalPointCount = totalPointCount;
this.maxDoc = maxDoc;
this.offlineSorterBufferMB = OfflineSorter.BufferSize.megabytes(offlineSorterBufferMB);
this.offlineSorterMaxTempFiles = offlineSorterMaxTempFiles;
docsSeen = new FixedBitSet(maxDoc);
packedBytesLength = numDims * bytesPerDim;
packedBytesLength = numDataDims * bytesPerDim;
packedIndexBytesLength = numIndexDims * bytesPerDim;
scratchDiff = new byte[bytesPerDim];
scratch1 = new byte[packedBytesLength];
scratch2 = new byte[packedBytesLength];
commonPrefixLengths = new int[numDims];
commonPrefixLengths = new int[numDataDims];
minPackedValue = new byte[packedBytesLength];
maxPackedValue = new byte[packedBytesLength];
minPackedValue = new byte[packedIndexBytesLength];
maxPackedValue = new byte[packedIndexBytesLength];
// If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes).
this.longOrds = longOrds;
@ -211,7 +220,7 @@ public class BKDWriter implements Closeable {
// bytes to points here. Each dimension has its own sorted partition, so
// we must divide by numDims as wel.
maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims));
maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDataDims));
// Finally, we must be able to hold at least the leaf node in heap during build:
if (maxPointsSortInHeap < maxPointsInLeafNode) {
@ -224,11 +233,14 @@ public class BKDWriter implements Closeable {
this.maxMBSortInHeap = maxMBSortInHeap;
}
public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
public static void verifyParams(int numDataDims, int numIndexDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
// We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
// remaining 4 bits for another purpose later
if (numDims < 1 || numDims > MAX_DIMS) {
throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
if (numDataDims < 1 || numDataDims > MAX_DIMS) {
throw new IllegalArgumentException("numDataDims must be 1 .. " + MAX_DIMS + " (got: " + numDataDims + ")");
}
if (numIndexDims < 1 || numIndexDims > numDataDims) {
throw new IllegalArgumentException("numIndexDims must be 1 .. " + numDataDims + " (got: " + numIndexDims + ")");
}
if (maxPointsInLeafNode <= 0) {
throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
@ -277,10 +289,10 @@ public class BKDWriter implements Closeable {
// TODO: we could specialize for the 1D case:
if (pointCount == 0) {
System.arraycopy(packedValue, 0, minPackedValue, 0, packedBytesLength);
System.arraycopy(packedValue, 0, maxPackedValue, 0, packedBytesLength);
System.arraycopy(packedValue, 0, minPackedValue, 0, packedIndexBytesLength);
System.arraycopy(packedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
} else {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(packedValue, offset, minPackedValue, offset, bytesPerDim);
@ -325,8 +337,9 @@ public class BKDWriter implements Closeable {
public MergeReader(BKDReader bkd, MergeState.DocMap docMap) throws IOException {
this.bkd = bkd;
state = new BKDReader.IntersectState(bkd.in.clone(),
bkd.numDims,
bkd.numDataDims,
bkd.packedBytesLength,
bkd.packedIndexBytesLength,
bkd.maxPointsInLeafNode,
null,
null);
@ -347,7 +360,7 @@ public class BKDWriter implements Closeable {
docsInBlock = bkd.readDocIDs(state.in, state.in.getFilePointer(), state.scratchDocIDs);
assert docsInBlock > 0;
docBlockUpto = 0;
bkd.visitDocValues(state.commonPrefixLengths, state.scratchPackedValue1, state.scratchPackedValue2, state.in, state.scratchDocIDs, docsInBlock, new IntersectVisitor() {
bkd.visitDocValues(state.commonPrefixLengths, state.scratchDataPackedValue, state.scratchMinIndexPackedValue, state.scratchMaxIndexPackedValue, state.in, state.scratchDocIDs, docsInBlock, new IntersectVisitor() {
int i = 0;
@Override
@ -385,7 +398,7 @@ public class BKDWriter implements Closeable {
if (mappedDocID != -1) {
// Not deleted!
docID = mappedDocID;
System.arraycopy(packedValues, index * bkd.packedBytesLength, state.scratchPackedValue1, 0, bkd.packedBytesLength);
System.arraycopy(packedValues, index * bkd.packedBytesLength, state.scratchDataPackedValue, 0, bkd.packedBytesLength);
return true;
}
}
@ -404,7 +417,7 @@ public class BKDWriter implements Closeable {
public boolean lessThan(MergeReader a, MergeReader b) {
assert a != b;
int cmp = FutureArrays.compareUnsigned(a.state.scratchPackedValue1, 0, bytesPerDim, b.state.scratchPackedValue1, 0, bytesPerDim);
int cmp = FutureArrays.compareUnsigned(a.state.scratchDataPackedValue, 0, bytesPerDim, b.state.scratchDataPackedValue, 0, bytesPerDim);
if (cmp < 0) {
return true;
} else if (cmp > 0) {
@ -422,7 +435,7 @@ public class BKDWriter implements Closeable {
* disk. This method does not use transient disk in order to reorder points.
*/
public long writeField(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
if (numDims == 1) {
if (numIndexDims == 1) {
return writeField1Dim(out, fieldName, reader);
} else {
return writeFieldNDims(out, fieldName, reader);
@ -465,7 +478,7 @@ public class BKDWriter implements Closeable {
Arrays.fill(maxPackedValue, (byte) 0);
for (int i = 0; i < Math.toIntExact(pointCount); ++i) {
values.getValue(i, scratchBytesRef1);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, scratchBytesRef1.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, minPackedValue, offset, bytesPerDim);
@ -478,12 +491,12 @@ public class BKDWriter implements Closeable {
docsSeen.set(values.getDocID(i));
}
final int[] parentSplits = new int[numDims];
final int[] parentSplits = new int[numIndexDims];
build(1, numLeaves, values, 0, Math.toIntExact(pointCount), out,
minPackedValue, maxPackedValue, parentSplits,
splitPackedValues, leafBlockFPs,
new int[maxPointsInLeafNode]);
assert Arrays.equals(parentSplits, new int[numDims]);
assert Arrays.equals(parentSplits, new int[numIndexDims]);
long indexFP = out.getFilePointer();
writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);
@ -493,7 +506,7 @@ public class BKDWriter implements Closeable {
/* In the 1D case, we can simply sort points in ascending order and use the
* same writing logic as we use at merge time. */
private long writeField1Dim(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
MutablePointsReaderUtils.sort(maxDoc, packedBytesLength, reader, 0, Math.toIntExact(reader.size()));
MutablePointsReaderUtils.sort(maxDoc, packedIndexBytesLength, reader, 0, Math.toIntExact(reader.size()));
final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
@ -546,7 +559,7 @@ public class BKDWriter implements Closeable {
MergeReader reader = queue.top();
// System.out.println("iter reader=" + reader);
oneDimWriter.add(reader.state.scratchPackedValue1, reader.docID);
oneDimWriter.add(reader.state.scratchDataPackedValue, reader.docID);
if (reader.next()) {
queue.updateTop();
@ -573,8 +586,8 @@ public class BKDWriter implements Closeable {
private int leafCount;
OneDimensionBKDWriter(IndexOutput out) {
if (numDims != 1) {
throw new UnsupportedOperationException("numDims must be 1 but got " + numDims);
if (numIndexDims != 1) {
throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
}
if (pointCount != 0) {
throw new IllegalStateException("cannot mix add and merge");
@ -651,9 +664,9 @@ public class BKDWriter implements Closeable {
private void writeLeafBlock() throws IOException {
assert leafCount != 0;
if (valueCount == 0) {
System.arraycopy(leafValues, 0, minPackedValue, 0, packedBytesLength);
System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);
}
System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedBytesLength);
System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedIndexBytesLength);
valueCount += leafCount;
@ -828,7 +841,7 @@ public class BKDWriter implements Closeable {
*/
private PointWriter sort(int dim) throws IOException {
assert dim >= 0 && dim < numDims;
assert dim >= 0 && dim < numDataDims;
if (heapPointWriter != null) {
@ -861,7 +874,7 @@ public class BKDWriter implements Closeable {
final int offset = bytesPerDim * dim;
Comparator<BytesRef> cmp;
if (dim == numDims - 1) {
if (dim == numDataDims - 1) {
// in that case the bytes for the dimension and for the doc id are contiguous,
// so we don't need a branch
cmp = new BytesRefComparator(bytesPerDim + Integer.BYTES) {
@ -946,7 +959,7 @@ public class BKDWriter implements Closeable {
}
LongBitSet ordBitSet;
if (numDims > 1) {
if (numDataDims > 1) {
if (singleValuePerDoc) {
ordBitSet = new LongBitSet(maxDoc);
} else {
@ -981,7 +994,7 @@ public class BKDWriter implements Closeable {
assert pointCount / numLeaves <= maxPointsInLeafNode: "pointCount=" + pointCount + " numLeaves=" + numLeaves + " maxPointsInLeafNode=" + maxPointsInLeafNode;
// Sort all docs once by each dimension:
PathSlice[] sortedPointWriters = new PathSlice[numDims];
PathSlice[] sortedPointWriters = new PathSlice[numDataDims];
// This is only used on exception; on normal code paths we close all files we opened:
List<Closeable> toCloseHeroically = new ArrayList<>();
@ -989,7 +1002,9 @@ public class BKDWriter implements Closeable {
boolean success = false;
try {
//long t0 = System.nanoTime();
for(int dim=0;dim<numDims;dim++) {
// even with selective indexing we create the sortedPointWriters so we can compress
// the leaf node data by common prefix
for(int dim=0;dim<numDataDims;dim++) {
sortedPointWriters[dim] = new PathSlice(sort(dim), 0, pointCount);
}
//long t1 = System.nanoTime();
@ -1003,7 +1018,7 @@ public class BKDWriter implements Closeable {
heapPointWriter = null;
}
final int[] parentSplits = new int[numDims];
final int[] parentSplits = new int[numIndexDims];
build(1, numLeaves, sortedPointWriters,
ordBitSet, out,
minPackedValue, maxPackedValue,
@ -1011,7 +1026,7 @@ public class BKDWriter implements Closeable {
splitPackedValues,
leafBlockFPs,
toCloseHeroically);
assert Arrays.equals(parentSplits, new int[numDims]);
assert Arrays.equals(parentSplits, new int[numIndexDims]);
for(PathSlice slice : sortedPointWriters) {
slice.writer.destroy();
@ -1046,7 +1061,7 @@ public class BKDWriter implements Closeable {
// Possibly rotate the leaf block FPs, if the index not fully balanced binary tree (only happens
// if it was created by OneDimensionBKDWriter). In this case the leaf nodes may straddle the two bottom
// levels of the binary tree:
if (numDims == 1 && numLeaves > 1) {
if (numIndexDims == 1 && numLeaves > 1) {
int levelCount = 2;
while (true) {
if (numLeaves >= levelCount && numLeaves <= 2*levelCount) {
@ -1072,9 +1087,9 @@ public class BKDWriter implements Closeable {
// This is the "file" we append the byte[] to:
List<byte[]> blocks = new ArrayList<>();
byte[] lastSplitValues = new byte[bytesPerDim * numDims];
byte[] lastSplitValues = new byte[bytesPerDim * numIndexDims];
//System.out.println("\npack index");
int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, 0l, blocks, 1, lastSplitValues, new boolean[numDims], false);
int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, 0l, blocks, 1, lastSplitValues, new boolean[numIndexDims], false);
// Compact the byte[] blocks into single byte index:
byte[] index = new byte[totalSize];
@ -1126,7 +1141,7 @@ public class BKDWriter implements Closeable {
if (isLeft == false) {
leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);
long delta = leftBlockFP - minBlockFP;
assert nodeID == 1 || delta > 0;
assert nodeID == 1 || delta > 0 : "expected nodeID=1 or delta > 0; got nodeID=" + nodeID + " and delta=" + delta;
writeBuffer.writeVLong(delta);
} else {
// The left tree's left most leaf block FP is always the minimal FP:
@ -1162,7 +1177,7 @@ public class BKDWriter implements Closeable {
}
// pack the prefix, splitDim and delta first diff byte into a single vInt:
int code = (firstDiffByteDelta * (1+bytesPerDim) + prefix) * numDims + splitDim;
int code = (firstDiffByteDelta * (1+bytesPerDim) + prefix) * numIndexDims + splitDim;
//System.out.println(" code=" + code);
//System.out.println(" splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));
@ -1248,14 +1263,15 @@ public class BKDWriter implements Closeable {
private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex) throws IOException {
CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
out.writeVInt(numDims);
out.writeVInt(numDataDims);
out.writeVInt(numIndexDims);
out.writeVInt(countPerLeaf);
out.writeVInt(bytesPerDim);
assert numLeaves > 0;
out.writeVInt(numLeaves);
out.writeBytes(minPackedValue, 0, packedBytesLength);
out.writeBytes(maxPackedValue, 0, packedBytesLength);
out.writeBytes(minPackedValue, 0, packedIndexBytesLength);
out.writeBytes(maxPackedValue, 0, packedIndexBytesLength);
out.writeVLong(pointCount);
out.writeVInt(docsSeen.cardinality());
@ -1275,7 +1291,7 @@ public class BKDWriter implements Closeable {
// all values in this block are equal
out.writeByte((byte) -1);
} else {
if (numDims != 1) {
if (numIndexDims != 1) {
writeActualBounds(out, commonPrefixLengths, count, packedValues);
}
assert commonPrefixLengths[sortedDim] < bytesPerDim;
@ -1298,7 +1314,7 @@ public class BKDWriter implements Closeable {
}
private void writeActualBounds(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {
for (int dim = 0; dim < numDims; ++dim) {
for (int dim = 0; dim < numIndexDims; ++dim) {
int commonPrefixLength = commonPrefixLengths[dim];
int suffixLength = bytesPerDim - commonPrefixLength;
if (suffixLength > 0) {
@ -1336,7 +1352,7 @@ public class BKDWriter implements Closeable {
BytesRef ref = packedValues.apply(i);
assert ref.length == packedBytesLength;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
out.writeBytes(ref.bytes, ref.offset + dim*bytesPerDim + prefix, bytesPerDim-prefix);
}
@ -1358,7 +1374,7 @@ public class BKDWriter implements Closeable {
}
private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue) throws IOException {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
out.writeVInt(commonPrefixes[dim]);
//System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);
out.writeBytes(packedValue, dim*bytesPerDim, commonPrefixes[dim]);
@ -1429,7 +1445,7 @@ public class BKDWriter implements Closeable {
boolean result = reader.next();
assert result: "rightCount=" + rightCount + " source.count=" + source.count + " source.writer=" + source.writer;
System.arraycopy(reader.packedValue(), splitDim*bytesPerDim, scratch1, 0, bytesPerDim);
if (numDims > 1) {
if (numDataDims > 1) {
assert ordBitSet.get(reader.ord()) == false;
ordBitSet.set(reader.ord());
// Subtract 1 from rightCount because we already did the first value above (so we could record the split value):
@ -1444,7 +1460,7 @@ public class BKDWriter implements Closeable {
/** Called only in assert */
private boolean valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim*dim;
if (FutureArrays.compareUnsigned(packedValue.bytes, packedValue.offset + offset, packedValue.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
return false;
@ -1472,7 +1488,7 @@ public class BKDWriter implements Closeable {
for (int numSplits : parentSplits) {
maxNumSplits = Math.max(maxNumSplits, numSplits);
}
for (int dim = 0; dim < numDims; ++dim) {
for (int dim = 0; dim < numIndexDims; ++dim) {
final int offset = dim * bytesPerDim;
if (parentSplits[dim] < maxNumSplits / 2 &&
FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) != 0) {
@ -1482,7 +1498,7 @@ public class BKDWriter implements Closeable {
// Find which dim has the largest span so we can split on it:
int splitDim = -1;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
if (splitDim == -1 || FutureArrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
@ -1532,7 +1548,7 @@ public class BKDWriter implements Closeable {
reader.getValue(from, scratchBytesRef1);
for (int i = from + 1; i < to; ++i) {
reader.getValue(i, scratchBytesRef2);
for (int dim=0;dim<numDims;dim++) {
for (int dim=0;dim<numDataDims;dim++) {
final int offset = dim * bytesPerDim;
for(int j=0;j<commonPrefixLengths[dim];j++) {
if (scratchBytesRef1.bytes[scratchBytesRef1.offset+offset+j] != scratchBytesRef2.bytes[scratchBytesRef2.offset+offset+j]) {
@ -1544,14 +1560,14 @@ public class BKDWriter implements Closeable {
}
// Find the dimension that has the least number of unique bytes at commonPrefixLengths[dim]
FixedBitSet[] usedBytes = new FixedBitSet[numDims];
for (int dim = 0; dim < numDims; ++dim) {
FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];
for (int dim = 0; dim < numDataDims; ++dim) {
if (commonPrefixLengths[dim] < bytesPerDim) {
usedBytes[dim] = new FixedBitSet(256);
}
}
for (int i = from + 1; i < to; ++i) {
for (int dim=0;dim<numDims;dim++) {
for (int dim=0;dim<numDataDims;dim++) {
if (usedBytes[dim] != null) {
byte b = reader.getByteAt(i, dim * bytesPerDim + commonPrefixLengths[dim]);
usedBytes[dim].set(Byte.toUnsignedInt(b));
@ -1560,7 +1576,7 @@ public class BKDWriter implements Closeable {
}
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
for (int dim = 0; dim < numDims; ++dim) {
for (int dim = 0; dim < numDataDims; ++dim) {
if (usedBytes[dim] != null) {
final int cardinality = usedBytes[dim].cardinality();
if (cardinality < sortedDimCardinality) {
@ -1631,8 +1647,8 @@ public class BKDWriter implements Closeable {
reader.getValue(mid, scratchBytesRef1);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim, splitPackedValues, address + 1, bytesPerDim);
byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedBytesLength);
byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedBytesLength);
byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);
byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
minSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
@ -1666,7 +1682,7 @@ public class BKDWriter implements Closeable {
assert slice.count == slices[0].count;
}
if (numDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
if (numDataDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
// Special case for 1D, to cutover to heap once we recurse deeply enough:
slices[0] = switchToHeap(slices[0], toCloseHeroically);
}
@ -1679,7 +1695,7 @@ public class BKDWriter implements Closeable {
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
for (int dim=0;dim<numDims;dim++) {
for (int dim=0;dim<numDataDims;dim++) {
if (slices[dim].writer instanceof HeapPointWriter == false) {
// Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started
// offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer
@ -1767,7 +1783,7 @@ public class BKDWriter implements Closeable {
// Inner node: partition/recurse
int splitDim;
if (numDims > 1) {
if (numIndexDims > 1) {
splitDim = split(minPackedValue, maxPackedValue, parentSplits);
} else {
splitDim = 0;
@ -1788,24 +1804,24 @@ public class BKDWriter implements Closeable {
// Partition all PathSlice that are not the split dim into sorted left and right sets, so we can recurse:
PathSlice[] leftSlices = new PathSlice[numDims];
PathSlice[] rightSlices = new PathSlice[numDims];
PathSlice[] leftSlices = new PathSlice[numDataDims];
PathSlice[] rightSlices = new PathSlice[numDataDims];
byte[] minSplitPackedValue = new byte[packedBytesLength];
System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedBytesLength);
byte[] minSplitPackedValue = new byte[packedIndexBytesLength];
System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);
byte[] maxSplitPackedValue = new byte[packedBytesLength];
System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedBytesLength);
byte[] maxSplitPackedValue = new byte[packedIndexBytesLength];
System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedIndexBytesLength);
// When we are on this dim, below, we clear the ordBitSet:
int dimToClear;
if (numDims - 1 == splitDim) {
dimToClear = numDims - 2;
if (numDataDims - 1 == splitDim) {
dimToClear = numDataDims - 2;
} else {
dimToClear = numDims - 1;
dimToClear = numDataDims - 1;
}
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
if (dim == splitDim) {
// No need to partition on this dim since it's a simple slice of the incoming already sorted slice, and we
@ -1842,7 +1858,7 @@ public class BKDWriter implements Closeable {
ordBitSet, out,
minPackedValue, maxSplitPackedValue, parentSplits,
splitPackedValues, leafBlockFPs, toCloseHeroically);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
leftSlices[dim].writer.destroy();
@ -1855,7 +1871,7 @@ public class BKDWriter implements Closeable {
ordBitSet, out,
minSplitPackedValue, maxPackedValue, parentSplits,
splitPackedValues, leafBlockFPs, toCloseHeroically);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
rightSlices[dim].writer.destroy();

11
lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
View File

@ -69,7 +69,7 @@ public class TestFieldType extends LuceneTestCase {
public void testPointsToString() {
FieldType ft = new FieldType();
ft.setDimensions(1, Integer.BYTES);
assertEquals("pointDimensionCount=1,pointNumBytes=4", ft.toString());
assertEquals("pointDataDimensionCount=1,pointIndexDimensionCount=1,pointNumBytes=4", ft.toString());
}
private static Object randomValue(Class<?> clazz) {
@ -85,15 +85,20 @@ public class TestFieldType extends LuceneTestCase {
private static FieldType randomFieldType() throws Exception {
// setDimensions handled special as values must be in-bounds.
Method setDimensionsMethod = FieldType.class.getMethod("setDimensions", int.class, int.class);
Method setDimensionsMethodA = FieldType.class.getMethod("setDimensions", int.class, int.class);
Method setDimensionsMethodB = FieldType.class.getMethod("setDimensions", int.class, int.class, int.class);
FieldType ft = new FieldType();
for (Method method : FieldType.class.getMethods()) {
if (method.getName().startsWith("set")) {
final Class<?>[] parameterTypes = method.getParameterTypes();
final Object[] args = new Object[parameterTypes.length];
if (method.equals(setDimensionsMethod)) {
if (method.equals(setDimensionsMethodA)) {
args[0] = 1 + random().nextInt(PointValues.MAX_DIMENSIONS);
args[1] = 1 + random().nextInt(PointValues.MAX_NUM_BYTES);
} else if (method.equals(setDimensionsMethodB)) {
args[0] = 1 + random().nextInt(PointValues.MAX_DIMENSIONS);
args[1] = 1 + random().nextInt((Integer)args[0]);
args[2] = 1 + random().nextInt(PointValues.MAX_NUM_BYTES);
} else {
for (int i = 0; i < args.length; ++i) {
args[i] = randomValue(parameterTypes[i]);

7
lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
View File

@ -91,7 +91,12 @@ public class TestIndexableField extends LuceneTestCase {
}
@Override
public int pointDimensionCount() {
public int pointDataDimensionCount() {
return 0;
}
@Override
public int pointIndexDimensionCount() {
return 0;
}

10
lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
View File

@ -164,7 +164,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
deletes.onNewReader(segmentReader, commitInfo);
reader.close();
writer.close();
FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0, true);
FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0, 0, true);
List<Integer> docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
for (DocValuesFieldUpdates update : updates) {
@ -185,7 +185,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
docsDeleted = Arrays.asList(1, 2, DocIdSetIterator.NO_MORE_DOCS);
updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0, true);
fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0, 0, true);
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
@ -228,7 +228,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo();
PendingDeletes deletes = newPendingDeletes(segmentInfo);
deletes.onNewReader(segmentReader, segmentInfo);
FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, 0, true);
List<Integer> docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 3, true));
for (DocValuesFieldUpdates update : updates) {
@ -276,7 +276,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo();
PendingDeletes deletes = newPendingDeletes(segmentInfo);
deletes.onNewReader(segmentReader, segmentInfo);
FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, 0, true);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(Arrays.asList(0, 1, DocIdSetIterator.NO_MORE_DOCS), 3, false));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
@ -295,7 +295,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
assertEquals(0, deletes.numPendingDeletes());
segmentInfo.advanceDocValuesGen();
fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, 0, true);
updates = Arrays.asList(singleUpdate(Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS), 3, true));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());

14
lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
View File

@ -73,7 +73,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w.addDocument(doc);
});
assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w.close();
dir.close();
}
@ -91,7 +91,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w.addDocument(doc2);
});
assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w.close();
dir.close();
@ -111,7 +111,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w.addDocument(doc2);
});
assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w.close();
dir.close();
@ -133,7 +133,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w2.addDocument(doc2);
});
assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w2.close();
dir.close();
@ -156,7 +156,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w2.addIndexes(new Directory[] {dir});
});
assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
IOUtils.close(w2, dir, dir2);
}
@ -179,7 +179,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w2.addIndexes(new CodecReader[] {(CodecReader) getOnlyLeafReader(r)});
});
assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
IOUtils.close(r, w2, dir, dir2);
}
@ -203,7 +203,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
TestUtil.addIndexesSlowly(w2, r);
});
assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
assertEquals("cannot change point data dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
IOUtils.close(r, w2, dir, dir2);
}

2
lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
View File

@ -1095,7 +1095,7 @@ public class TestPointQueries extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
s.count(BinaryPoint.newRangeQuery("value", point, point));
});
assertEquals("field=\"value\" was indexed with numDims=1 but this query has numDims=2", expected.getMessage());
assertEquals("field=\"value\" was indexed with numIndexDimensions=1 but this query has numDims=2", expected.getMessage());
IOUtils.close(r, w, dir);
}

7
lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
View File

@ -327,7 +327,12 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
}
@Override
public int getNumDimensions() throws IOException {
public int getNumDataDimensions() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int getNumIndexDimensions() throws IOException {
throw new UnsupportedOperationException();
}

4
lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
View File

@ -41,7 +41,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES,
BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1, Long.BYTES,
BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, 26L * numDocs, false);
int counter = 0;
byte[] packedBytes = new byte[Long.BYTES];
@ -78,7 +78,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES,
BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, 2, Long.BYTES,
BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, 26L * numDocs, false);
int counter = 0;
byte[] packedBytes = new byte[2*Long.BYTES];

142
lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
View File

@ -48,7 +48,7 @@ public class TestBKD extends LuceneTestCase {
public void testBasicInts1D() throws Exception {
try (Directory dir = getDirectory(100)) {
BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, 100, true);
BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 1, 4, 2, 1.0f, 100, true);
byte[] scratch = new byte[4];
for(int docID=0;docID<100;docID++) {
NumericUtils.intToSortableBytes(docID, scratch, 0);
@ -121,12 +121,13 @@ public class TestBKD extends LuceneTestCase {
int numDocs = atLeast(1000);
try (Directory dir = getDirectory(numDocs)) {
int numDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDims);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, numDocs, true);
BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numIndexDims, 4, maxPointsInLeafNode, maxMB, numDocs, true);
if (VERBOSE) {
System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs);
System.out.println("TEST: numDims=" + numDims + " numIndexDims=" + numIndexDims + " numDocs=" + numDocs);
}
int[][] docs = new int[numDocs][];
byte[] scratch = new byte[4*numDims];
@ -167,7 +168,7 @@ public class TestBKD extends LuceneTestCase {
byte[] minPackedValue = r.getMinPackedValue();
byte[] maxPackedValue = r.getMaxPackedValue();
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
assertEquals(minValue[dim], NumericUtils.sortableBytesToInt(minPackedValue, dim * Integer.BYTES));
assertEquals(maxValue[dim], NumericUtils.sortableBytesToInt(maxPackedValue, dim * Integer.BYTES));
}
@ -181,7 +182,7 @@ public class TestBKD extends LuceneTestCase {
// Random N dims rect query:
int[] queryMin = new int[numDims];
int[] queryMax = new int[numDims];
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
queryMin[dim] = random().nextInt();
queryMax[dim] = random().nextInt();
if (queryMin[dim] > queryMax[dim]) {
@ -202,7 +203,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public void visit(int docID, byte[] packedValue) {
//System.out.println("visit check docID=" + docID);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int x = NumericUtils.sortableBytesToInt(packedValue, dim * Integer.BYTES);
if (x < queryMin[dim] || x > queryMax[dim]) {
//System.out.println(" no");
@ -217,7 +218,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public Relation compare(byte[] minPacked, byte[] maxPacked) {
boolean crosses = false;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int min = NumericUtils.sortableBytesToInt(minPacked, dim * Integer.BYTES);
int max = NumericUtils.sortableBytesToInt(maxPacked, dim * Integer.BYTES);
assert max >= min;
@ -240,7 +241,7 @@ public class TestBKD extends LuceneTestCase {
for(int docID=0;docID<numDocs;docID++) {
int[] docValues = docs[docID];
boolean expected = true;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
int x = docValues[dim];
if (x < queryMin[dim] || x > queryMax[dim]) {
expected = false;
@ -264,7 +265,7 @@ public class TestBKD extends LuceneTestCase {
int numDims = TestUtil.nextInt(random(), 1, 5);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, numDocs, true);
BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, numDocs, true);
BigInteger[][] docs = new BigInteger[numDocs][];
byte[] scratch = new byte[numBytesPerDim*numDims];
@ -380,13 +381,14 @@ public class TestBKD extends LuceneTestCase {
public void testWithExceptions() throws Exception {
int numDocs = atLeast(10000);
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 1, 5);
int numDataDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@ -401,7 +403,7 @@ public class TestBKD extends LuceneTestCase {
try {
dir.setRandomIOExceptionRate(0.05);
dir.setRandomIOExceptionRateOnOpen(0.05);
verify(dir, docValues, null, numDims, numBytesPerDim, 50, maxMBHeap);
verify(dir, docValues, null, numDataDims, numIndexDims, numBytesPerDim, 50, maxMBHeap);
} catch (IllegalArgumentException iae) {
// This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry w/ more heap
assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
@ -439,7 +441,7 @@ public class TestBKD extends LuceneTestCase {
public void testTooLittleHeap() throws Exception {
try (Directory dir = getDirectory(0)) {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, 0, true);
new BKDWriter(1, dir, "bkd", 1, 1, 16, 1000000, 0.001, 0, true);
});
assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
}
@ -449,33 +451,35 @@ public class TestBKD extends LuceneTestCase {
int numDocs = TestUtil.nextInt(random(), count, count*2);
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 1, 5);
int numDataDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
docValues[docID] = values;
}
verify(docValues, null, numDims, numBytesPerDim);
verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
public void testAllEqual() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 1, 5);
int numDataDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
if (docID == 0) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@ -485,20 +489,21 @@ public class TestBKD extends LuceneTestCase {
}
}
verify(docValues, null, numDims, numBytesPerDim);
verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
public void testOneDimEqual() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 1, 5);
int numDataDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
int theEqualDim = random().nextInt(numDims);
int theEqualDim = random().nextInt(numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@ -509,17 +514,18 @@ public class TestBKD extends LuceneTestCase {
}
// Use a small number of points in leaf blocks to trigger a lot of splitting
verify(docValues, null, numDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
}
// This triggers the logic that makes sure all dimensions get indexed
// by looking at how many times each dim has been split
public void testOneDimLowCard() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 2, 5);
int numDataDims = TestUtil.nextInt(random(), 2, 5);
int numIndexDims = TestUtil.nextInt(random(), 2, numDataDims);
int numDocs = atLeast(10000);
int theLowCardDim = random().nextInt(numDims);
int theLowCardDim = random().nextInt(numDataDims);
byte[] value1 = new byte[numBytesPerDim];
random().nextBytes(value1);
@ -533,8 +539,8 @@ public class TestBKD extends LuceneTestCase {
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
if (dim == theLowCardDim) {
values[dim] = random().nextBoolean() ? value1 : value2;
} else {
@ -546,16 +552,17 @@ public class TestBKD extends LuceneTestCase {
}
// Use a small number of points in leaf blocks to trigger a lot of splitting
verify(docValues, null, numDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
}
// this should trigger run-length compression with lengths that are greater than 255
public void testOneDimTwoValues() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 1, 5);
int numDataDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
int theDim = random().nextInt(numDims);
int theDim = random().nextInt(numDataDims);
byte[] value1 = new byte[numBytesPerDim];
random().nextBytes(value1);
byte[] value2 = new byte[numBytesPerDim];
@ -563,8 +570,8 @@ public class TestBKD extends LuceneTestCase {
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
if (dim == theDim) {
values[dim] = random().nextBoolean() ? value1 : value2;
} else {
@ -575,12 +582,13 @@ public class TestBKD extends LuceneTestCase {
docValues[docID] = values;
}
verify(docValues, null, numDims, numBytesPerDim);
verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
public void testMultiValued() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
int numDims = TestUtil.nextInt(random(), 1, 5);
int numDataDims = TestUtil.nextInt(random(), 1, 5);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
List<byte[][]> docValues = new ArrayList<>();
@ -590,8 +598,8 @@ public class TestBKD extends LuceneTestCase {
int numValuesInDoc = TestUtil.nextInt(random(), 1, 5);
for(int ord=0;ord<numValuesInDoc;ord++) {
docIDs.add(docID);
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@ -605,33 +613,33 @@ public class TestBKD extends LuceneTestCase {
docIDsArray[i] = docIDs.get(i);
}
verify(docValuesArray, docIDsArray, numDims, numBytesPerDim);
verify(docValuesArray, docIDsArray, numDataDims, numIndexDims, numBytesPerDim);
}
/** docIDs can be null, for the single valued case, else it maps value to docID */
private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim) throws Exception {
verify(docValues, docIDs, numDims, numBytesPerDim, TestUtil.nextInt(random(), 50, 1000));
private void verify(byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim) throws Exception {
verify(docValues, docIDs, numDataDims, numIndexDims, numBytesPerDim, TestUtil.nextInt(random(), 50, 1000));
}
private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim,
private void verify(byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim,
int maxPointsInLeafNode) throws Exception {
try (Directory dir = getDirectory(docValues.length)) {
double maxMB = (float) 3.0 + (3*random().nextDouble());
verify(dir, docValues, docIDs, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
verify(dir, docValues, docIDs, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
}
}
private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim, int maxPointsInLeafNode, double maxMB) throws Exception {
private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim, int maxPointsInLeafNode, double maxMB) throws Exception {
int numValues = docValues.length;
if (VERBOSE) {
System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim + " maxPointsInLeafNode=" + maxPointsInLeafNode + " maxMB=" + maxMB);
System.out.println("TEST: numValues=" + numValues + " numDataDims=" + numDataDims + " numIndexDims=" + numIndexDims + " numBytesPerDim=" + numBytesPerDim + " maxPointsInLeafNode=" + maxPointsInLeafNode + " maxMB=" + maxMB);
}
List<Long> toMerge = null;
List<MergeState.DocMap> docMaps = null;
int seg = 0;
BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
IndexInput in = null;
@ -639,9 +647,9 @@ public class TestBKD extends LuceneTestCase {
try {
byte[] scratch = new byte[numBytesPerDim*numDims];
byte[] scratch = new byte[numBytesPerDim*numDataDims];
int lastDocIDBase = 0;
boolean useMerge = numDims == 1 && numValues >= 10 && random().nextBoolean();
boolean useMerge = numDataDims == 1 && numValues >= 10 && random().nextBoolean();
int valuesInThisSeg;
if (useMerge) {
// Sometimes we will call merge with a single segment:
@ -662,7 +670,7 @@ public class TestBKD extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" ord=" + ord + " docID=" + docID + " lastDocIDBase=" + lastDocIDBase);
}
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
if (VERBOSE) {
System.out.println(" " + dim + " -> " + new BytesRef(docValues[ord][dim]));
}
@ -691,7 +699,7 @@ public class TestBKD extends LuceneTestCase {
seg++;
maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000);
maxMB = (float) 3.0 + (3*random().nextDouble());
w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
w = new BKDWriter(numValues, dir, "_" + seg, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
lastDocIDBase = docID;
}
}
@ -712,7 +720,7 @@ public class TestBKD extends LuceneTestCase {
out.close();
in = dir.openInput("bkd", IOContext.DEFAULT);
seg++;
w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
w = new BKDWriter(numValues, dir, "_" + seg, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
List<BKDReader> readers = new ArrayList<>();
for(long fp : toMerge) {
in.seek(fp);
@ -739,9 +747,9 @@ public class TestBKD extends LuceneTestCase {
}
// Random N dims rect query:
byte[][] queryMin = new byte[numDims][];
byte[][] queryMax = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] queryMin = new byte[numDataDims][];
byte[][] queryMax = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
queryMin[dim] = new byte[numBytesPerDim];
random().nextBytes(queryMin[dim]);
queryMax[dim] = new byte[numBytesPerDim];
@ -764,7 +772,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public void visit(int docID, byte[] packedValue) {
//System.out.println("visit check docID=" + docID);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
//System.out.println(" no");
@ -779,7 +787,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public Relation compare(byte[] minPacked, byte[] maxPacked) {
boolean crosses = false;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(maxPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(minPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
return Relation.CELL_OUTSIDE_QUERY;
@ -800,7 +808,7 @@ public class TestBKD extends LuceneTestCase {
BitSet expected = new BitSet();
for(int ord=0;ord<numValues;ord++) {
boolean matches = true;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
byte[] x = docValues[ord][dim];
if (FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
@ -897,7 +905,7 @@ public class TestBKD extends LuceneTestCase {
};
CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
});
assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
}
@ -945,7 +953,7 @@ public class TestBKD extends LuceneTestCase {
};
Throwable t = expectThrows(CorruptIndexException.class, () -> {
verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
});
assertCorruptionDetected(t);
}
@ -971,7 +979,7 @@ public class TestBKD extends LuceneTestCase {
public void testTieBreakOrder() throws Exception {
try (Directory dir = newDirectory()) {
int numDocs = 10000;
BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, Integer.BYTES, 2, 0.01f, numDocs, true);
BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, 1, Integer.BYTES, 2, 0.01f, numDocs, true);
for(int i=0;i<numDocs;i++) {
w.add(new byte[Integer.BYTES], i);
}
@ -1012,7 +1020,7 @@ public class TestBKD extends LuceneTestCase {
boolean singleValuePerDoc = false;
boolean longOrds = true;
int offlineSorterMaxTempFiles = TestUtil.nextInt(random(), 2, 20);
BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 2, Integer.BYTES, 2, 0.01f, numDocs,
BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 2, 2, Integer.BYTES, 2, 0.01f, numDocs,
singleValuePerDoc, longOrds, 1, offlineSorterMaxTempFiles);
byte[] buffer = new byte[2*Integer.BYTES];
for(int i=0;i<numDocs;i++) {
@ -1058,12 +1066,13 @@ public class TestBKD extends LuceneTestCase {
// values as a LongPoint:
public void testWastedLeadingBytes() throws Exception {
int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
int numIndexDims = TestUtil.nextInt(random(), 1, numDims);
int bytesPerDim = PointValues.MAX_NUM_BYTES;
int bytesUsed = TestUtil.nextInt(random(), 1, 3);
Directory dir = newFSDirectory(createTempDir());
int numDocs = 100000;
BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", numDims, bytesPerDim, 32, 1f, numDocs, true);
BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", numDims, numIndexDims, bytesPerDim, 32, 1f, numDocs, true);
byte[] tmp = new byte[bytesUsed];
byte[] buffer = new byte[numDims * bytesPerDim];
for(int i=0;i<numDocs;i++) {
@ -1091,11 +1100,14 @@ public class TestBKD extends LuceneTestCase {
@Override
public void visit(int docID, byte[] packedValue) {
assert packedValue.length == numDims * bytesPerDim;
visit(docID);
}
@Override
public Relation compare(byte[] minPacked, byte[] maxPacked) {
assert minPacked.length == numIndexDims * bytesPerDim;
assert maxPacked.length == numIndexDims * bytesPerDim;
if (random().nextInt(7) == 1) {
return Relation.CELL_CROSSES_QUERY;
} else {
@ -1117,7 +1129,7 @@ public class TestBKD extends LuceneTestCase {
final byte[] uniquePointValue = new byte[numBytesPerDim];
random().nextBytes(uniquePointValue);
BKDWriter w = new BKDWriter(numValues, dir, "_temp", 1, numBytesPerDim, maxPointsInLeafNode,
BKDWriter w = new BKDWriter(numValues, dir, "_temp", 1, 1, numBytesPerDim, maxPointsInLeafNode,
BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, numValues, true);
for (int i = 0; i < numValues; ++i) {
if (i == numValues / 2) {

7
lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java
View File

@ -232,7 +232,12 @@ public class TestMutablePointsReaderUtils extends LuceneTestCase {
}
@Override
public int getNumDimensions() throws IOException {
public int getNumDataDimensions() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int getNumIndexDimensions() throws IOException {
throw new UnsupportedOperationException();
}

2
lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
View File

@ -81,7 +81,7 @@ public class TermVectorLeafReader extends LeafReader {
}
FieldInfo fieldInfo = new FieldInfo(field, 0,
true, true, terms.hasPayloads(),
indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, false);
indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}

4
lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
View File

@ -146,8 +146,8 @@ abstract class PointInSetIncludingScoreQuery extends Query {
if (fieldInfo == null) {
return null;
}
if (fieldInfo.getPointDimensionCount() != 1) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + fieldInfo.getPointDimensionCount() + " but this query has numDims=1");
if (fieldInfo.getPointDataDimensionCount() != 1) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + fieldInfo.getPointDataDimensionCount() + " but this query has numDims=1");
}
if (fieldInfo.getPointNumBytes() != bytesPerDim) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + fieldInfo.getPointNumBytes() + " but this query has bytesPerDim=" + bytesPerDim);

24
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -410,7 +410,7 @@ public class MemoryIndex {
storeDocValues(info, docValuesType, docValuesValue);
}
if (field.fieldType().pointDimensionCount() > 0) {
if (field.fieldType().pointDataDimensionCount() > 0) {
storePointValues(info, field.binaryValue());
}
@ -486,9 +486,9 @@ public class MemoryIndex {
if (info == null) {
fields.put(fieldName, info = new Info(createFieldInfo(fieldName, fields.size(), fieldType), byteBlockPool));
}
if (fieldType.pointDimensionCount() != info.fieldInfo.getPointDimensionCount()) {
if (fieldType.pointDimensionCount() > 0)
info.fieldInfo.setPointDimensions(fieldType.pointDimensionCount(), fieldType.pointNumBytes());
if (fieldType.pointDataDimensionCount() != info.fieldInfo.getPointDataDimensionCount()) {
if (fieldType.pointDataDimensionCount() > 0)
info.fieldInfo.setPointDimensions(fieldType.pointDataDimensionCount(), fieldType.pointIndexDimensionCount(), fieldType.pointNumBytes());
}
if (fieldType.docValuesType() != info.fieldInfo.getDocValuesType()) {
if (fieldType.docValuesType() != DocValuesType.NONE)
@ -501,7 +501,7 @@ public class MemoryIndex {
IndexOptions indexOptions = storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
return new FieldInfo(fieldName, ord, fieldType.storeTermVectors(), fieldType.omitNorms(), storePayloads,
indexOptions, fieldType.docValuesType(), -1, Collections.emptyMap(),
fieldType.pointDimensionCount(), fieldType.pointNumBytes(), false);
fieldType.pointDataDimensionCount(), fieldType.pointIndexDimensionCount(), fieldType.pointNumBytes(), false);
}
private void storePointValues(Info info, BytesRef pointValue) {
@ -520,7 +520,8 @@ public class MemoryIndex {
info.fieldInfo = new FieldInfo(
info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(),
info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(),
info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes(), info.fieldInfo.isSoftDeletesField()
info.fieldInfo.getPointDataDimensionCount(), info.fieldInfo.getPointIndexDimensionCount(), info.fieldInfo.getPointNumBytes(),
info.fieldInfo.isSoftDeletesField()
);
} else if (existingDocValuesType != docValuesType) {
throw new IllegalArgumentException("Can't add [" + docValuesType + "] doc values field [" + fieldName + "], because [" + existingDocValuesType + "] doc values field already exists");
@ -870,7 +871,7 @@ public class MemoryIndex {
if (pointValues != null) {
assert pointValues[0].bytes.length == pointValues[0].length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
final int numDimensions = fieldInfo.getPointDimensionCount();
final int numDimensions = fieldInfo.getPointDataDimensionCount();
final int numBytesPerDimension = fieldInfo.getPointNumBytes();
if (numDimensions == 1) {
// PointInSetQuery.MergePointVisitor expects values to be visited in increasing order,
@ -1576,8 +1577,13 @@ public class MemoryIndex {
}
@Override
public int getNumDimensions() throws IOException {
return info.fieldInfo.getPointDimensionCount();
public int getNumDataDimensions() throws IOException {
return info.fieldInfo.getPointDataDimensionCount();
}
@Override
public int getNumIndexDimensions() throws IOException {
return info.fieldInfo.getPointDataDimensionCount();
}
@Override

10
lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java
View File

@ -62,8 +62,8 @@ public class BigIntegerPoint extends Field {
/** Change the values of this field */
public void setBigIntegerValues(BigInteger... point) {
if (type.pointDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
if (type.pointDataDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@ -75,8 +75,8 @@ public class BigIntegerPoint extends Field {
@Override
public Number numericValue() {
if (type.pointDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
if (type.pointDataDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == BYTES;
@ -119,7 +119,7 @@ public class BigIntegerPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}

10
lucene/sandbox/src/java/org/apache/lucene/document/HalfFloatPoint.java
View File

@ -213,8 +213,8 @@ public final class HalfFloatPoint extends Field {
/** Change the values of this field */
public void setFloatValues(float... point) {
if (type.pointDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
if (type.pointDataDimensionCount() != point.length) {
throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@ -226,8 +226,8 @@ public final class HalfFloatPoint extends Field {
@Override
public Number numericValue() {
if (type.pointDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
if (type.pointDataDimensionCount() != 1) {
throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == BYTES;
@ -270,7 +270,7 @@ public final class HalfFloatPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}

2
lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
View File

@ -148,7 +148,7 @@ public class BBoxStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numQuads++;
}
if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numQuads++;
}

2
lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
View File

@ -139,7 +139,7 @@ public class PointVectorStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numPairs++;
}
if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numPairs++;
}
this.fieldsLen = numPairs * 2;

4
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
View File

@ -131,8 +131,8 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException {
if (fieldInfo.getPointDimensionCount() == 0) {
throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDimensionalCount is 0");
if (fieldInfo.getPointDataDimensionCount() == 0) {
throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDataDimensionalCount is 0");
}
in.writeField(fieldInfo, values);
}

12
lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
View File

@ -155,11 +155,19 @@ class CrankyPointsFormat extends PointsFormat {
}
@Override
public int getNumDimensions() throws IOException {
public int getNumDataDimensions() throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException");
}
return delegate.getNumDimensions();
return delegate.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException");
}
return delegate.getNumIndexDimensions();
}
@Override

41
lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
View File

@ -1048,7 +1048,7 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public void intersect(IntersectVisitor visitor) throws IOException {
in.intersect(new AssertingIntersectVisitor(in.getNumDimensions(), in.getBytesPerDimension(), visitor));
in.intersect(new AssertingIntersectVisitor(in.getNumDataDimensions(), in.getNumIndexDimensions(), in.getBytesPerDimension(), visitor));
}
@Override
@ -1069,8 +1069,13 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
public int getNumDimensions() throws IOException {
return in.getNumDimensions();
public int getNumDataDimensions() throws IOException {
return in.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
return in.getNumIndexDimensions();
}
@Override
@ -1093,7 +1098,8 @@ public class AssertingLeafReader extends FilterLeafReader {
/** Validates in the 1D case that all points are visited in order, and point values are in bounds of the last cell checked */
static class AssertingIntersectVisitor implements IntersectVisitor {
final IntersectVisitor in;
final int numDims;
final int numDataDims;
final int numIndexDims;
final int bytesPerDim;
final byte[] lastDocValue;
final byte[] lastMinPackedValue;
@ -1102,13 +1108,14 @@ public class AssertingLeafReader extends FilterLeafReader {
private int lastDocID = -1;
private int docBudget;
AssertingIntersectVisitor(int numDims, int bytesPerDim, IntersectVisitor in) {
AssertingIntersectVisitor(int numDataDims, int numIndexDims, int bytesPerDim, IntersectVisitor in) {
this.in = in;
this.numDims = numDims;
this.numDataDims = numDataDims;
this.numIndexDims = numIndexDims;
this.bytesPerDim = bytesPerDim;
lastMaxPackedValue = new byte[numDims*bytesPerDim];
lastMinPackedValue = new byte[numDims*bytesPerDim];
if (numDims == 1) {
lastMaxPackedValue = new byte[numDataDims*bytesPerDim];
lastMinPackedValue = new byte[numDataDims*bytesPerDim];
if (numDataDims == 1) {
lastDocValue = new byte[bytesPerDim];
} else {
lastDocValue = null;
@ -1132,14 +1139,14 @@ public class AssertingLeafReader extends FilterLeafReader {
assert lastCompareResult == PointValues.Relation.CELL_CROSSES_QUERY;
// This doc's packed value should be contained in the last cell passed to compare:
for(int dim=0;dim<numDims;dim++) {
assert FutureArrays.compareUnsigned(lastMinPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) <= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
assert FutureArrays.compareUnsigned(lastMaxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) >= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
for(int dim=0;dim<numIndexDims;dim++) {
assert FutureArrays.compareUnsigned(lastMinPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) <= 0: "dim=" + dim + " of " + numDataDims + " value=" + new BytesRef(packedValue);
assert FutureArrays.compareUnsigned(lastMaxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) >= 0: "dim=" + dim + " of " + numDataDims + " value=" + new BytesRef(packedValue);
}
// TODO: we should assert that this "matches" whatever relation the last call to compare had returned
assert packedValue.length == numDims * bytesPerDim;
if (numDims == 1) {
assert packedValue.length == numDataDims * bytesPerDim;
if (numDataDims == 1) {
int cmp = FutureArrays.compareUnsigned(lastDocValue, 0, bytesPerDim, packedValue, 0, bytesPerDim);
if (cmp < 0) {
// ok
@ -1163,11 +1170,11 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
assert FutureArrays.compareUnsigned(minPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, maxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) <= 0;
}
System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, numDims*bytesPerDim);
System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, numDims*bytesPerDim);
System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, numIndexDims*bytesPerDim);
System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, numIndexDims*bytesPerDim);
lastCompareResult = in.compare(minPackedValue, maxPackedValue);
return lastCompareResult;
}

2
lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
View File

@ -325,7 +325,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
FieldInfo proto = oneDocReader.getFieldInfos().fieldInfo("field");
FieldInfo field = new FieldInfo(proto.name, proto.number, proto.hasVectors(), proto.omitsNorms(), proto.hasPayloads(),
proto.getIndexOptions(), proto.getDocValuesType(), proto.getDocValuesGen(), new HashMap<>(),
proto.getPointDimensionCount(), proto.getPointNumBytes(), proto.isSoftDeletesField());
proto.getPointDataDimensionCount(), proto.getPointIndexDimensionCount(), proto.getPointNumBytes(), proto.isSoftDeletesField());
FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { field } );

76
lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
@ -520,13 +521,14 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
private void doTestRandomBinary(int count) throws Exception {
int numDocs = TestUtil.nextInt(random(), count, count*2);
int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
int numDataDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
byte[][] values = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] values = new byte[numDataDims][];
for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
// TODO: sometimes test on a "small" volume too, so we test the high density cases, higher chance of boundary, etc. cases:
random().nextBytes(values[dim]);
@ -534,17 +536,22 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
docValues[docID] = values;
}
verify(docValues, null, numDims, numBytesPerDim);
verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim) throws Exception {
verify(docValues, docIDs, numDims, numDims, numBytesPerDim);
}
/** docIDs can be null, for the single valued case, else it maps value to docID, but all values for one doc must be adjacent */
private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim) throws Exception {
private void verify(byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim) throws Exception {
try (Directory dir = getDirectory(docValues.length)) {
while (true) {
try {
verify(dir, docValues, docIDs, numDims, numBytesPerDim, false);
verify(dir, docValues, docIDs, numDataDims, numIndexDims, numBytesPerDim, false);
return;
} catch (IllegalArgumentException iae) {
iae.printStackTrace();
// This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry
assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
}
@ -553,9 +560,22 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
private void verify(Directory dir, byte[][][] docValues, int[] ids, int numDims, int numBytesPerDim, boolean expectExceptions) throws Exception {
verify(dir, docValues, ids, numDims, numDims, numBytesPerDim, expectExceptions);
}
private byte[] flattenBinaryPoint(byte[][] value, int numDataDims, int numBytesPerDim) {
byte[] result = new byte[value.length * numBytesPerDim];
for (int d = 0; d < numDataDims; ++d) {
System.arraycopy(value[d], 0, result, d * numBytesPerDim, numBytesPerDim);
}
return result;
}
/** test selective indexing */
private void verify(Directory dir, byte[][][] docValues, int[] ids, int numDataDims, int numIndexDims, int numBytesPerDim, boolean expectExceptions) throws Exception {
int numValues = docValues.length;
if (VERBOSE) {
System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim);
System.out.println("TEST: numValues=" + numValues + " numDataDims=" + numDataDims + " numIndexDims=" + numIndexDims + " numBytesPerDim=" + numBytesPerDim);
}
// RandomIndexWriter is too slow:
@ -578,10 +598,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
DirectoryReader r = null;
// Compute actual min/max values:
byte[][] expectedMinValues = new byte[numDims][];
byte[][] expectedMaxValues = new byte[numDims][];
byte[][] expectedMinValues = new byte[numDataDims][];
byte[][] expectedMaxValues = new byte[numDataDims][];
for(int ord=0;ord<docValues.length;ord++) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
if (ord == 0) {
expectedMinValues[dim] = new byte[numBytesPerDim];
System.arraycopy(docValues[ord][dim], 0, expectedMinValues[dim], 0, numBytesPerDim);
@ -629,6 +649,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
try {
FieldType fieldType = new FieldType();
fieldType.setDimensions(numDataDims, numIndexDims, numBytesPerDim);
fieldType.freeze();
Document doc = null;
int lastID = -1;
for(int ord=0;ord<numValues;ord++) {
@ -649,7 +673,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
doc = new Document();
doc.add(new NumericDocValuesField("id", id));
}
doc.add(new BinaryPoint("field", docValues[ord]));
// pack the binary point
byte[] val = flattenBinaryPoint(docValues[ord], numDataDims, numBytesPerDim);
doc.add(new BinaryPoint("field", val, fieldType));
lastID = id;
if (random().nextInt(30) == 17) {
@ -667,7 +694,8 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
if (random().nextInt(30) == 17) {
// randomly index some documents with this field, but we will delete them:
Document xdoc = new Document();
xdoc.add(new BinaryPoint("field", docValues[ord]));
val = flattenBinaryPoint(docValues[ord], numDataDims, numBytesPerDim);
xdoc.add(new BinaryPoint("field", val, fieldType));
xdoc.add(new StringField("nukeme", "yes", Field.Store.NO));
if (useRealWriter) {
w.w.addDocument(xdoc);
@ -689,7 +717,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
if (VERBOSE) {
System.out.println(" ord=" + ord + " id=" + id);
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numDataDims;dim++) {
System.out.println(" dim=" + dim + " value=" + new BytesRef(docValues[ord][dim]));
}
}
@ -731,10 +759,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
Bits liveDocs = MultiFields.getLiveDocs(r);
// Verify min/max values are correct:
byte[] minValues = new byte[numDims*numBytesPerDim];
byte[] minValues = new byte[numIndexDims*numBytesPerDim];
Arrays.fill(minValues, (byte) 0xff);
byte[] maxValues = new byte[numDims*numBytesPerDim];
byte[] maxValues = new byte[numIndexDims*numBytesPerDim];
for(LeafReaderContext ctx : r.leaves()) {
PointValues dimValues = ctx.reader().getPointValues("field");
@ -744,7 +772,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
byte[] leafMinValues = dimValues.getMinPackedValue();
byte[] leafMaxValues = dimValues.getMaxPackedValue();
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(leafMinValues, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, minValues, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim) < 0) {
System.arraycopy(leafMinValues, dim*numBytesPerDim, minValues, dim*numBytesPerDim, numBytesPerDim);
}
@ -755,7 +783,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
byte[] scratch = new byte[numBytesPerDim];
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
System.arraycopy(minValues, dim*numBytesPerDim, scratch, 0, numBytesPerDim);
//System.out.println("dim=" + dim + " expectedMin=" + new BytesRef(expectedMinValues[dim]) + " min=" + new BytesRef(scratch));
assertTrue(Arrays.equals(expectedMinValues[dim], scratch));
@ -771,9 +799,9 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
// Random N dims rect query:
byte[][] queryMin = new byte[numDims][];
byte[][] queryMax = new byte[numDims][];
for(int dim=0;dim<numDims;dim++) {
byte[][] queryMin = new byte[numIndexDims][];
byte[][] queryMax = new byte[numIndexDims][];
for(int dim=0;dim<numIndexDims;dim++) {
queryMin[dim] = new byte[numBytesPerDim];
random().nextBytes(queryMin[dim]);
queryMax[dim] = new byte[numBytesPerDim];
@ -786,7 +814,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
if (VERBOSE) {
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
System.out.println(" dim=" + dim + "\n queryMin=" + new BytesRef(queryMin[dim]) + "\n queryMax=" + new BytesRef(queryMax[dim]));
}
}
@ -816,7 +844,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
return;
}
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
//System.out.println(" dim=" + dim + " value=" + new BytesRef(packedValue, dim*numBytesPerDim, numBytesPerDim));
if (FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
@ -833,7 +861,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
public Relation compare(byte[] minPacked, byte[] maxPacked) {
boolean crosses = false;
//System.out.println("compare");
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(maxPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(minPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
//System.out.println(" query_outside_cell");
@ -858,7 +886,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
BitSet expected = new BitSet();
for(int ord=0;ord<numValues;ord++) {
boolean matches = true;
for(int dim=0;dim<numDims;dim++) {
for(int dim=0;dim<numIndexDims;dim++) {
byte[] x = docValues[ord][dim];
if (FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {

3
lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
View File

@ -76,7 +76,8 @@ public class MismatchedLeafReader extends FilterLeafReader {
oldInfo.getDocValuesType(), // docValuesType
oldInfo.getDocValuesGen(), // dvGen
oldInfo.attributes(), // attributes
oldInfo.getPointDimensionCount(), // dimension count
oldInfo.getPointDataDimensionCount(), // data dimension count
oldInfo.getPointIndexDimensionCount(), // index dimension count
oldInfo.getPointNumBytes(), // dimension numBytes
oldInfo.isSoftDeletesField()); // used as soft-deletes field
shuffled.set(i, newInfo);

9
lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
View File

@ -111,7 +111,8 @@ public class RandomCodec extends AssertingCodec {
try (BKDWriter writer = new RandomlySplittingBKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointDataDimensionCount(),
fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
maxMBSortInHeap,
@ -259,10 +260,10 @@ public class RandomCodec extends AssertingCodec {
final Random random;
public RandomlySplittingBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims,
public RandomlySplittingBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims,
int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap,
long totalPointCount, boolean singleValuePerDoc, int randomSeed) throws IOException {
super(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount,
super(maxDoc, tempDir, tempFileNamePrefix, numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount,
getRandomSingleValuePerDoc(singleValuePerDoc, randomSeed),
getRandomLongOrds(totalPointCount, singleValuePerDoc, randomSeed),
getRandomOfflineSorterBufferMB(randomSeed),
@ -291,7 +292,7 @@ public class RandomCodec extends AssertingCodec {
@Override
protected int split(byte[] minPackedValue, byte[] maxPackedValue, int[] parentDims) {
// BKD normally defaults by the widest dimension, to try to make as squarish cells as possible, but we just pick a random one ;)
return random.nextInt(numDims);
return random.nextInt(numIndexDims);
}
}
}

4
lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
View File

@ -130,7 +130,7 @@ public class RandomPostingsTester {
fieldInfoArray[fieldUpto] = new FieldInfo(field, fieldUpto, false, false, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE, -1, new HashMap<>(),
0, 0, false);
0, 0, 0, false);
fieldUpto++;
SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@ -651,7 +651,7 @@ public class RandomPostingsTester {
DocValuesType.NONE,
-1,
new HashMap<>(),
0, 0, false);
0, 0, 0, false);
}
FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);

16
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -2627,10 +2627,12 @@ public abstract class LuceneTestCase extends Assert {
FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(leftReader);
FieldInfos fieldInfos2 = MultiFields.getMergedFieldInfos(rightReader);
for(FieldInfo fieldInfo1 : fieldInfos1) {
if (fieldInfo1.getPointDimensionCount() != 0) {
if (fieldInfo1.getPointDataDimensionCount() != 0) {
FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldInfo1.name);
// same dimension count?
assertEquals(info, fieldInfo2.getPointDimensionCount(), fieldInfo2.getPointDimensionCount());
// same data dimension count?
assertEquals(info, fieldInfo2.getPointDataDimensionCount(), fieldInfo2.getPointDataDimensionCount());
// same index dimension count?
assertEquals(info, fieldInfo2.getPointIndexDimensionCount(), fieldInfo2.getPointIndexDimensionCount());
// same bytes per dimension?
assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo2.getPointNumBytes());
@ -2642,10 +2644,12 @@ public abstract class LuceneTestCase extends Assert {
// make sure FieldInfos2 doesn't have any point fields that FieldInfo1 didn't have
for(FieldInfo fieldInfo2 : fieldInfos2) {
if (fieldInfo2.getPointDimensionCount() != 0) {
if (fieldInfo2.getPointDataDimensionCount() != 0) {
FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldInfo2.name);
// same dimension count?
assertEquals(info, fieldInfo2.getPointDimensionCount(), fieldInfo1.getPointDimensionCount());
// same data dimension count?
assertEquals(info, fieldInfo2.getPointDataDimensionCount(), fieldInfo1.getPointDataDimensionCount());
// same index dimension count?
assertEquals(info, fieldInfo2.getPointIndexDimensionCount(), fieldInfo1.getPointIndexDimensionCount());
// same bytes per dimension?
assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo1.getPointNumBytes());

2
lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
View File

@ -1082,7 +1082,7 @@ public final class TestUtil {
final Field field1 = (Field) f;
final Field field2;
final DocValuesType dvType = field1.fieldType().docValuesType();
final int dimCount = field1.fieldType().pointDimensionCount();
final int dimCount = field1.fieldType().pointDataDimensionCount();
if (dvType != DocValuesType.NONE) {
switch(dvType) {
case NUMERIC:

3
solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
View File

@ -795,7 +795,8 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
DocValuesType.NONE,
fieldInfo.getDocValuesGen(),
fieldInfo.attributes(),
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointDataDimensionCount(),
fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
fieldInfo.isSoftDeletesField());
newInfos.add(f);

2
solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
View File

@ -173,7 +173,7 @@ public class BBoxStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numQuads++;
}
if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numQuads++;
}
if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {

2
solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
View File

@ -152,7 +152,7 @@ public class PointVectorStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numPairs++;
}
if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numPairs++;
}
if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {

7
solr/core/src/java/org/apache/solr/schema/SchemaField.java
View File

@ -415,7 +415,12 @@ public final class SchemaField extends FieldProperties implements IndexableField
}
@Override
public int pointDimensionCount() {
public int pointDataDimensionCount() {
return 0;
}
@Override
public int pointIndexDimensionCount() {
return 0;
}

2
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -431,7 +431,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
DocValuesType.NONE,
fieldInfo.getDocValuesGen(),
fieldInfo.attributes(),
0, 0, fieldInfo.isSoftDeletesField());
0, 0, 0, fieldInfo.isSoftDeletesField());
newInfos.add(f);
} else {

2
solr/core/src/java/org/apache/solr/search/Insanity.java
View File

@ -67,7 +67,7 @@ public class Insanity {
if (fi.name.equals(insaneField)) {
filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), DocValuesType.NONE, -1, Collections.emptyMap(),
fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
} else {
filteredInfos.add(fi);
}

6
solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
View File

@ -592,11 +592,11 @@ public class FieldCacheImpl implements FieldCache {
if (parser instanceof PointParser) {
// points case
// no points in this segment
if (info.getPointDimensionCount() == 0) {
if (info.getPointDataDimensionCount() == 0) {
return DocValues.emptyNumeric();
}
if (info.getPointDimensionCount() != 1) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDimensionCount());
if (info.getPointDataDimensionCount() != 1) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDataDimensionCount());
}
PointValues values = reader.getPointValues(field);
// no actual points for this field (e.g. all points deleted)

6
solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
View File

@ -237,12 +237,12 @@ public class UninvertingReader extends FilterLeafReader {
DocValuesType type = fi.getDocValuesType();
// fields which currently don't have docValues, but are uninvertable (indexed or points data present)
if (type == DocValuesType.NONE &&
(fi.getIndexOptions() != IndexOptions.NONE || (fi.getPointNumBytes() > 0 && fi.getPointDimensionCount() == 1))) {
(fi.getIndexOptions() != IndexOptions.NONE || (fi.getPointNumBytes() > 0 && fi.getPointDataDimensionCount() == 1))) {
Type t = mapping.apply(fi.name); // could definitely return null, thus still can't uninvert it
if (t != null) {
if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) {
// type uses points
if (fi.getPointDimensionCount() == 0) {
if (fi.getPointDataDimensionCount() == 0) {
continue;
}
} else {
@ -284,7 +284,7 @@ public class UninvertingReader extends FilterLeafReader {
wrap = true;
newFieldInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(),
fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
} else {
newFieldInfos.add(fi);
}

6
solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
View File

@ -379,12 +379,14 @@ public class TestUninvertingReader extends LuceneTestCase {
FieldInfo intFInfo = fieldInfos.fieldInfo("int");
assertEquals(DocValuesType.NUMERIC, intFInfo.getDocValuesType());
assertEquals(0, intFInfo.getPointDimensionCount());
assertEquals(0, intFInfo.getPointDataDimensionCount());
assertEquals(0, intFInfo.getPointIndexDimensionCount());
assertEquals(0, intFInfo.getPointNumBytes());
FieldInfo dintFInfo = fieldInfos.fieldInfo("dint");
assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType());
assertEquals(1, dintFInfo.getPointDimensionCount());
assertEquals(1, dintFInfo.getPointDataDimensionCount());
assertEquals(1, dintFInfo.getPointIndexDimensionCount());
assertEquals(4, dintFInfo.getPointNumBytes());
FieldInfo dvFInfo = fieldInfos.fieldInfo("dv");