mirror of https://github.com/apache/lucene.git
Introduce TestLucene90DocValuesFormatVariableSkipInterval for testing docvalues skipper index (#13550)
this commit makes possible to configure dynamically the interval size for doc values skipperfor testing, and add a new test suite that changes the interval size randomly.
This commit is contained in:
parent
4baaedaa67
commit
392ddc154f
|
@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene90;
|
||||||
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
|
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
|
||||||
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT;
|
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT;
|
||||||
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE;
|
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE;
|
||||||
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_INTERVAL_SIZE;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -63,10 +62,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
|
||||||
IndexOutput data, meta;
|
IndexOutput data, meta;
|
||||||
final int maxDoc;
|
final int maxDoc;
|
||||||
private byte[] termsDictBuffer;
|
private byte[] termsDictBuffer;
|
||||||
|
private final int skipIndexIntervalSize;
|
||||||
|
|
||||||
/** expert: Creates a new writer */
|
/** expert: Creates a new writer */
|
||||||
public Lucene90DocValuesConsumer(
|
public Lucene90DocValuesConsumer(
|
||||||
SegmentWriteState state,
|
SegmentWriteState state,
|
||||||
|
int skipIndexIntervalSize,
|
||||||
String dataCodec,
|
String dataCodec,
|
||||||
String dataExtension,
|
String dataExtension,
|
||||||
String metaCodec,
|
String metaCodec,
|
||||||
|
@ -96,6 +97,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
|
||||||
state.segmentInfo.getId(),
|
state.segmentInfo.getId(),
|
||||||
state.segmentSuffix);
|
state.segmentSuffix);
|
||||||
maxDoc = state.segmentInfo.maxDoc();
|
maxDoc = state.segmentInfo.maxDoc();
|
||||||
|
this.skipIndexIntervalSize = skipIndexIntervalSize;
|
||||||
success = true;
|
success = true;
|
||||||
} finally {
|
} finally {
|
||||||
if (!success) {
|
if (!success) {
|
||||||
|
@ -239,7 +241,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
|
||||||
for (int i = 0, end = values.docValueCount(); i < end; ++i) {
|
for (int i = 0, end = values.docValueCount(); i < end; ++i) {
|
||||||
accumulator.accumulate(values.nextValue());
|
accumulator.accumulate(values.nextValue());
|
||||||
}
|
}
|
||||||
if (++counter == SKIP_INDEX_INTERVAL_SIZE) {
|
if (++counter == skipIndexIntervalSize) {
|
||||||
globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
|
globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
|
||||||
globalMinValue = Math.min(globalMinValue, accumulator.minValue);
|
globalMinValue = Math.min(globalMinValue, accumulator.minValue);
|
||||||
globalDocCount += accumulator.docCount;
|
globalDocCount += accumulator.docCount;
|
||||||
|
|
|
@ -138,15 +138,27 @@ import org.apache.lucene.util.packed.DirectWriter;
|
||||||
*/
|
*/
|
||||||
public final class Lucene90DocValuesFormat extends DocValuesFormat {
|
public final class Lucene90DocValuesFormat extends DocValuesFormat {
|
||||||
|
|
||||||
|
private final int skipIndexIntervalSize;
|
||||||
|
|
||||||
/** Default constructor. */
|
/** Default constructor. */
|
||||||
public Lucene90DocValuesFormat() {
|
public Lucene90DocValuesFormat() {
|
||||||
|
this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Doc values fields format with specified skipIndexIntervalSize. */
|
||||||
|
public Lucene90DocValuesFormat(int skipIndexIntervalSize) {
|
||||||
super("Lucene90");
|
super("Lucene90");
|
||||||
|
if (skipIndexIntervalSize < 2) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");
|
||||||
|
}
|
||||||
|
this.skipIndexIntervalSize = skipIndexIntervalSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
|
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
|
||||||
return new Lucene90DocValuesConsumer(
|
return new Lucene90DocValuesConsumer(
|
||||||
state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
|
state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -182,6 +194,5 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
|
||||||
static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;
|
static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;
|
||||||
static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;
|
static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;
|
||||||
|
|
||||||
static final int SKIP_INDEX_INTERVAL_SHIFT = 12;
|
private static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096;
|
||||||
static final int SKIP_INDEX_INTERVAL_SIZE = 1 << SKIP_INDEX_INTERVAL_SHIFT;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.lucene.codecs.lucene90;
|
||||||
|
|
||||||
|
import org.apache.lucene.codecs.Codec;
|
||||||
|
import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
|
||||||
|
import org.apache.lucene.tests.util.TestUtil;
|
||||||
|
|
||||||
|
/** Tests Lucene90DocValuesFormat */
|
||||||
|
public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValuesFormatTestCase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Codec getCodec() {
|
||||||
|
return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(2, 1024)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testSkipIndexIntervalSize() {
|
||||||
|
IllegalArgumentException ex =
|
||||||
|
expectThrows(
|
||||||
|
IllegalArgumentException.class,
|
||||||
|
() -> new Lucene90DocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2)));
|
||||||
|
assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue