Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce TestLucene90DocValuesFormatVariableSkipInterval for testing docvalues skipper index (#13550) 

this commit makes possible to configure dynamically the interval size for doc values skipperfor testing, and add a new 
test suite that changes the interval size randomly.
This commit is contained in:
Ignacio Vera 2024-07-09 15:46:15 +02:00 committed by GitHub
parent 4baaedaa67
commit 392ddc154f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 56 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene90;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_INTERVAL_SIZE;
import java.io.IOException;
import java.util.Arrays;
@ -63,10 +62,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
IndexOutput data, meta;
final int maxDoc;
private byte[] termsDictBuffer;
private final int skipIndexIntervalSize;
/** expert: Creates a new writer */
public Lucene90DocValuesConsumer(
SegmentWriteState state,
int skipIndexIntervalSize,
String dataCodec,
String dataExtension,
String metaCodec,
@ -96,6 +97,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
state.segmentInfo.getId(),
state.segmentSuffix);
maxDoc = state.segmentInfo.maxDoc();
this.skipIndexIntervalSize = skipIndexIntervalSize;
success = true;
} finally {
if (!success) {
@ -239,7 +241,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
for (int i = 0, end = values.docValueCount(); i < end; ++i) {
accumulator.accumulate(values.nextValue());
}
if (++counter == SKIP_INDEX_INTERVAL_SIZE) {
if (++counter == skipIndexIntervalSize) {
globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
globalMinValue = Math.min(globalMinValue, accumulator.minValue);
globalDocCount += accumulator.docCount;

17
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
View File

@ -138,15 +138,27 @@ import org.apache.lucene.util.packed.DirectWriter;
*/
public final class Lucene90DocValuesFormat extends DocValuesFormat {
private final int skipIndexIntervalSize;
/** Default constructor. */
public Lucene90DocValuesFormat() {
this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE);
}
/** Doc values fields format with specified skipIndexIntervalSize. */
public Lucene90DocValuesFormat(int skipIndexIntervalSize) {
super("Lucene90");
if (skipIndexIntervalSize < 2) {
throw new IllegalArgumentException(
"skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");
}
this.skipIndexIntervalSize = skipIndexIntervalSize;
}
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
return new Lucene90DocValuesConsumer(
state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
}
@Override
@ -182,6 +194,5 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;
static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;
static final int SKIP_INDEX_INTERVAL_SHIFT = 12;
static final int SKIP_INDEX_INTERVAL_SIZE = 1 << SKIP_INDEX_INTERVAL_SHIFT;
private static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096;
}

38
lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.codecs.lucene90;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
/** Tests Lucene90DocValuesFormat */
public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValuesFormatTestCase {
@Override
protected Codec getCodec() {
return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(2, 1024)));
}
public void testSkipIndexIntervalSize() {
IllegalArgumentException ex =
expectThrows(
IllegalArgumentException.class,
() -> new Lucene90DocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2)));
assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));
}
}