From 392ddc154f229e141ddbf32dde945b0fefe99b0e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 9 Jul 2024 15:46:15 +0200 Subject: [PATCH] Introduce TestLucene90DocValuesFormatVariableSkipInterval for testing docvalues skipper index (#13550) this commit makes possible to configure dynamically the interval size for doc values skipperfor testing, and add a new test suite that changes the interval size randomly. --- .../lucene90/Lucene90DocValuesConsumer.java | 6 ++- .../lucene90/Lucene90DocValuesFormat.java | 17 +++++++-- ...90DocValuesFormatVariableSkipInterval.java | 38 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java index 63e4891960c..021eacacd3b 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java @@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene90; import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT; import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE; -import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_INTERVAL_SIZE; import java.io.IOException; import java.util.Arrays; @@ -63,10 +62,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer { IndexOutput data, meta; final int maxDoc; private byte[] termsDictBuffer; + private final int skipIndexIntervalSize; /** expert: Creates a new writer */ public Lucene90DocValuesConsumer( SegmentWriteState state, + int skipIndexIntervalSize, String dataCodec, String dataExtension, String metaCodec, @@ -96,6 +97,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer { state.segmentInfo.getId(), state.segmentSuffix); maxDoc = state.segmentInfo.maxDoc(); + this.skipIndexIntervalSize = skipIndexIntervalSize; success = true; } finally { if (!success) { @@ -239,7 +241,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer { for (int i = 0, end = values.docValueCount(); i < end; ++i) { accumulator.accumulate(values.nextValue()); } - if (++counter == SKIP_INDEX_INTERVAL_SIZE) { + if (++counter == skipIndexIntervalSize) { globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue); globalMinValue = Math.min(globalMinValue, accumulator.minValue); globalDocCount += accumulator.docCount; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java index 847a5341584..0ae0a7ac2aa 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java @@ -138,15 +138,27 @@ import org.apache.lucene.util.packed.DirectWriter; */ public final class Lucene90DocValuesFormat extends DocValuesFormat { + private final int skipIndexIntervalSize; + /** Default constructor. */ public Lucene90DocValuesFormat() { + this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE); + } + + /** Doc values fields format with specified skipIndexIntervalSize. */ + public Lucene90DocValuesFormat(int skipIndexIntervalSize) { super("Lucene90"); + if (skipIndexIntervalSize < 2) { + throw new IllegalArgumentException( + "skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]"); + } + this.skipIndexIntervalSize = skipIndexIntervalSize; } @Override public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { return new Lucene90DocValuesConsumer( - state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION); + state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION); } @Override @@ -182,6 +194,5 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat { static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT; static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1; - static final int SKIP_INDEX_INTERVAL_SHIFT = 12; - static final int SKIP_INDEX_INTERVAL_SIZE = 1 << SKIP_INDEX_INTERVAL_SHIFT; + private static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096; } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java new file mode 100644 index 00000000000..94204fb2a8d --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene90; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; + +/** Tests Lucene90DocValuesFormat */ +public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValuesFormatTestCase { + + @Override + protected Codec getCodec() { + return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(2, 1024))); + } + + public void testSkipIndexIntervalSize() { + IllegalArgumentException ex = + expectThrows( + IllegalArgumentException.class, + () -> new Lucene90DocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2))); + assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1")); + } +}