mirror of https://github.com/apache/lucene.git
Decrease test time for TestManyKnnDocs.testLargeSegment (#11945)
* Improve speed of TestManyKnnDocs
This commit is contained in:
parent
b6ebfd1861
commit
a18b62ded4
|
@ -23,19 +23,20 @@ import org.apache.lucene.search.KnnVectorQuery;
|
||||||
import org.apache.lucene.search.TopDocs;
|
import org.apache.lucene.search.TopDocs;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.FSDirectory;
|
import org.apache.lucene.store.FSDirectory;
|
||||||
|
import org.apache.lucene.tests.codecs.vector.ConfigurableMCodec;
|
||||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||||
import org.apache.lucene.tests.util.LuceneTestCase.Monster;
|
import org.apache.lucene.tests.util.LuceneTestCase.Monster;
|
||||||
import org.apache.lucene.tests.util.TestUtil;
|
|
||||||
|
|
||||||
@TimeoutSuite(millis = 86_400_000) // 24 hour timeout
|
@TimeoutSuite(millis = 86_400_000) // 24 hour timeout
|
||||||
@Monster("takes ~2 hours and needs extra heap, disk space, file handles")
|
@Monster("takes ~10 minutes and needs extra heap, disk space, file handles")
|
||||||
public class TestManyKnnDocs extends LuceneTestCase {
|
public class TestManyKnnDocs extends LuceneTestCase {
|
||||||
// gradlew -p lucene/core test --tests TestManyKnnDocs -Ptests.heapsize=16g -Dtests.monster=true
|
// gradlew -p lucene/core test --tests TestManyKnnDocs -Ptests.heapsize=16g -Dtests.monster=true
|
||||||
|
|
||||||
public void testLargeSegment() throws Exception {
|
public void testLargeSegment() throws Exception {
|
||||||
IndexWriterConfig iwc = new IndexWriterConfig();
|
IndexWriterConfig iwc = new IndexWriterConfig();
|
||||||
iwc.setCodec(
|
iwc.setCodec(
|
||||||
TestUtil.getDefaultCodec()); // Make sure to use the default codec instead of a random one
|
new ConfigurableMCodec(
|
||||||
|
128)); // Make sure to use the ConfigurableMCodec instead of a random one
|
||||||
iwc.setRAMBufferSizeMB(64); // Use a 64MB buffer to create larger initial segments
|
iwc.setRAMBufferSizeMB(64); // Use a 64MB buffer to create larger initial segments
|
||||||
TieredMergePolicy mp = new TieredMergePolicy();
|
TieredMergePolicy mp = new TieredMergePolicy();
|
||||||
mp.setMaxMergeAtOnce(256); // avoid intermediate merges (waste of time with HNSW?)
|
mp.setMaxMergeAtOnce(256); // avoid intermediate merges (waste of time with HNSW?)
|
||||||
|
@ -47,7 +48,7 @@ public class TestManyKnnDocs extends LuceneTestCase {
|
||||||
try (Directory dir = FSDirectory.open(createTempDir("ManyKnnVectorDocs"));
|
try (Directory dir = FSDirectory.open(createTempDir("ManyKnnVectorDocs"));
|
||||||
IndexWriter iw = new IndexWriter(dir, iwc)) {
|
IndexWriter iw = new IndexWriter(dir, iwc)) {
|
||||||
|
|
||||||
int numVectors = 16268816;
|
int numVectors = 2088992;
|
||||||
float[] vector = new float[1];
|
float[] vector = new float[1];
|
||||||
Document doc = new Document();
|
Document doc = new Document();
|
||||||
doc.add(new KnnVectorField(fieldName, vector, similarityFunction));
|
doc.add(new KnnVectorField(fieldName, vector, similarityFunction));
|
||||||
|
|
|
@ -1500,8 +1500,7 @@ public class Plane extends Vector {
|
||||||
} else {
|
} else {
|
||||||
// Since a==b==0, any plane including the Z axis suffices.
|
// Since a==b==0, any plane including the Z axis suffices.
|
||||||
// System.err.println(" Perpendicular to z");
|
// System.err.println(" Perpendicular to z");
|
||||||
GeoPoint[] points =
|
GeoPoint[] points = findIntersections(planetModel, normalYPlane, NO_BOUNDS, NO_BOUNDS);
|
||||||
findIntersections(planetModel, normalYPlane, NO_BOUNDS, NO_BOUNDS);
|
|
||||||
if (points.length == 0) {
|
if (points.length == 0) {
|
||||||
points = findIntersections(planetModel, normalXPlane, NO_BOUNDS, NO_BOUNDS);
|
points = findIntersections(planetModel, normalXPlane, NO_BOUNDS, NO_BOUNDS);
|
||||||
}
|
}
|
||||||
|
@ -2047,8 +2046,7 @@ public class Plane extends Vector {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Horizontal circle. Since a==b, any vertical plane suffices.
|
// Horizontal circle. Since a==b, any vertical plane suffices.
|
||||||
GeoPoint[] points =
|
GeoPoint[] points = findIntersections(planetModel, normalXPlane, NO_BOUNDS, NO_BOUNDS);
|
||||||
findIntersections(planetModel, normalXPlane, NO_BOUNDS, NO_BOUNDS);
|
|
||||||
if (points.length == 0) {
|
if (points.length == 0) {
|
||||||
points = findIntersections(planetModel, normalYPlane, NO_BOUNDS, NO_BOUNDS);
|
points = findIntersections(planetModel, normalYPlane, NO_BOUNDS, NO_BOUNDS);
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,6 +40,7 @@ module org.apache.lucene.test_framework {
|
||||||
exports org.apache.lucene.tests.codecs.ramonly;
|
exports org.apache.lucene.tests.codecs.ramonly;
|
||||||
exports org.apache.lucene.tests.codecs.uniformsplit.sharedterms;
|
exports org.apache.lucene.tests.codecs.uniformsplit.sharedterms;
|
||||||
exports org.apache.lucene.tests.codecs.uniformsplit;
|
exports org.apache.lucene.tests.codecs.uniformsplit;
|
||||||
|
exports org.apache.lucene.tests.codecs.vector;
|
||||||
exports org.apache.lucene.tests.geo;
|
exports org.apache.lucene.tests.geo;
|
||||||
exports org.apache.lucene.tests.index;
|
exports org.apache.lucene.tests.index;
|
||||||
exports org.apache.lucene.tests.mockfile;
|
exports org.apache.lucene.tests.mockfile;
|
||||||
|
@ -58,7 +59,8 @@ module org.apache.lucene.test_framework {
|
||||||
org.apache.lucene.tests.codecs.compressing.FastDecompressionCompressingCodec,
|
org.apache.lucene.tests.codecs.compressing.FastDecompressionCompressingCodec,
|
||||||
org.apache.lucene.tests.codecs.compressing.HighCompressionCompressingCodec,
|
org.apache.lucene.tests.codecs.compressing.HighCompressionCompressingCodec,
|
||||||
org.apache.lucene.tests.codecs.compressing.LZ4WithPresetCompressingCodec,
|
org.apache.lucene.tests.codecs.compressing.LZ4WithPresetCompressingCodec,
|
||||||
org.apache.lucene.tests.codecs.compressing.dummy.DummyCompressingCodec;
|
org.apache.lucene.tests.codecs.compressing.dummy.DummyCompressingCodec,
|
||||||
|
org.apache.lucene.tests.codecs.vector.ConfigurableMCodec;
|
||||||
provides org.apache.lucene.codecs.DocValuesFormat with
|
provides org.apache.lucene.codecs.DocValuesFormat with
|
||||||
org.apache.lucene.tests.codecs.asserting.AssertingDocValuesFormat;
|
org.apache.lucene.tests.codecs.asserting.AssertingDocValuesFormat;
|
||||||
provides org.apache.lucene.codecs.KnnVectorsFormat with
|
provides org.apache.lucene.codecs.KnnVectorsFormat with
|
||||||
|
|
|
@ -0,0 +1,47 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.lucene.tests.codecs.vector;
|
||||||
|
|
||||||
|
import org.apache.lucene.codecs.FilterCodec;
|
||||||
|
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||||
|
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
|
||||||
|
import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This codec allows customization of the number of connections made for an hnsw index. Increasing
|
||||||
|
* the number of connections can decrease the time of certain tests while still achieving the same
|
||||||
|
* test coverage.
|
||||||
|
*/
|
||||||
|
public class ConfigurableMCodec extends FilterCodec {
|
||||||
|
|
||||||
|
private final KnnVectorsFormat knnVectorsFormat;
|
||||||
|
|
||||||
|
public ConfigurableMCodec() {
|
||||||
|
super("ConfigurableMCodec", new Lucene94Codec());
|
||||||
|
knnVectorsFormat = new Lucene94HnswVectorsFormat(128, 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ConfigurableMCodec(int maxConn) {
|
||||||
|
super("ConfigurableMCodec", new Lucene94Codec());
|
||||||
|
knnVectorsFormat = new Lucene94HnswVectorsFormat(maxConn, 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public KnnVectorsFormat knnVectorsFormat() {
|
||||||
|
return knnVectorsFormat;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Codecs for testing different vector formats. */
|
||||||
|
package org.apache.lucene.tests.codecs.vector;
|
|
@ -21,3 +21,4 @@ org.apache.lucene.tests.codecs.compressing.FastDecompressionCompressingCodec
|
||||||
org.apache.lucene.tests.codecs.compressing.HighCompressionCompressingCodec
|
org.apache.lucene.tests.codecs.compressing.HighCompressionCompressingCodec
|
||||||
org.apache.lucene.tests.codecs.compressing.LZ4WithPresetCompressingCodec
|
org.apache.lucene.tests.codecs.compressing.LZ4WithPresetCompressingCodec
|
||||||
org.apache.lucene.tests.codecs.compressing.dummy.DummyCompressingCodec
|
org.apache.lucene.tests.codecs.compressing.dummy.DummyCompressingCodec
|
||||||
|
org.apache.lucene.tests.codecs.vector.ConfigurableMCodec
|
||||||
|
|
Loading…
Reference in New Issue