From 6ee19c022c3a24627fed48f8838e713d7e38a172 Mon Sep 17 00:00:00 2001 From: Zhang Chao <80152403@qq.com> Date: Sat, 16 Mar 2024 08:38:53 +0800 Subject: [PATCH] Introduce IORunnable to fix test failure TestIndexWriterOnDiskFull.testAddIndexOnDiskFull (#13172) --- .../lucene86/Lucene86PointsWriter.java | 7 +++-- .../codecs/lucene90/Lucene90PointsWriter.java | 7 +++-- .../org/apache/lucene/util/IORunnable.java | 29 +++++++++++++++++++ .../org/apache/lucene/util/bkd/BKDWriter.java | 28 +++++++----------- .../lucene/util/bkd/Test4BBKDPoints.java | 5 ++-- .../org/apache/lucene/util/bkd/TestBKD.java | 25 ++++++++-------- .../lucene/tests/index/RandomCodec.java | 3 +- 7 files changed, 65 insertions(+), 39 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/util/IORunnable.java diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java index f808fdbf6b7..89555b370ea 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java @@ -33,6 +33,7 @@ import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.bkd.BKDConfig; import org.apache.lucene.util.bkd.BKDWriter; @@ -143,7 +144,7 @@ public class Lucene86PointsWriter extends PointsWriter { values.size())) { if (values instanceof MutablePointTree) { - Runnable finalizer = + IORunnable finalizer = writer.writeField( metaOut, indexOut, dataOut, fieldInfo.name, (MutablePointTree) values); if (finalizer != null) { @@ -172,7 +173,7 @@ public class Lucene86PointsWriter extends PointsWriter { }); // We could have 0 points on merge since all docs with dimensional fields may be deleted: - Runnable finalizer = writer.finish(metaOut, indexOut, dataOut); + IORunnable finalizer = writer.finish(metaOut, indexOut, dataOut); if (finalizer != null) { metaOut.writeInt(fieldInfo.number); finalizer.run(); @@ -267,7 +268,7 @@ public class Lucene86PointsWriter extends PointsWriter { } } - Runnable finalizer = writer.merge(metaOut, indexOut, dataOut, docMaps, pointValues); + IORunnable finalizer = writer.merge(metaOut, indexOut, dataOut, docMaps, pointValues); if (finalizer != null) { metaOut.writeInt(fieldInfo.number); finalizer.run(); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java index cab4f99cf40..e50d6a0fdb5 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java @@ -32,6 +32,7 @@ import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.bkd.BKDConfig; import org.apache.lucene.util.bkd.BKDWriter; @@ -137,7 +138,7 @@ public class Lucene90PointsWriter extends PointsWriter { values.size())) { if (values instanceof MutablePointTree) { - Runnable finalizer = + IORunnable finalizer = writer.writeField( metaOut, indexOut, dataOut, fieldInfo.name, (MutablePointTree) values); if (finalizer != null) { @@ -166,7 +167,7 @@ public class Lucene90PointsWriter extends PointsWriter { }); // We could have 0 points on merge since all docs with dimensional fields may be deleted: - Runnable finalizer = writer.finish(metaOut, indexOut, dataOut); + IORunnable finalizer = writer.finish(metaOut, indexOut, dataOut); if (finalizer != null) { metaOut.writeInt(fieldInfo.number); finalizer.run(); @@ -261,7 +262,7 @@ public class Lucene90PointsWriter extends PointsWriter { } } - Runnable finalizer = writer.merge(metaOut, indexOut, dataOut, docMaps, pointValues); + IORunnable finalizer = writer.merge(metaOut, indexOut, dataOut, docMaps, pointValues); if (finalizer != null) { metaOut.writeInt(fieldInfo.number); finalizer.run(); diff --git a/lucene/core/src/java/org/apache/lucene/util/IORunnable.java b/lucene/core/src/java/org/apache/lucene/util/IORunnable.java new file mode 100644 index 00000000000..34a8534f95f --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/IORunnable.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.io.IOException; + +/** + * A Runnable that may throw an IOException + * + * @see java.lang.Runnable + */ +@FunctionalInterface +public interface IORunnable { + public abstract void run() throws IOException; +} diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index ab869cd6a95..b370f552642 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -18,7 +18,6 @@ package org.apache.lucene.util.bkd; import java.io.Closeable; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -40,6 +39,7 @@ import org.apache.lucene.util.ArrayUtil.ByteArrayComparator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.PriorityQueue; @@ -423,7 +423,7 @@ public class BKDWriter implements Closeable { * before writing them to disk. This method does not use transient disk in order to reorder * points. */ - public Runnable writeField( + public IORunnable writeField( IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, @@ -492,7 +492,7 @@ public class BKDWriter implements Closeable { /* In the 2+D case, we recursively pick the split dimension, compute the * median value and partition other values around it. */ - private Runnable writeFieldNDims( + private IORunnable writeFieldNDims( IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, @@ -556,7 +556,7 @@ public class BKDWriter implements Closeable { /* In the 1D case, we can simply sort points in ascending order and use the * same writing logic as we use at merge time. */ - private Runnable writeField1Dim( + private IORunnable writeField1Dim( IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, @@ -595,7 +595,7 @@ public class BKDWriter implements Closeable { * already sorted values and currently only works when numDims==1. This returns -1 if all * documents containing dimensional values were deleted. */ - public Runnable merge( + public IORunnable merge( IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, @@ -723,7 +723,7 @@ public class BKDWriter implements Closeable { assert (lastDocID = docID) >= 0; // only assign when asserts are enabled } - public Runnable finish() throws IOException { + public IORunnable finish() throws IOException { if (leafCount > 0) { writeLeafBlock(leafCardinality); leafCardinality = 0; @@ -763,11 +763,7 @@ public class BKDWriter implements Closeable { } }; return () -> { - try { - writeIndex(metaOut, indexOut, config.maxPointsInLeafNode, leafNodes, dataStartFP); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + writeIndex(metaOut, indexOut, config.maxPointsInLeafNode, leafNodes, dataStartFP); }; } @@ -879,7 +875,7 @@ public class BKDWriter implements Closeable { * Writes the BKD tree to the provided {@link IndexOutput}s and returns a {@link Runnable} that * writes the index of the tree if at least one point has been added, or {@code null} otherwise. */ - public Runnable finish(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut) + public IORunnable finish(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut) throws IOException { // System.out.println("\nBKDTreeWriter.finish pointCount=" + pointCount + " out=" + out + " // heapWriter=" + heapPointWriter); @@ -974,7 +970,7 @@ public class BKDWriter implements Closeable { return makeWriter(metaOut, indexOut, splitDimensionValues, leafBlockFPs, dataStartFP); } - private Runnable makeWriter( + private IORunnable makeWriter( IndexOutput metaOut, IndexOutput indexOut, byte[] splitDimensionValues, @@ -1006,11 +1002,7 @@ public class BKDWriter implements Closeable { return () -> { // Write index: - try { - writeIndex(metaOut, indexOut, config.maxPointsInLeafNode, leafNodes, dataStartFP); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + writeIndex(metaOut, indexOut, config.maxPointsInLeafNode, leafNodes, dataStartFP); }; } diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test4BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test4BBKDPoints.java index da20f914ad1..3a5fc2cff2b 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test4BBKDPoints.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test4BBKDPoints.java @@ -26,6 +26,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.LuceneTestCase.Monster; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.NumericUtils; // e.g. run like this: ant test -Dtestcase=Test4BBKDPoints -Dtests.nightly=true -Dtests.verbose=true @@ -66,7 +67,7 @@ public class Test4BBKDPoints extends LuceneTestCase { } } IndexOutput out = dir.createOutput("1d.bkd", IOContext.DEFAULT); - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); long indexFP = out.getFilePointer(); finalizer.run(); out.close(); @@ -115,7 +116,7 @@ public class Test4BBKDPoints extends LuceneTestCase { } } IndexOutput out = dir.createOutput("2d.bkd", IOContext.DEFAULT); - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); long indexFP = out.getFilePointer(); finalizer.run(); out.close(); diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index cb15e513bf8..2b88c8d0541 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -41,6 +41,7 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.NumericUtils; @@ -62,7 +63,7 @@ public class TestBKD extends LuceneTestCase { long indexFP; try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); indexFP = out.getFilePointer(); finalizer.run(); } @@ -133,7 +134,7 @@ public class TestBKD extends LuceneTestCase { long indexFP; try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); indexFP = out.getFilePointer(); finalizer.run(); } @@ -228,7 +229,7 @@ public class TestBKD extends LuceneTestCase { long indexFP; try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); indexFP = out.getFilePointer(); finalizer.run(); } @@ -735,7 +736,7 @@ public class TestBKD extends LuceneTestCase { } final int curDocIDBase = lastDocIDBase; docMaps.add(docID1 -> curDocIDBase + docID1); - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); toMerge.add(out.getFilePointer()); finalizer.run(); valuesInThisSeg = TestUtil.nextInt(random(), numValues / 10, numValues / 2); @@ -760,7 +761,7 @@ public class TestBKD extends LuceneTestCase { if (toMerge != null) { if (segCount > 0) { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); toMerge.add(out.getFilePointer()); finalizer.run(); final int curDocIDBase = lastDocIDBase; @@ -783,14 +784,14 @@ public class TestBKD extends LuceneTestCase { readers.add(getPointValues(in)); } out = dir.createOutput("bkd2", IOContext.DEFAULT); - Runnable finalizer = w.merge(out, out, out, docMaps, readers); + IORunnable finalizer = w.merge(out, out, out, docMaps, readers); indexFP = out.getFilePointer(); finalizer.run(); out.close(); in.close(); in = dir.openInput("bkd2", IOContext.DEFAULT); } else { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); indexFP = out.getFilePointer(); finalizer.run(); out.close(); @@ -1204,7 +1205,7 @@ public class TestBKD extends LuceneTestCase { } IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT); - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); long fp = out.getFilePointer(); finalizer.run(); out.close(); @@ -1270,7 +1271,7 @@ public class TestBKD extends LuceneTestCase { } final long indexFP; try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); indexFP = out.getFilePointer(); finalizer.run(); w.close(); @@ -1332,7 +1333,7 @@ public class TestBKD extends LuceneTestCase { } IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT); - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); long fp = out.getFilePointer(); finalizer.run(); out.close(); @@ -1399,7 +1400,7 @@ public class TestBKD extends LuceneTestCase { } IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT); - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); long fp = out.getFilePointer(); finalizer.run(); out.close(); @@ -1467,7 +1468,7 @@ public class TestBKD extends LuceneTestCase { } final long indexFP; try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) { - Runnable finalizer = w.finish(out, out, out); + IORunnable finalizer = w.finish(out, out, out); indexFP = out.getFilePointer(); finalizer.run(); w.close(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java index d9b0920e618..8059ad09cdc 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java @@ -53,6 +53,7 @@ import org.apache.lucene.tests.codecs.bloom.TestBloomFilteredLucenePostings; import org.apache.lucene.tests.codecs.mockrandom.MockRandomPostingsFormat; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.bkd.BKDConfig; import org.apache.lucene.util.bkd.BKDWriter; @@ -149,7 +150,7 @@ public class RandomCodec extends AssertingCodec { // We could have 0 points on merge since all docs with dimensional fields may be // deleted: - Runnable finalizer = writer.finish(metaOut, indexOut, dataOut); + IORunnable finalizer = writer.finish(metaOut, indexOut, dataOut); if (finalizer != null) { metaOut.writeInt(fieldInfo.number); finalizer.run();