Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-10155: Refactor TestMultiMMap into a BaseChunkedDirectoryTestCase (#360) 

BaseChunkedDirectoryTestCase is an extension of BaseDirectoryTestCase
where the concrete test class instantiates with a specified chunk size.
It then tries to test boundary conditions around all the chunking.
This commit is contained in:
Robert Muir 2021-10-09 11:55:41 -04:00 committed by GitHub
parent 61c15c8c10
commit 6c6a3bd5bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 510 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java
View File

@ -80,9 +80,16 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountab
}
}
/** Default {@code minBitsPerBlock} */
public static final int DEFAULT_MIN_BITS_PER_BLOCK = 10; // 1024 B
/** Default {@code maxBitsPerBlock} */
public static final int DEFAULT_MAX_BITS_PER_BLOCK = 26; // 64 MB
/** Smallest {@code minBitsPerBlock} allowed */
public static final int LIMIT_MIN_BITS_PER_BLOCK = 1;
/** Largest {@code maxBitsPerBlock} allowed */
public static final int LIMIT_MAX_BITS_PER_BLOCK = 31;
/**
* Maximum number of blocks at the current {@link #blockBits} block size before we increase the
* block size (and thus decrease the number of blocks).
@ -110,6 +117,14 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountab
/** The current-or-next write block. */
private ByteBuffer currentBlock = EMPTY;
/**
* Create a new output, suitable for writing a file of around {@code expectedSize} bytes.
*
* <p>Memory allocation will be optimized based on the {@code expectedSize} hint, so that there is
* less overhead for larger files.
*
* @param expectedSize estimated size of the output file
*/
public ByteBuffersDataOutput(long expectedSize) {
this(
computeBlockSizeBitsFor(expectedSize),
@ -118,18 +133,47 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountab
NO_REUSE);
}
/** Creates a new output with all defaults. */
public ByteBuffersDataOutput() {
this(DEFAULT_MIN_BITS_PER_BLOCK, DEFAULT_MAX_BITS_PER_BLOCK, ALLOCATE_BB_ON_HEAP, NO_REUSE);
}
/**
* Expert: Creates a new output with custom parameters.
*
* @param minBitsPerBlock minimum bits per block
* @param maxBitsPerBlock maximum bits per block
* @param blockAllocate block allocator
* @param blockReuse block recycler
*/
public ByteBuffersDataOutput(
int minBitsPerBlock,
int maxBitsPerBlock,
IntFunction<ByteBuffer> blockAllocate,
Consumer<ByteBuffer> blockReuse) {
if (minBitsPerBlock < 10 || minBitsPerBlock > maxBitsPerBlock || maxBitsPerBlock > 31) {
if (minBitsPerBlock < LIMIT_MIN_BITS_PER_BLOCK) {
throw new IllegalArgumentException(
String.format(Locale.ROOT, "Invalid arguments: %s %s", minBitsPerBlock, maxBitsPerBlock));
String.format(
Locale.ROOT,
"minBitsPerBlock (%s) too small, must be at least %s",
minBitsPerBlock,
LIMIT_MIN_BITS_PER_BLOCK));
}
if (maxBitsPerBlock > LIMIT_MAX_BITS_PER_BLOCK) {
throw new IllegalArgumentException(
String.format(
Locale.ROOT,
"maxBitsPerBlock (%s) too large, must not exceed %s",
maxBitsPerBlock,
LIMIT_MAX_BITS_PER_BLOCK));
}
if (minBitsPerBlock > maxBitsPerBlock) {
throw new IllegalArgumentException(
String.format(
Locale.ROOT,
"minBitsPerBlock (%s) cannot exceed maxBitsPerBlock (%s)",
minBitsPerBlock,
maxBitsPerBlock));
}
this.maxBitsPerBlock = maxBitsPerBlock;
this.blockBits = minBitsPerBlock;

8
lucene/core/src/test/org/apache/lucene/store/BaseDataOutputTestCase.java
View File

@ -16,9 +16,6 @@
*/
package org.apache.lucene.store;
import static org.junit.Assert.*;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.Xoroshiro128PlusRandom;
import com.carrotsearch.randomizedtesting.generators.RandomBytes;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
@ -32,9 +29,10 @@ import java.util.List;
import java.util.Random;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils.IOConsumer;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
public abstract class BaseDataOutputTestCase<T extends DataOutput> extends RandomizedTest {
public abstract class BaseDataOutputTestCase<T extends DataOutput> extends LuceneTestCase {
protected abstract T newInstance();
protected abstract byte[] toBytes(T instance);
@ -50,7 +48,7 @@ public abstract class BaseDataOutputTestCase<T extends DataOutput> extends Rando
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutput ref = new OutputStreamDataOutput(baos);
long seed = randomLong();
long seed = random().nextLong();
int max = 50_000;
addRandomData(dst, new Xoroshiro128PlusRandom(seed), max);
addRandomData(ref, new Xoroshiro128PlusRandom(seed), max);

78
lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java
View File

@ -16,8 +16,6 @@
*/
package org.apache.lucene.store;
import static org.junit.Assert.*;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
@ -26,6 +24,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.TestUtil;
import org.junit.Assert;
import org.junit.Test;
@ -58,8 +57,8 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
reuser::reuse);
// Add some random data first.
long genSeed = randomLong();
int addCount = randomIntBetween(1000, 5000);
long genSeed = random().nextLong();
int addCount = TestUtil.nextInt(random(), 1000, 5000);
addRandomData(o, new Random(genSeed), addCount);
byte[] data = o.toArrayCopy();
@ -84,7 +83,7 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
{
long MB = 1024 * 1024;
long expectedSize = randomLongBetween(MB, MB * 1024);
long expectedSize = TestUtil.nextLong(random(), MB, MB * 1024);
ByteBuffersDataOutput o = new ByteBuffersDataOutput(expectedSize);
o.writeByte((byte) 0);
int cap = o.toBufferList().get(0).capacity();
@ -96,6 +95,63 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
}
}
public void testIllegalMinBitsPerBlock() {
expectThrows(
IllegalArgumentException.class,
() -> {
new ByteBuffersDataOutput(
ByteBuffersDataOutput.LIMIT_MIN_BITS_PER_BLOCK - 1,
ByteBuffersDataOutput.DEFAULT_MAX_BITS_PER_BLOCK,
ByteBuffersDataOutput.ALLOCATE_BB_ON_HEAP,
ByteBuffersDataOutput.NO_REUSE);
});
}
public void testIllegalMaxBitsPerBlock() {
expectThrows(
IllegalArgumentException.class,
() -> {
new ByteBuffersDataOutput(
ByteBuffersDataOutput.DEFAULT_MIN_BITS_PER_BLOCK,
ByteBuffersDataOutput.LIMIT_MAX_BITS_PER_BLOCK + 1,
ByteBuffersDataOutput.ALLOCATE_BB_ON_HEAP,
ByteBuffersDataOutput.NO_REUSE);
});
}
public void testIllegalBitsPerBlockRange() {
expectThrows(
IllegalArgumentException.class,
() -> {
new ByteBuffersDataOutput(
20, 19, ByteBuffersDataOutput.ALLOCATE_BB_ON_HEAP, ByteBuffersDataOutput.NO_REUSE);
});
}
public void testNullAllocator() {
expectThrows(
NullPointerException.class,
() -> {
new ByteBuffersDataOutput(
ByteBuffersDataOutput.DEFAULT_MIN_BITS_PER_BLOCK,
ByteBuffersDataOutput.DEFAULT_MAX_BITS_PER_BLOCK,
null,
ByteBuffersDataOutput.NO_REUSE);
});
}
public void testNullRecycler() {
expectThrows(
NullPointerException.class,
() -> {
new ByteBuffersDataOutput(
ByteBuffersDataOutput.DEFAULT_MIN_BITS_PER_BLOCK,
ByteBuffersDataOutput.DEFAULT_MAX_BITS_PER_BLOCK,
ByteBuffersDataOutput.ALLOCATE_BB_ON_HEAP,
null);
});
}
@Test
public void testSanity() {
ByteBuffersDataOutput o = newInstance();
@ -116,9 +172,10 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
@Test
public void testWriteByteBuffer() {
ByteBuffersDataOutput o = new ByteBuffersDataOutput();
byte[] bytes = randomBytesOfLength(1024 * 8 + 10);
byte[] bytes = new byte[1024 * 8 + 10];
random().nextBytes(bytes);
ByteBuffer src = ByteBuffer.wrap(bytes);
int offset = randomIntBetween(0, 100);
int offset = TestUtil.nextInt(random(), 0, 100);
int len = bytes.length - offset;
src.position(offset);
src.limit(offset + len);
@ -134,11 +191,12 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
int MB = 1024 * 1024;
final byte[] bytes;
if (LuceneTestCase.TEST_NIGHTLY) {
bytes = randomBytesOfLength(5 * MB, 15 * MB);
bytes = new byte[TestUtil.nextInt(random(), 5 * MB, 15 * MB)];
} else {
bytes = randomBytesOfLength(MB / 2, MB);
bytes = new byte[TestUtil.nextInt(random(), MB / 2, MB)];
}
int offset = randomIntBetween(0, 100);
random().nextBytes(bytes);
int offset = TestUtil.nextInt(random(), 0, 100);
int len = bytes.length - offset;
o.writeBytes(bytes, offset, len);
assertEquals(len, o.size());

49
lucene/core/src/test/org/apache/lucene/store/TestMultiByteBuffersDirectory.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
import java.util.function.Supplier;
import org.apache.lucene.util.BitUtil;
/** Tests ByteBuffersDirectory's chunking */
public class TestMultiByteBuffersDirectory extends BaseChunkedDirectoryTestCase {
@Override
protected Directory getDirectory(Path path, int maxChunkSize) throws IOException {
// round down huge values (above 20) to keep RAM usage low in tests (especially in nightly)
int bitsPerBlock =
Math.min(
20,
Math.max(
ByteBuffersDataOutput.LIMIT_MIN_BITS_PER_BLOCK,
Integer.numberOfTrailingZeros(BitUtil.nextHighestPowerOfTwo(maxChunkSize))));
Supplier<ByteBuffersDataOutput> outputSupplier =
() -> {
return new ByteBuffersDataOutput(
bitsPerBlock,
bitsPerBlock,
ByteBuffersDataOutput.ALLOCATE_BB_ON_HEAP,
ByteBuffersDataOutput.NO_REUSE);
};
return new ByteBuffersDirectory(
new SingleInstanceLockFactory(),
outputSupplier,
ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS);
}
}

311
lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
View File

@ -18,14 +18,7 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TestUtil;
/**
* Tests MMapDirectory's MultiMMapIndexInput
@ -33,11 +26,11 @@ import org.apache.lucene.util.TestUtil;
* <p>Because Java's ByteBuffer uses an int to address the values, it's necessary to access a file
* &gt; Integer.MAX_VALUE in size using multiple byte buffers.
*/
public class TestMultiMMap extends BaseDirectoryTestCase {
public class TestMultiMMap extends BaseChunkedDirectoryTestCase {
@Override
protected Directory getDirectory(Path path) throws IOException {
return new MMapDirectory(path, 1 << TestUtil.nextInt(random(), 10, 28));
protected Directory getDirectory(Path path, int maxChunkSize) throws IOException {
return new MMapDirectory(path, maxChunkSize);
}
@Override
@ -46,8 +39,11 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
assumeTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
}
// TODO: can we improve ByteBuffersDirectory (without overhead) and move these clone safety tests
// to the base test case?
public void testCloneSafety() throws Exception {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testCloneSafety"));
Directory mmapDir = getDirectory(createTempDir("testCloneSafety"));
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
io.writeVInt(5);
io.close();
@ -78,29 +74,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
mmapDir.close();
}
public void testCloneClose() throws Exception {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testCloneClose"));
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
io.writeVInt(5);
io.close();
IndexInput one = mmapDir.openInput("bytes", IOContext.DEFAULT);
IndexInput two = one.clone();
IndexInput three = two.clone(); // clone of clone
two.close();
assertEquals(5, one.readVInt());
expectThrows(
AlreadyClosedException.class,
() -> {
two.readVInt();
});
assertEquals(5, three.readVInt());
one.close();
three.close();
mmapDir.close();
}
public void testCloneSliceSafety() throws Exception {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testCloneSliceSafety"));
Directory mmapDir = getDirectory(createTempDir("testCloneSliceSafety"));
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
io.writeInt(1);
io.writeInt(2);
@ -141,238 +116,11 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
mmapDir.close();
}
public void testCloneSliceClose() throws Exception {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testCloneSliceClose"));
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
io.writeInt(1);
io.writeInt(2);
io.close();
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput one = slicer.slice("first int", 0, 4);
IndexInput two = slicer.slice("second int", 4, 4);
one.close();
expectThrows(
AlreadyClosedException.class,
() -> {
one.readInt();
});
assertEquals(2, two.readInt());
// reopen a new slice "another":
IndexInput another = slicer.slice("first int", 0, 4);
assertEquals(1, another.readInt());
another.close();
two.close();
slicer.close();
mmapDir.close();
}
public void testSeekZero() throws Exception {
int upto = TEST_NIGHTLY ? 31 : 3;
for (int i = 0; i < upto; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekZero"), 1 << i);
IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random()));
io.close();
IndexInput ii = mmapDir.openInput("zeroBytes", newIOContext(random()));
ii.seek(0L);
ii.close();
mmapDir.close();
}
}
public void testSeekSliceZero() throws Exception {
int upto = TEST_NIGHTLY ? 31 : 3;
for (int i = 0; i < upto; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceZero"), 1 << i);
IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random()));
io.close();
IndexInput slicer = mmapDir.openInput("zeroBytes", newIOContext(random()));
IndexInput ii = slicer.slice("zero-length slice", 0, 0);
ii.seek(0L);
ii.close();
slicer.close();
mmapDir.close();
}
}
public void testSeekEnd() throws Exception {
for (int i = 0; i < 17; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekEnd"), 1 << i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << i];
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = mmapDir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << i];
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
ii.seek(1 << i);
ii.close();
mmapDir.close();
}
}
public void testSeekSliceEnd() throws Exception {
for (int i = 0; i < 17; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceEnd"), 1 << i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << i];
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput ii = slicer.slice("full slice", 0, bytes.length);
byte[] actual = new byte[1 << i];
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
ii.seek(1 << i);
ii.close();
slicer.close();
mmapDir.close();
}
}
public void testSeeking() throws Exception {
int numIters = TEST_NIGHTLY ? 10 : 1;
for (int i = 0; i < numIters; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeeking"), 1 << i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << (i + 1)]; // make sure we switch buffers
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = mmapDir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << (i + 1)]; // first read all bytes
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) {
for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) {
byte[] slice = new byte[sliceLength];
ii.seek(sliceStart);
ii.readBytes(slice, 0, slice.length);
assertEquals(new BytesRef(bytes, sliceStart, sliceLength), new BytesRef(slice));
}
}
ii.close();
mmapDir.close();
}
}
// note instead of seeking to offset and reading length, this opens slices at the
// the various offset+length and just does readBytes.
public void testSlicedSeeking() throws Exception {
int numIters = TEST_NIGHTLY ? 10 : 1;
for (int i = 0; i < numIters; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSlicedSeeking"), 1 << i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << (i + 1)]; // make sure we switch buffers
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = mmapDir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << (i + 1)]; // first read all bytes
ii.readBytes(actual, 0, actual.length);
ii.close();
assertEquals(new BytesRef(bytes), new BytesRef(actual));
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) {
for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) {
assertSlice(bytes, slicer, 0, sliceStart, sliceLength);
}
}
slicer.close();
mmapDir.close();
}
}
@Override
public void testSliceOfSlice() throws Exception {
int upto = TEST_NIGHTLY ? 10 : 8;
for (int i = 0; i < upto; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSliceOfSlice"), 1 << i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << (i + 1)]; // make sure we switch buffers
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = mmapDir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << (i + 1)]; // first read all bytes
ii.readBytes(actual, 0, actual.length);
ii.close();
assertEquals(new BytesRef(bytes), new BytesRef(actual));
IndexInput outerSlicer = mmapDir.openInput("bytes", newIOContext(random()));
final int outerSliceStart = random().nextInt(bytes.length / 2);
final int outerSliceLength = random().nextInt(bytes.length - outerSliceStart);
IndexInput innerSlicer =
outerSlicer.slice("parentBytesSlice", outerSliceStart, outerSliceLength);
for (int sliceStart = 0; sliceStart < outerSliceLength; sliceStart++) {
for (int sliceLength = 0; sliceLength < outerSliceLength - sliceStart; sliceLength++) {
assertSlice(bytes, innerSlicer, outerSliceStart, sliceStart, sliceLength);
}
}
innerSlicer.close();
outerSlicer.close();
mmapDir.close();
}
}
private void assertSlice(
byte[] bytes, IndexInput slicer, int outerSliceStart, int sliceStart, int sliceLength)
throws IOException {
byte[] slice = new byte[sliceLength];
IndexInput input = slicer.slice("bytesSlice", sliceStart, slice.length);
input.readBytes(slice, 0, slice.length);
input.close();
assertEquals(
new BytesRef(bytes, outerSliceStart + sliceStart, sliceLength), new BytesRef(slice));
}
public void testRandomChunkSizes() throws Exception {
int num = TEST_NIGHTLY ? atLeast(10) : 3;
for (int i = 0; i < num; i++) {
assertChunking(random(), TestUtil.nextInt(random(), 20, 100));
}
}
private void assertChunking(Random random, int chunkSize) throws Exception {
Path path = createTempDir("mmap" + chunkSize);
MMapDirectory mmapDir = new MMapDirectory(path, chunkSize);
// we will map a lot, try to turn on the unmap hack
if (MMapDirectory.UNMAP_SUPPORTED) mmapDir.setUseUnmap(true);
MockDirectoryWrapper dir = new MockDirectoryWrapper(random, mmapDir);
RandomIndexWriter writer =
new RandomIndexWriter(
random,
dir,
newIndexWriterConfig(new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
Document doc = new Document();
Field docid = newStringField("docid", "0", Field.Store.YES);
Field junk = newStringField("junk", "", Field.Store.YES);
doc.add(docid);
doc.add(junk);
int numDocs = 100;
for (int i = 0; i < numDocs; i++) {
docid.setStringValue("" + i);
junk.setStringValue(TestUtil.randomUnicodeString(random));
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
int numAsserts = atLeast(100);
for (int i = 0; i < numAsserts; i++) {
int docID = random.nextInt(numDocs);
assertEquals("" + docID, reader.document(docID).get("docid"));
}
reader.close();
dir.close();
}
// test has asserts specific to mmap impl...
public void testImplementations() throws Exception {
for (int i = 2; i < 12; i++) {
final int chunkSize = 1 << i;
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testImplementations"), chunkSize);
Directory mmapDir = getDirectory(createTempDir("testImplementations"), chunkSize);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
int size = random().nextInt(chunkSize * 2) + 3; // add some buffer of 3 for slice tests
byte[] bytes = new byte[size];
@ -420,43 +168,4 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
mmapDir.close();
}
}
public void testLittleEndianLongsCrossBoundary() throws Exception {
try (Directory dir =
new MMapDirectory(createTempDir("testLittleEndianLongsCrossBoundary"), 16)) {
try (IndexOutput out = dir.createOutput("littleEndianLongs", newIOContext(random()))) {
out.writeByte((byte) 2);
out.writeLong(3L);
out.writeLong(Long.MAX_VALUE);
out.writeLong(-3L);
}
try (IndexInput input = dir.openInput("littleEndianLongs", newIOContext(random()))) {
assertEquals(25, input.length());
assertEquals(2, input.readByte());
long[] l = new long[4];
input.readLongs(l, 1, 3);
assertArrayEquals(new long[] {0L, 3L, Long.MAX_VALUE, -3L}, l);
assertEquals(25, input.getFilePointer());
}
}
}
public void testLittleEndianFloatsCrossBoundary() throws Exception {
try (Directory dir = new MMapDirectory(createTempDir("testFloatsCrossBoundary"), 8)) {
try (IndexOutput out = dir.createOutput("Floats", newIOContext(random()))) {
out.writeByte((byte) 2);
out.writeInt(Float.floatToIntBits(3f));
out.writeInt(Float.floatToIntBits(Float.MAX_VALUE));
out.writeInt(Float.floatToIntBits(-3f));
}
try (IndexInput input = dir.openInput("Floats", newIOContext(random()))) {
assertEquals(13, input.length());
assertEquals(2, input.readByte());
float[] ff = new float[4];
input.readFloats(ff, 1, 3);
assertArrayEquals(new float[] {0, 3f, Float.MAX_VALUE, -3f}, ff, 0);
assertEquals(13, input.getFilePointer());
}
}
}
}

334
lucene/test-framework/src/java/org/apache/lucene/store/BaseChunkedDirectoryTestCase.java Normal file
View File

@ -0,0 +1,334 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TestUtil;
/**
* Base class for Directories that "chunk" the input into blocks.
*
* <p>It tries to explicitly chunk with different sizes and test boundary conditions around the
* chunks.
*/
public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(Path path) throws IOException {
return getDirectory(path, 1 << TestUtil.nextInt(random(), 10, 20));
}
/** Creates a new directory with the specified max chunk size */
protected abstract Directory getDirectory(Path path, int maxChunkSize) throws IOException;
public void testCloneClose() throws Exception {
Directory dir = getDirectory(createTempDir("testCloneClose"));
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
io.writeVInt(5);
io.close();
IndexInput one = dir.openInput("bytes", IOContext.DEFAULT);
IndexInput two = one.clone();
IndexInput three = two.clone(); // clone of clone
two.close();
assertEquals(5, one.readVInt());
expectThrows(
AlreadyClosedException.class,
() -> {
two.readVInt();
});
assertEquals(5, three.readVInt());
one.close();
three.close();
dir.close();
}
public void testCloneSliceClose() throws Exception {
Directory dir = getDirectory(createTempDir("testCloneSliceClose"));
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
io.writeInt(1);
io.writeInt(2);
io.close();
IndexInput slicer = dir.openInput("bytes", newIOContext(random()));
IndexInput one = slicer.slice("first int", 0, 4);
IndexInput two = slicer.slice("second int", 4, 4);
one.close();
expectThrows(
AlreadyClosedException.class,
() -> {
one.readInt();
});
assertEquals(2, two.readInt());
// reopen a new slice "another":
IndexInput another = slicer.slice("first int", 0, 4);
assertEquals(1, another.readInt());
another.close();
two.close();
slicer.close();
dir.close();
}
public void testSeekZero() throws Exception {
int upto = TEST_NIGHTLY ? 31 : 3;
for (int i = 0; i < upto; i++) {
Directory dir = getDirectory(createTempDir("testSeekZero"), 1 << i);
IndexOutput io = dir.createOutput("zeroBytes", newIOContext(random()));
io.close();
IndexInput ii = dir.openInput("zeroBytes", newIOContext(random()));
ii.seek(0L);
ii.close();
dir.close();
}
}
public void testSeekSliceZero() throws Exception {
int upto = TEST_NIGHTLY ? 31 : 3;
for (int i = 0; i < upto; i++) {
Directory dir = getDirectory(createTempDir("testSeekSliceZero"), 1 << i);
IndexOutput io = dir.createOutput("zeroBytes", newIOContext(random()));
io.close();
IndexInput slicer = dir.openInput("zeroBytes", newIOContext(random()));
IndexInput ii = slicer.slice("zero-length slice", 0, 0);
ii.seek(0L);
ii.close();
slicer.close();
dir.close();
}
}
public void testSeekEnd() throws Exception {
for (int i = 0; i < 17; i++) {
Directory dir = getDirectory(createTempDir("testSeekEnd"), 1 << i);
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << i];
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = dir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << i];
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
ii.seek(1 << i);
ii.close();
dir.close();
}
}
public void testSeekSliceEnd() throws Exception {
for (int i = 0; i < 17; i++) {
Directory dir = getDirectory(createTempDir("testSeekSliceEnd"), 1 << i);
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << i];
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput slicer = dir.openInput("bytes", newIOContext(random()));
IndexInput ii = slicer.slice("full slice", 0, bytes.length);
byte[] actual = new byte[1 << i];
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
ii.seek(1 << i);
ii.close();
slicer.close();
dir.close();
}
}
public void testSeeking() throws Exception {
int numIters = TEST_NIGHTLY ? 10 : 1;
for (int i = 0; i < numIters; i++) {
Directory dir = getDirectory(createTempDir("testSeeking"), 1 << i);
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << (i + 1)]; // make sure we switch buffers
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = dir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << (i + 1)]; // first read all bytes
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) {
for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) {
byte[] slice = new byte[sliceLength];
ii.seek(sliceStart);
ii.readBytes(slice, 0, slice.length);
assertEquals(new BytesRef(bytes, sliceStart, sliceLength), new BytesRef(slice));
}
}
ii.close();
dir.close();
}
}
// note instead of seeking to offset and reading length, this opens slices at the
// the various offset+length and just does readBytes.
public void testSlicedSeeking() throws Exception {
int numIters = TEST_NIGHTLY ? 10 : 1;
for (int i = 0; i < numIters; i++) {
Directory dir = getDirectory(createTempDir("testSlicedSeeking"), 1 << i);
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << (i + 1)]; // make sure we switch buffers
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = dir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << (i + 1)]; // first read all bytes
ii.readBytes(actual, 0, actual.length);
ii.close();
assertEquals(new BytesRef(bytes), new BytesRef(actual));
IndexInput slicer = dir.openInput("bytes", newIOContext(random()));
for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) {
for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) {
assertSlice(bytes, slicer, 0, sliceStart, sliceLength);
}
}
slicer.close();
dir.close();
}
}
@Override
public void testSliceOfSlice() throws Exception {
int upto = TEST_NIGHTLY ? 10 : 8;
for (int i = 0; i < upto; i++) {
Directory dir = getDirectory(createTempDir("testSliceOfSlice"), 1 << i);
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
byte[] bytes = new byte[1 << (i + 1)]; // make sure we switch buffers
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInput ii = dir.openInput("bytes", newIOContext(random()));
byte[] actual = new byte[1 << (i + 1)]; // first read all bytes
ii.readBytes(actual, 0, actual.length);
ii.close();
assertEquals(new BytesRef(bytes), new BytesRef(actual));
IndexInput outerSlicer = dir.openInput("bytes", newIOContext(random()));
final int outerSliceStart = random().nextInt(bytes.length / 2);
final int outerSliceLength = random().nextInt(bytes.length - outerSliceStart);
IndexInput innerSlicer =
outerSlicer.slice("parentBytesSlice", outerSliceStart, outerSliceLength);
for (int sliceStart = 0; sliceStart < outerSliceLength; sliceStart++) {
for (int sliceLength = 0; sliceLength < outerSliceLength - sliceStart; sliceLength++) {
assertSlice(bytes, innerSlicer, outerSliceStart, sliceStart, sliceLength);
}
}
innerSlicer.close();
outerSlicer.close();
dir.close();
}
}
private void assertSlice(
byte[] bytes, IndexInput slicer, int outerSliceStart, int sliceStart, int sliceLength)
throws IOException {
byte[] slice = new byte[sliceLength];
IndexInput input = slicer.slice("bytesSlice", sliceStart, slice.length);
input.readBytes(slice, 0, slice.length);
input.close();
assertEquals(
new BytesRef(bytes, outerSliceStart + sliceStart, sliceLength), new BytesRef(slice));
}
public void testRandomChunkSizes() throws Exception {
int num = TEST_NIGHTLY ? atLeast(10) : 3;
for (int i = 0; i < num; i++) {
assertChunking(random(), TestUtil.nextInt(random(), 20, 100));
}
}
private void assertChunking(Random random, int chunkSize) throws Exception {
Path path = createTempDir("mmap" + chunkSize);
Directory chunkedDir = getDirectory(path, chunkSize);
// we will map a lot, try to turn on the unmap hack
if (chunkedDir instanceof MMapDirectory && MMapDirectory.UNMAP_SUPPORTED) {
((MMapDirectory) chunkedDir).setUseUnmap(true);
}
MockDirectoryWrapper dir = new MockDirectoryWrapper(random, chunkedDir);
RandomIndexWriter writer =
new RandomIndexWriter(
random,
dir,
newIndexWriterConfig(new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
Document doc = new Document();
Field docid = newStringField("docid", "0", Field.Store.YES);
Field junk = newStringField("junk", "", Field.Store.YES);
doc.add(docid);
doc.add(junk);
int numDocs = 100;
for (int i = 0; i < numDocs; i++) {
docid.setStringValue("" + i);
junk.setStringValue(TestUtil.randomUnicodeString(random));
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
int numAsserts = atLeast(100);
for (int i = 0; i < numAsserts; i++) {
int docID = random.nextInt(numDocs);
assertEquals("" + docID, reader.document(docID).get("docid"));
}
reader.close();
dir.close();
}
public void testLittleEndianLongsCrossBoundary() throws Exception {
try (Directory dir = getDirectory(createTempDir("testLittleEndianLongsCrossBoundary"), 16)) {
try (IndexOutput out = dir.createOutput("littleEndianLongs", newIOContext(random()))) {
out.writeByte((byte) 2);
out.writeLong(3L);
out.writeLong(Long.MAX_VALUE);
out.writeLong(-3L);
}
try (IndexInput input = dir.openInput("littleEndianLongs", newIOContext(random()))) {
assertEquals(25, input.length());
assertEquals(2, input.readByte());
long[] l = new long[4];
input.readLongs(l, 1, 3);
assertArrayEquals(new long[] {0L, 3L, Long.MAX_VALUE, -3L}, l);
assertEquals(25, input.getFilePointer());
}
}
}
public void testLittleEndianFloatsCrossBoundary() throws Exception {
try (Directory dir = getDirectory(createTempDir("testFloatsCrossBoundary"), 8)) {
try (IndexOutput out = dir.createOutput("Floats", newIOContext(random()))) {
out.writeByte((byte) 2);
out.writeInt(Float.floatToIntBits(3f));
out.writeInt(Float.floatToIntBits(Float.MAX_VALUE));
out.writeInt(Float.floatToIntBits(-3f));
}
try (IndexInput input = dir.openInput("Floats", newIOContext(random()))) {
assertEquals(13, input.length());
assertEquals(2, input.readByte());
float[] ff = new float[4];
input.readFloats(ff, 1, 3);
assertArrayEquals(new float[] {0, 3f, Float.MAX_VALUE, -3f}, ff, 0);
assertEquals(13, input.getFilePointer());
}
}
}
}