Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5067: add a BaseDirectoryTestCase 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1587440 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-04-15 06:12:29 +00:00
parent 4aa0a1db1c
commit a33712ebc3
12 changed files with 621 additions and 495 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java
View File

@ -1,173 +0,0 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.Test;
public class TestCopyBytes extends LuceneTestCase {
private byte value(int idx) {
return (byte) ((idx % 256) * (1 + (idx / 256)));
}
@Test
public void testCopyBytes() throws Exception {
int num = atLeast(10);
for (int iter = 0; iter < num; iter++) {
Directory dir = newDirectory();
if (VERBOSE) {
System.out.println("TEST: iter=" + iter + " dir=" + dir);
}
// make random file
IndexOutput out = dir.createOutput("test", newIOContext(random()));
byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)];
final int size = TestUtil.nextInt(random(), 1, 1777777);
int upto = 0;
int byteUpto = 0;
while (upto < size) {
bytes[byteUpto++] = value(upto);
upto++;
if (byteUpto == bytes.length) {
out.writeBytes(bytes, 0, bytes.length);
byteUpto = 0;
}
}
out.writeBytes(bytes, 0, byteUpto);
assertEquals(size, out.getFilePointer());
out.close();
assertEquals(size, dir.fileLength("test"));
// copy from test -> test2
final IndexInput in = dir.openInput("test", newIOContext(random()));
out = dir.createOutput("test2", newIOContext(random()));
upto = 0;
while (upto < size) {
if (random().nextBoolean()) {
out.writeByte(in.readByte());
upto++;
} else {
final int chunk = Math.min(
TestUtil.nextInt(random(), 1, bytes.length), size - upto);
out.copyBytes(in, chunk);
upto += chunk;
}
}
assertEquals(size, upto);
out.close();
in.close();
// verify
IndexInput in2 = dir.openInput("test2", newIOContext(random()));
upto = 0;
while (upto < size) {
if (random().nextBoolean()) {
final byte v = in2.readByte();
assertEquals(value(upto), v);
upto++;
} else {
final int limit = Math.min(
TestUtil.nextInt(random(), 1, bytes.length), size - upto);
in2.readBytes(bytes, 0, limit);
for (int byteIdx = 0; byteIdx < limit; byteIdx++) {
assertEquals(value(upto), bytes[byteIdx]);
upto++;
}
}
}
in2.close();
dir.deleteFile("test");
dir.deleteFile("test2");
dir.close();
}
}
// LUCENE-3541
public void testCopyBytesWithThreads() throws Exception {
int datalen = TestUtil.nextInt(random(), 101, 10000);
byte data[] = new byte[datalen];
random().nextBytes(data);
Directory d = newDirectory();
IndexOutput output = d.createOutput("data", IOContext.DEFAULT);
output.writeBytes(data, 0, datalen);
output.close();
IndexInput input = d.openInput("data", IOContext.DEFAULT);
IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT);
// copy our 100-byte header
outputHeader.copyBytes(input, 100);
outputHeader.close();
// now make N copies of the remaining bytes
CopyThread copies[] = new CopyThread[10];
for (int i = 0; i < copies.length; i++) {
copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT));
}
for (int i = 0; i < copies.length; i++) {
copies[i].start();
}
for (int i = 0; i < copies.length; i++) {
copies[i].join();
}
for (int i = 0; i < copies.length; i++) {
IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT);
byte[] dataCopy = new byte[datalen];
System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing
copiedData.readBytes(dataCopy, 100, datalen-100);
assertArrayEquals(data, dataCopy);
copiedData.close();
}
input.close();
d.close();
}
static class CopyThread extends Thread {
final IndexInput src;
final IndexOutput dst;
CopyThread(IndexInput src, IndexOutput dst) {
this.src = src;
this.dst = dst;
}
@Override
public void run() {
try {
dst.copyBytes(src, src.length()-100);
dst.close();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}
}

145
lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
View File

@ -24,114 +24,18 @@ import java.nio.file.NoSuchFileException;
import java.util.Arrays;
import java.util.Collections;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
public class TestDirectory extends LuceneTestCase {
public void testDetectClose() throws Throwable {
File tempDir = createTempDir(LuceneTestCase.getTestClass().getSimpleName());
Directory[] dirs = new Directory[] {
new RAMDirectory(),
new SimpleFSDirectory(tempDir),
new NIOFSDirectory(tempDir)
};
public class TestDirectory extends BaseDirectoryTestCase {
for (Directory dir : dirs) {
dir.close();
try {
dir.createOutput("test", newIOContext(random()));
fail("did not hit expected exception");
} catch (AlreadyClosedException ace) {
}
@Override
protected Directory getDirectory(File path) throws IOException {
if (random().nextBoolean()) {
return newDirectory();
} else {
return newFSDirectory(path);
}
}
// test is occasionally very slow, i dont know why
// try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE
@Nightly
public void testThreadSafety() throws Exception {
final BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we arent making an index
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setThrottling(Throttling.NEVER); // makes this test really slow
}
if (VERBOSE) {
System.out.println(dir);
}
class TheThread extends Thread {
private String name;
public TheThread(String name) {
this.name = name;
}
@Override
public void run() {
for (int i = 0; i < 3000; i++) {
String fileName = this.name + i;
try {
//System.out.println("create:" + fileName);
IndexOutput output = dir.createOutput(fileName, newIOContext(random()));
output.close();
assertTrue(slowFileExists(dir, fileName));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
class TheThread2 extends Thread {
private String name;
public TheThread2(String name) {
this.name = name;
}
@Override
public void run() {
for (int i = 0; i < 10000; i++) {
try {
String[] files = dir.listAll();
for (String file : files) {
//System.out.println("file:" + file);
try {
IndexInput input = dir.openInput(file, newIOContext(random()));
input.close();
} catch (FileNotFoundException | NoSuchFileException e) {
// ignore
} catch (IOException e) {
if (e.getMessage().contains("still open for writing")) {
// ignore
} else {
throw new RuntimeException(e);
}
}
if (random().nextBoolean()) {
break;
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
TheThread theThread = new TheThread("t1");
TheThread2 theThread2 = new TheThread2("t2");
theThread.start();
theThread2.start();
theThread.join();
theThread2.join();
dir.close();
}
// Test that different instances of FSDirectory can coexist on the same
// path, can read, write, and lock files.
@ -222,41 +126,6 @@ public class TestDirectory extends LuceneTestCase {
TestUtil.rm(path);
}
// LUCENE-1464
public void testDontCreate() throws Throwable {
File path = new File(createTempDir(LuceneTestCase.getTestClass().getSimpleName()), "doesnotexist");
try {
assertTrue(!path.exists());
Directory dir = new SimpleFSDirectory(path, null);
assertTrue(!path.exists());
dir.close();
} finally {
TestUtil.rm(path);
}
}
// LUCENE-1468
public void testRAMDirectoryFilter() throws IOException {
checkDirectoryFilter(new RAMDirectory());
}
// LUCENE-1468
public void testFSDirectoryFilter() throws IOException {
checkDirectoryFilter(newFSDirectory(createTempDir("test")));
}
// LUCENE-1468
private void checkDirectoryFilter(Directory dir) throws IOException {
String name = "file";
try {
dir.createOutput(name, newIOContext(random())).close();
assertTrue(slowFileExists(dir, name));
assertTrue(Arrays.asList(dir.listAll()).contains(name));
} finally {
dir.close();
}
}
// LUCENE-1468
public void testCopySubdir() throws Throwable {
File path = createTempDir("testsubdir");

70
lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
View File

@ -35,7 +35,8 @@ import org.apache.lucene.index.TestIndexWriterReader;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
public class TestFileSwitchDirectory extends LuceneTestCase {
public class TestFileSwitchDirectory extends BaseDirectoryTestCase {
/**
* Test if writing doc stores to disk and everything else to ram works.
*/
@ -115,51 +116,28 @@ public class TestFileSwitchDirectory extends LuceneTestCase {
}
dir.close();
}
// LUCENE-3380 test that we can add a file, and then when we call list() we get it back
public void testDirectoryFilter() throws IOException {
Directory dir = newFSSwitchDirectory(Collections.<String>emptySet());
String name = "file";
try {
dir.createOutput(name, newIOContext(random())).close();
assertTrue(slowFileExists(dir, name));
assertTrue(Arrays.asList(dir.listAll()).contains(name));
} finally {
dir.close();
@Override
protected Directory getDirectory(File path) throws IOException {
Set<String> extensions = new HashSet<String>();
if (random().nextBoolean()) {
extensions.add("cfs");
}
if (random().nextBoolean()) {
extensions.add("prx");
}
if (random().nextBoolean()) {
extensions.add("frq");
}
if (random().nextBoolean()) {
extensions.add("tip");
}
if (random().nextBoolean()) {
extensions.add("tim");
}
if (random().nextBoolean()) {
extensions.add("del");
}
return newFSSwitchDirectory(extensions);
}
// LUCENE-3380 test that delegate compound files correctly.
public void testCompoundFileAppendTwice() throws IOException {
Directory newDir = newFSSwitchDirectory(Collections.singleton("cfs"));
CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
createSequenceFile(newDir, "d1", (byte) 0, 15);
IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
out.writeInt(0);
out.close();
assertEquals(1, csw.listAll().length);
assertEquals("d.xyz", csw.listAll()[0]);
csw.close();
CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
assertEquals(1, cfr.listAll().length);
assertEquals("d.xyz", cfr.listAll()[0]);
cfr.close();
newDir.close();
}
/** Creates a file of the specified size with sequential data. The first
* byte is written as the start byte provided. All subsequent bytes are
* computed as start + offset where offset is the number of the byte.
*/
private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException {
IndexOutput os = dir.createOutput(name, newIOContext(random()));
for (int i=0; i < size; i++) {
os.writeByte(start);
start ++;
}
os.close();
}
}

32
lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java Normal file
View File

@ -0,0 +1,32 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
/**
* Tests MMapDirectory
*/
public class TestMmapDirectory extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(File path) throws IOException {
return new MMapDirectory(path);
}
}

10
lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.File;
import java.io.IOException;
import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
@ -27,7 +28,6 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory.IndexInputSlicer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
/**
@ -37,7 +37,13 @@ import org.apache.lucene.util.TestUtil;
* values, it's necessary to access a file >
* Integer.MAX_VALUE in size using multiple byte buffers.
*/
public class TestMultiMMap extends LuceneTestCase {
public class TestMultiMMap extends BaseDirectoryTestCase {
File workDir;
@Override
protected Directory getDirectory(File path) throws IOException {
return new MMapDirectory(path, null, 1<<TestUtil.nextInt(random(), 10, 28));
}
@Override
public void setUp() throws Exception {

32
lucene/core/src/test/org/apache/lucene/store/TestNIOFSDirectory.java Normal file
View File

@ -0,0 +1,32 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
/**
* Tests NIOFSDirectory
*/
public class TestNIOFSDirectory extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(File path) throws IOException {
return new NIOFSDirectory(path);
}
}

82
lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
View File

@ -18,9 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
@ -36,10 +34,16 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
public class TestNRTCachingDirectory extends LuceneTestCase {
public class TestNRTCachingDirectory extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(File path) {
return new NRTCachingDirectory(newFSDirectory(path),
.1 + 2.0*random().nextDouble(),
.1 + 5.0*random().nextDouble());
}
public void testNRTAndCommit() throws Exception {
Directory dir = newDirectory();
@ -111,73 +115,7 @@ public class TestNRTCachingDirectory extends LuceneTestCase {
NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 2.0, 25.0);
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
IndexWriter writer = new IndexWriter(cachedFSDir, conf);
}
public void testDeleteFile() throws Exception {
Directory dir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0);
dir.createOutput("foo.txt", IOContext.DEFAULT).close();
dir.deleteFile("foo.txt");
assertEquals(0, dir.listAll().length);
dir.close();
}
// LUCENE-3382 -- make sure we get exception if the directory really does not exist.
public void testNoDir() throws Throwable {
File tempDir = createTempDir("doesnotexist");
TestUtil.rm(tempDir);
Directory dir = new NRTCachingDirectory(newFSDirectory(tempDir), 2.0, 25.0);
try {
DirectoryReader.open(dir);
fail("did not hit expected exception");
} catch (NoSuchDirectoryException nsde) {
// expected
}
dir.close();
}
// LUCENE-3382 test that we can add a file, and then when we call list() we get it back
public void testDirectoryFilter() throws IOException {
Directory dir = new NRTCachingDirectory(newFSDirectory(createTempDir("foo")), 2.0, 25.0);
String name = "file";
try {
dir.createOutput(name, newIOContext(random())).close();
assertTrue(slowFileExists(dir, name));
assertTrue(Arrays.asList(dir.listAll()).contains(name));
} finally {
dir.close();
}
}
// LUCENE-3382 test that delegate compound files correctly.
public void testCompoundFileAppendTwice() throws IOException {
Directory newDir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0);
CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
createSequenceFile(newDir, "d1", (byte) 0, 15);
IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
out.writeInt(0);
out.close();
assertEquals(1, csw.listAll().length);
assertEquals("d.xyz", csw.listAll()[0]);
csw.close();
CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
assertEquals(1, cfr.listAll().length);
assertEquals("d.xyz", cfr.listAll()[0]);
cfr.close();
newDir.close();
}
/** Creates a file of the specified size with sequential data. The first
* byte is written as the start byte provided. All subsequent bytes are
* computed as start + offset where offset is the number of the byte.
*/
private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException {
IndexOutput os = dir.createOutput(name, newIOContext(random()));
for (int i=0; i < size; i++) {
os.writeByte(start);
start ++;
}
os.close();
writer.close();
cachedFSDir.close();
}
}

95
lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
View File

@ -20,39 +20,38 @@ package org.apache.lucene.store;
import java.io.File;
import java.io.IOException;
import org.apache.lucene.document.Field;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.English;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
/**
* JUnit testcase to test RAMDirectory. RAMDirectory itself is used in many testcases,
* but not one of them uses an different constructor other than the default constructor.
*/
public class TestRAMDirectory extends LuceneTestCase {
public class TestRAMDirectory extends BaseDirectoryTestCase {
private File indexDir = null;
@Override
protected Directory getDirectory(File path) {
return new RAMDirectory();
}
// add enough document so that the index will be larger than RAMDirectory.READ_BUFFER_SIZE
private final int docsToAdd = 500;
// setup the index
@Override
public void setUp() throws Exception {
super.setUp();
indexDir = createTempDir("RAMDirIndex");
private File buildIndex() throws IOException {
File path = createTempDir("buildIndex");
Directory dir = newFSDirectory(indexDir);
Directory dir = newFSDirectory(path);
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
// add some documents
@ -65,9 +64,27 @@ public class TestRAMDirectory extends LuceneTestCase {
assertEquals(docsToAdd, writer.maxDoc());
writer.shutdown();
dir.close();
return path;
}
// LUCENE-1468
public void testCopySubdir() throws Throwable {
File path = createTempDir("testsubdir");
Directory fsDir = null;
try {
path.mkdirs();
new File(path, "subdir").mkdirs();
fsDir = newFSDirectory(path);
assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length);
} finally {
TestUtil.rm(path);
IOUtils.close(fsDir);
}
}
public void testRAMDirectory () throws IOException {
File indexDir = buildIndex();
Directory dir = newFSDirectory(indexDir);
MockDirectoryWrapper ramDir = new MockDirectoryWrapper(random(), new RAMDirectory(dir, newIOContext(random())));
@ -99,6 +116,8 @@ public class TestRAMDirectory extends LuceneTestCase {
private final int docsPerThread = 40;
public void testRAMDirectorySize() throws IOException, InterruptedException {
File indexDir = buildIndex();
Directory dir = newFSDirectory(indexDir);
final MockDirectoryWrapper ramDir = new MockDirectoryWrapper(random(), new RAMDirectory(dir, newIOContext(random())));
@ -138,52 +157,4 @@ public class TestRAMDirectory extends LuceneTestCase {
writer.shutdown();
}
@Override
public void tearDown() throws Exception {
// cleanup
if (indexDir != null && indexDir.exists()) {
rmDir (indexDir);
}
super.tearDown();
}
// LUCENE-1196
public void testIllegalEOF() throws Exception {
RAMDirectory dir = new RAMDirectory();
IndexOutput o = dir.createOutput("out", newIOContext(random()));
byte[] b = new byte[1024];
o.writeBytes(b, 0, 1024);
o.close();
IndexInput i = dir.openInput("out", newIOContext(random()));
i.seek(1024);
i.close();
dir.close();
}
private void rmDir(File dir) {
File[] files = dir.listFiles();
for (int i = 0; i < files.length; i++) {
files[i].delete();
}
dir.delete();
}
// LUCENE-2852
public void testSeekToEOFThenBack() throws Exception {
RAMDirectory dir = new RAMDirectory();
IndexOutput o = dir.createOutput("out", newIOContext(random()));
byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE];
o.writeBytes(bytes, 0, bytes.length);
o.close();
IndexInput i = dir.openInput("out", newIOContext(random()));
i.seek(2*RAMInputStream.BUFFER_SIZE-1);
i.seek(3*RAMInputStream.BUFFER_SIZE);
i.seek(RAMInputStream.BUFFER_SIZE);
i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE);
i.close();
dir.close();
}
}

31
lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java Normal file
View File

@ -0,0 +1,31 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
public class TestRateLimitedDirectoryWrapper extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(File path) {
RateLimitedDirectoryWrapper dir = new RateLimitedDirectoryWrapper(newFSDirectory(path));
RateLimiter limiter = new RateLimiter.SimpleRateLimiter(.1 + 3*random().nextDouble());
dir.setRateLimiter(limiter, IOContext.Context.MERGE);
return dir;
}
}

32
lucene/core/src/test/org/apache/lucene/store/TestSimpleFSDirectory.java Normal file
View File

@ -0,0 +1,32 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
/**
* Tests SimpleFSDirectory
*/
public class TestSimpleFSDirectory extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(File path) throws IOException {
return new SimpleFSDirectory(path);
}
}

397
lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java Normal file
View File

@ -0,0 +1,397 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.Arrays;
import java.util.Collections;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexNotFoundException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
/** Base class for per-Directory tests. */
public abstract class BaseDirectoryTestCase extends LuceneTestCase {
/** Subclass returns the Directory to be tested; if it's
* an FS-based directory it should point to the specified
* path, else it can ignore it. */
protected abstract Directory getDirectory(File path) throws IOException;
/** Make sure directory throws AlreadyClosedException if
* you try to createOutput after closing. */
public void testDetectClose() throws Throwable {
Directory dir = getDirectory(createTempDir("testDetectClose"));
dir.close();
try {
dir.createOutput("test", newIOContext(random()));
fail("did not hit expected exception");
} catch (AlreadyClosedException ace) {
// expected
}
}
// test is occasionally very slow, i dont know why
// try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE
@Nightly
public void testThreadSafety() throws Exception {
final Directory raw = getDirectory(createTempDir("testThreadSafety"));
final BaseDirectoryWrapper dir = newDirectory(raw);
dir.setCheckIndexOnClose(false); // we arent making an index
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); // makes this test really slow
}
if (VERBOSE) {
System.out.println(dir);
}
class TheThread extends Thread {
private String name;
public TheThread(String name) {
this.name = name;
}
@Override
public void run() {
for (int i = 0; i < 3000; i++) {
String fileName = this.name + i;
try {
//System.out.println("create:" + fileName);
IndexOutput output = dir.createOutput(fileName, newIOContext(random()));
output.close();
assertTrue(slowFileExists(dir, fileName));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
class TheThread2 extends Thread {
private String name;
public TheThread2(String name) {
this.name = name;
}
@Override
public void run() {
for (int i = 0; i < 10000; i++) {
try {
String[] files = dir.listAll();
for (String file : files) {
//System.out.println("file:" + file);
try {
IndexInput input = dir.openInput(file, newIOContext(random()));
input.close();
} catch (FileNotFoundException | NoSuchFileException e) {
// ignore
} catch (IOException e) {
if (e.getMessage().contains("still open for writing")) {
// ignore
} else {
throw new RuntimeException(e);
}
}
if (random().nextBoolean()) {
break;
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
TheThread theThread = new TheThread("t1");
TheThread2 theThread2 = new TheThread2("t2");
theThread.start();
theThread2.start();
theThread.join();
theThread2.join();
dir.close();
raw.close();
}
/** LUCENE-1464: just creating a Directory should not
* mkdir the underling directory in the filesystem. */
public void testDontCreate() throws Throwable {
File path = createTempDir("doesnotexist");
TestUtil.rm(path);
assertTrue(!path.exists());
Directory dir = getDirectory(path);
assertTrue(!path.exists());
dir.close();
}
/** LUCENE-1468: once we create an output, we should see
* it in the dir listing and be able to open it with
* openInput. */
public void testDirectoryFilter() throws IOException {
String name = "file";
Directory dir = getDirectory(createTempDir("testDirectoryFilter"));
try {
dir.createOutput(name, newIOContext(random())).close();
assertTrue(slowFileExists(dir, name));
assertTrue(Arrays.asList(dir.listAll()).contains(name));
} finally {
dir.close();
}
}
// LUCENE-2852
public void testSeekToEOFThenBack() throws Exception {
Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"));
IndexOutput o = dir.createOutput("out", newIOContext(random()));
byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE];
o.writeBytes(bytes, 0, bytes.length);
o.close();
IndexInput i = dir.openInput("out", newIOContext(random()));
i.seek(2*RAMInputStream.BUFFER_SIZE-1);
i.seek(3*RAMInputStream.BUFFER_SIZE);
i.seek(RAMInputStream.BUFFER_SIZE);
i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE);
i.close();
dir.close();
}
// LUCENE-1196
public void testIllegalEOF() throws Exception {
Directory dir = getDirectory(createTempDir("testIllegalEOF"));
IndexOutput o = dir.createOutput("out", newIOContext(random()));
byte[] b = new byte[1024];
o.writeBytes(b, 0, 1024);
o.close();
IndexInput i = dir.openInput("out", newIOContext(random()));
i.seek(1024);
i.close();
dir.close();
}
public void testDeleteFile() throws Exception {
Directory dir = getDirectory(createTempDir("testDeleteFile"));
dir.createOutput("foo.txt", IOContext.DEFAULT).close();
dir.deleteFile("foo.txt");
assertEquals(0, dir.listAll().length);
dir.close();
}
// LUCENE-3382 -- make sure we get exception if the directory really does not exist.
public void testNoDir() throws Throwable {
File tempDir = createTempDir("doesnotexist");
TestUtil.rm(tempDir);
Directory dir = getDirectory(tempDir);
try {
DirectoryReader.open(dir);
fail("did not hit expected exception");
} catch (NoSuchDirectoryException | IndexNotFoundException nsde) {
// expected
}
dir.close();
}
// LUCENE-3382 test that delegate compound files correctly.
public void testCompoundFileAppendTwice() throws IOException {
Directory newDir = getDirectory(createTempDir("testCompoundFileAppendTwice"));
CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
createSequenceFile(newDir, "d1", (byte) 0, 15);
IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
out.writeInt(0);
out.close();
assertEquals(1, csw.listAll().length);
assertEquals("d.xyz", csw.listAll()[0]);
csw.close();
CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
assertEquals(1, cfr.listAll().length);
assertEquals("d.xyz", cfr.listAll()[0]);
cfr.close();
newDir.close();
}
/** Creates a file of the specified size with sequential data. The first
* byte is written as the start byte provided. All subsequent bytes are
* computed as start + offset where offset is the number of the byte.
*/
private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException {
IndexOutput os = dir.createOutput(name, newIOContext(random()));
for (int i=0; i < size; i++) {
os.writeByte(start);
start ++;
}
os.close();
}
public void testCopyBytes() throws Exception {
testCopyBytes(getDirectory(createTempDir("testCopyBytes")));
}
private static byte value(int idx) {
return (byte) ((idx % 256) * (1 + (idx / 256)));
}
public static void testCopyBytes(Directory dir) throws Exception {
// make random file
IndexOutput out = dir.createOutput("test", newIOContext(random()));
byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)];
final int size = TestUtil.nextInt(random(), 1, 1777777);
int upto = 0;
int byteUpto = 0;
while (upto < size) {
bytes[byteUpto++] = value(upto);
upto++;
if (byteUpto == bytes.length) {
out.writeBytes(bytes, 0, bytes.length);
byteUpto = 0;
}
}
out.writeBytes(bytes, 0, byteUpto);
assertEquals(size, out.getFilePointer());
out.close();
assertEquals(size, dir.fileLength("test"));
// copy from test -> test2
final IndexInput in = dir.openInput("test", newIOContext(random()));
out = dir.createOutput("test2", newIOContext(random()));
upto = 0;
while (upto < size) {
if (random().nextBoolean()) {
out.writeByte(in.readByte());
upto++;
} else {
final int chunk = Math.min(
TestUtil.nextInt(random(), 1, bytes.length), size - upto);
out.copyBytes(in, chunk);
upto += chunk;
}
}
assertEquals(size, upto);
out.close();
in.close();
// verify
IndexInput in2 = dir.openInput("test2", newIOContext(random()));
upto = 0;
while (upto < size) {
if (random().nextBoolean()) {
final byte v = in2.readByte();
assertEquals(value(upto), v);
upto++;
} else {
final int limit = Math.min(
TestUtil.nextInt(random(), 1, bytes.length), size - upto);
in2.readBytes(bytes, 0, limit);
for (int byteIdx = 0; byteIdx < limit; byteIdx++) {
assertEquals(value(upto), bytes[byteIdx]);
upto++;
}
}
}
in2.close();
dir.deleteFile("test");
dir.deleteFile("test2");
dir.close();
}
// LUCENE-3541
public void testCopyBytesWithThreads() throws Exception {
testCopyBytesWithThreads(getDirectory(createTempDir("testCopyBytesWithThreads")));
}
public static void testCopyBytesWithThreads(Directory d) throws Exception {
int datalen = TestUtil.nextInt(random(), 101, 10000);
byte data[] = new byte[datalen];
random().nextBytes(data);
IndexOutput output = d.createOutput("data", IOContext.DEFAULT);
output.writeBytes(data, 0, datalen);
output.close();
IndexInput input = d.openInput("data", IOContext.DEFAULT);
IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT);
// copy our 100-byte header
outputHeader.copyBytes(input, 100);
outputHeader.close();
// now make N copies of the remaining bytes
CopyThread copies[] = new CopyThread[10];
for (int i = 0; i < copies.length; i++) {
copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT));
}
for (int i = 0; i < copies.length; i++) {
copies[i].start();
}
for (int i = 0; i < copies.length; i++) {
copies[i].join();
}
for (int i = 0; i < copies.length; i++) {
IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT);
byte[] dataCopy = new byte[datalen];
System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing
copiedData.readBytes(dataCopy, 100, datalen-100);
assertArrayEquals(data, dataCopy);
copiedData.close();
}
input.close();
d.close();
}
static class CopyThread extends Thread {
final IndexInput src;
final IndexOutput dst;
CopyThread(IndexInput src, IndexOutput dst) {
this.src = src;
this.dst = dst;
}
@Override
public void run() {
try {
dst.copyBytes(src, src.length()-100);
dst.close();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}
}

17
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -1255,10 +1255,23 @@ public abstract class LuceneTestCase extends Assert {
// If it is a FSDirectory type, try its ctor(File)
if (FSDirectory.class.isAssignableFrom(clazz)) {
final File dir = createTempDir("index-" + clazzName);
dir.mkdirs(); // ensure it's created so we 'have' it.
return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), dir);
}
// See if it has a File ctor even though it's not an
// FSDir subclass:
Constructor<? extends Directory> fileCtor = null;
try {
fileCtor = clazz.getConstructor(File.class);
} catch (NoSuchMethodException nsme) {
// Ignore
}
if (fileCtor != null) {
final File dir = createTempDir("index");
return fileCtor.newInstance(dir);
}
// try empty ctor
return clazz.newInstance();
} catch (Exception e) {
@ -1445,7 +1458,7 @@ public abstract class LuceneTestCase extends Assert {
}
if (ex != null) {
if (VERBOSE) {
System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads");
System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads");
}
r.addReaderClosedListener(new ReaderClosedListener() {
@Override