honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simplify tests, doc file length

This commit is contained in:
Shay Banon 2012-06-29 16:01:17 +02:00
parent f2e39e4ee2
commit 8bab859822
5 changed files with 209 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
src/main/java/org/elasticsearch/common/compress/CompressedDirectory.java Normal file
View File

@ -0,0 +1,190 @@
package org.elasticsearch.common.compress;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.store.*;
import org.elasticsearch.index.store.support.ForceSyncDirectory;
import java.io.IOException;
import java.util.Collection;
/**
*/
public class CompressedDirectory extends Directory implements ForceSyncDirectory {
private final Directory dir;
private final Compressor compressor;
private final boolean actualLength;
private final ImmutableSet<String> compressExtensions;
private final ImmutableSet<String> decompressExtensions;
private volatile boolean compress = true;
public CompressedDirectory(Directory dir, Compressor compressor, boolean actualLength, String... extensions) {
this(dir, compressor, actualLength, extensions, extensions);
}
public CompressedDirectory(Directory dir, Compressor compressor, boolean actualLength, String[] compressExtensions, String[] decompressExtensions) {
this.dir = dir;
this.actualLength = actualLength;
this.compressor = compressor;
this.compressExtensions = ImmutableSet.copyOf(compressExtensions);
this.decompressExtensions = ImmutableSet.copyOf(decompressExtensions);
this.lockFactory = dir.getLockFactory();
}
@Override
public String[] listAll() throws IOException {
return dir.listAll();
}
public void setCompress(boolean compress) {
this.compress = compress;
}
/**
* Utility method to return a file's extension.
*/
public static String getExtension(String name) {
int i = name.lastIndexOf('.');
if (i == -1) {
return "";
}
return name.substring(i + 1, name.length());
}
@Override
public boolean fileExists(String name) throws IOException {
return dir.fileExists(name);
}
@Override
public long fileModified(String name) throws IOException {
return dir.fileModified(name);
}
@Override
public void touchFile(String name) throws IOException {
dir.touchFile(name);
}
@Override
public void deleteFile(String name) throws IOException {
dir.deleteFile(name);
}
/**
* Returns the actual file size, so will work with compound file format
* when compressed. Its the only one that really uses it for offsets...
*/
@Override
public long fileLength(String name) throws IOException {
if (actualLength && decompressExtensions.contains(getExtension(name))) {
IndexInput in = openInput(name);
try {
return in.length();
} catch (Exception e) {
in.close();
}
}
return dir.fileLength(name);
}
@Override
public void sync(Collection<String> names) throws IOException {
dir.sync(names);
}
@Override
public void sync(String name) throws IOException {
dir.sync(name);
}
@Override
public void forceSync(String name) throws IOException {
if (dir instanceof ForceSyncDirectory) {
((ForceSyncDirectory) dir).forceSync(name);
} else {
dir.sync(name);
}
}
@Override
public IndexInput openInput(String name) throws IOException {
if (decompressExtensions.contains(getExtension(name))) {
IndexInput in = dir.openInput(name);
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return dir.openInput(name);
}
@Override
public IndexInput openInput(String name, int bufferSize) throws IOException {
if (decompressExtensions.contains(getExtension(name))) {
IndexInput in = dir.openInput(name, bufferSize);
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return dir.openInput(name, bufferSize);
}
@Override
public IndexOutput createOutput(String name) throws IOException {
if (compress && compressExtensions.contains(getExtension(name))) {
return compressor.indexOutput(dir.createOutput(name));
}
return dir.createOutput(name);
}
// can't override this one, we need to open the correct compression
// @Override
// public void copy(Directory to, String src, String dest) throws IOException {
// dir.copy(to, src, dest);
// }
@Override
public void close() throws IOException {
dir.close();
}
@Override
public void setLockFactory(LockFactory lockFactory) throws IOException {
dir.setLockFactory(lockFactory);
}
@Override
public LockFactory getLockFactory() {
return dir.getLockFactory();
}
@Override
public String getLockID() {
return dir.getLockID();
}
@Override
public Lock makeLock(String name) {
return dir.makeLock(name);
}
@Override
public void clearLock(String name) throws IOException {
dir.clearLock(name);
}
@Override
public String toString() {
return "compressed(" + compressExtensions + "):" + dir.toString();
}
}

4
src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -433,6 +433,10 @@ public class Store extends AbstractIndexShardComponent {
}
}
/**
* Returns the *actual* file length, not the uncompressed one if compression is enabled, this
* messes things up when using compound file format, but it shouldn't be used in any case...
*/
@Override
public long fileLength(String name) throws IOException {
StoreFileMetaData metaData = filesMetadata.get(name);

4
src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java
View File

@ -36,6 +36,7 @@ public class StoreFileMetaData implements Streamable {
private long lastModified;
// the actual file size on "disk", if compressed, the compressed size
private long length;
private String checksum;
@ -69,6 +70,9 @@ public class StoreFileMetaData implements Streamable {
return this.lastModified;
}
/**
* the actual file size on "disk", if compressed, the compressed size
*/
public long length() {
return length;
}

60
src/test/java/org/elasticsearch/benchmark/compress/LuceneCompressionBenchmark.java
View File

@ -23,11 +23,10 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NIOFSDirectory;
import org.elasticsearch.common.compress.CompressedIndexInput;
import org.elasticsearch.common.compress.CompressedDirectory;
import org.elasticsearch.common.compress.Compressor;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.io.FileSystemUtils;
@ -37,7 +36,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import java.io.File;
import java.io.IOException;
/**
*/
@ -56,59 +54,7 @@ public class LuceneCompressionBenchmark {
FSDirectory uncompressedDir = new NIOFSDirectory(new File(testFile, "uncompressed"));
IndexWriter uncompressedWriter = new IndexWriter(uncompressedDir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
FSDirectory compressedDir = new NIOFSDirectory(new File(testFile, "compressed")) {
@Override
public IndexOutput createOutput(String name) throws IOException {
if (name.endsWith(".fdt")) {
return compressor.indexOutput(super.createOutput(name));
}
if (WITH_TV && name.endsWith(".tvf")) {
return compressor.indexOutput(super.createOutput(name));
}
return super.createOutput(name);
}
@Override
public IndexInput openInput(String name) throws IOException {
if (name.endsWith(".fdt")) {
IndexInput in = super.openInput(name);
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
if (WITH_TV && name.endsWith(".tvf")) {
IndexInput in = super.openInput(name);
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return super.openInput(name);
}
@Override
public IndexInput openInput(String name, int bufferSize) throws IOException {
if (name.endsWith(".fdt") || name.endsWith(".tvf")) {
IndexInput in = super.openInput(name, bufferSize);
// in case the override called openInput(String)
if (in instanceof CompressedIndexInput) {
return in;
}
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return super.openInput(name, bufferSize);
}
};
Directory compressedDir = new CompressedDirectory(new NIOFSDirectory(new File(testFile, "compressed")), compressor, false, "fdt", "tvf");
IndexWriter compressedWriter = new IndexWriter(compressedDir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));

59
src/test/java/org/elasticsearch/test/unit/common/compress/CompressIndexInputOutputTests.java
View File

@ -4,6 +4,7 @@ import jsr166y.ThreadLocalRandom;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.MapFieldSelector;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@ -12,18 +13,13 @@ import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory;
import org.elasticsearch.common.RandomStringGenerator;
import org.elasticsearch.common.compress.CompressedIndexInput;
import org.elasticsearch.common.compress.CompressedIndexOutput;
import org.elasticsearch.common.compress.Compressor;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.compress.*;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.unit.SizeValue;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.EOFException;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
@ -210,50 +206,7 @@ public class CompressIndexInputOutputTests {
@Test
public void lucene() throws Exception {
final AtomicBoolean compressed = new AtomicBoolean(true);
Directory dir = new RAMDirectory() {
@Override
public IndexOutput createOutput(String name) throws IOException {
if (compressed.get() && name.endsWith(".fdt")) {
return compressor.indexOutput(super.createOutput(name));
}
return super.createOutput(name);
}
@Override
public IndexInput openInput(String name) throws IOException {
if (name.endsWith(".fdt")) {
IndexInput in = super.openInput(name);
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return super.openInput(name);
}
@Override
public IndexInput openInput(String name, int bufferSize) throws IOException {
if (name.endsWith(".fdt")) {
IndexInput in = super.openInput(name, bufferSize);
// in case the override called openInput(String)
if (in instanceof CompressedIndexInput) {
return in;
}
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return super.openInput(name, bufferSize);
}
};
CompressedDirectory dir = new CompressedDirectory(new RAMDirectory(), compressor, false, "fdt");
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
writer.addDocument(createDoc(1, (int) SizeValue.parseSizeValue("100b").singles()));
writer.addDocument(createDoc(2, (int) SizeValue.parseSizeValue("5k").singles()));
@ -265,16 +218,20 @@ public class CompressIndexInputOutputTests {
writer.forceMerge(1);
writer.waitForMerges();
verify(writer);
compressed.set(false);
dir.setCompress(false);
writer.addDocument(createDoc(5, (int) SizeValue.parseSizeValue("2k").singles()));
writer.addDocument(createDoc(6, (int) SizeValue.parseSizeValue("1k").singles()));
verify(writer);
writer.forceMerge(1);
writer.waitForMerges();
verify(writer);
writer.close();
}
private void verify(IndexWriter writer) throws Exception {
CheckIndex checkIndex = new CheckIndex(writer.getDirectory());
CheckIndex.Status status = checkIndex.checkIndex();
assertThat(status.clean, equalTo(true));
IndexReader reader = IndexReader.open(writer, true);
for (int i = 0; i < reader.maxDoc(); i++) {
if (reader.isDeleted(i)) {