Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4666: Simplify CompressingStoredFieldsFormat merging. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1430755 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2013-01-09 10:00:16 +00:00
parent c86d102bfc
commit 21388e0e37
6 changed files with 4 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
View File

@ -395,8 +395,10 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
* Copy compressed data.
*/
void copyCompressedData(DataOutput out) throws IOException {
final int chunkSize = chunkSize();
decompressor.copyCompressedData(fieldsStream, chunkSize, out);
final long chunkEnd = docBase + chunkDocs == numDocs
? fieldsStream.length()
: indexReader.getStartPointer(docBase + chunkDocs);
out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
}
}

15
lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
View File

@ -140,14 +140,6 @@ public abstract class CompressionMode {
bytes.length = length;
}
@Override
public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
final int copied = LZ4.copyCompressedData(in, originalLength, out);
if (copied != originalLength) {
throw new CorruptIndexException("Currupted compressed stream: expected " + originalLength + " bytes, but got at least" + copied);
}
}
@Override
public Decompressor clone() {
return this;
@ -224,13 +216,6 @@ public abstract class CompressionMode {
bytes.length = length;
}
@Override
public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
final int compressedLength = in.readVInt();
out.writeVInt(compressedLength);
out.copyBytes(in, compressedLength);
}
@Override
public Decompressor clone() {
return new DeflateDecompressor();

5
lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.codecs.compressing;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
/**
@ -47,10 +46,6 @@ public abstract class Decompressor implements Cloneable {
*/
public abstract void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;
/** Copy a compressed stream whose original length is
* <code>originalLength</code> from <code>in</code> to <code>out</code>. */
public abstract void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException;
@Override
public abstract Decompressor clone();

47
lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
View File

@ -506,51 +506,4 @@ class LZ4 {
encodeLastLiterals(src, anchor, srcEnd - anchor, out);
}
/** Copy bytes from <code>in</code> to <code>out</code> where
* <code>in</code> is a LZ4-encoded stream. This method copies enough bytes
* so that <code>out</code> can be used later on to restore the first
* <code>length</code> bytes of the stream. This method always reads at
* least one byte from <code>in</code> so make sure not to call this method
* if <code>in</code> reached the end of the stream, even if
* <code>length=0</code>. */
public static int copyCompressedData(DataInput in, int length, DataOutput out) throws IOException {
int n = 0;
do {
// literals
final byte token = in.readByte();
out.writeByte(token);
int literalLen = (token & 0xFF) >>> 4;
if (literalLen == 0x0F) {
byte len;
while ((len = in.readByte()) == (byte) 0xFF) {
literalLen += 0xFF;
out.writeByte(len);
}
literalLen += len & 0xFF;
out.writeByte(len);
}
out.copyBytes(in, literalLen);
n += literalLen;
if (n >= length) {
break;
}
// matchs
out.copyBytes(in, 2); // match dec
int matchLen = token & 0x0F;
if (matchLen == 0x0F) {
byte len;
while ((len = in.readByte()) == (byte) 0xFF) {
matchLen += 0xFF;
out.writeByte(len);
}
matchLen += len & 0xFF;
out.writeByte(len);
}
matchLen += MIN_MATCH;
n += matchLen;
} while (n < length);
return n;
}
}

17
lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
View File

@ -80,16 +80,6 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
return Arrays.copyOfRange(bytes.bytes, bytes.offset, bytes.offset + bytes.length);
}
static byte[] copyCompressedData(Decompressor decompressor, byte[] compressed, int originalLength) throws IOException {
GrowableByteArrayDataOutput out = new GrowableByteArrayDataOutput(compressed.length);
decompressor.copyCompressedData(new ByteArrayDataInput(compressed), originalLength, out);
return Arrays.copyOf(out.bytes, out.length);
}
byte[] copyCompressedData(byte[] compressed, int originalLength) throws IOException {
return copyCompressedData(mode.newDecompressor(), compressed, originalLength);
}
public void testDecompress() throws IOException {
final int iterations = atLeast(10);
for (int i = 0; i < iterations; ++i) {
@ -117,17 +107,10 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
}
}
public void testCopyCompressedData() throws IOException {
final byte[] decompressed = randomArray();
final byte[] compressed = compress(decompressed);
assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
}
public byte[] test(byte[] decompressed) throws IOException {
final byte[] compressed = compress(decompressed);
final byte[] restored = decompress(compressed, decompressed.length);
assertEquals(decompressed.length, restored.length);
assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
return compressed;
}

5
lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
View File

@ -66,11 +66,6 @@ public class DummyCompressingCodec extends CompressingCodec {
bytes.length = length;
}
@Override
public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
out.copyBytes(in, originalLength);
}
@Override
public Decompressor clone() {
return this;