mirror of https://github.com/apache/lucene.git
LUCENE-4666: Simplify CompressingStoredFieldsFormat merging.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1430755 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
c86d102bfc
commit
21388e0e37
|
@ -395,8 +395,10 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
|
|||
* Copy compressed data.
|
||||
*/
|
||||
void copyCompressedData(DataOutput out) throws IOException {
|
||||
final int chunkSize = chunkSize();
|
||||
decompressor.copyCompressedData(fieldsStream, chunkSize, out);
|
||||
final long chunkEnd = docBase + chunkDocs == numDocs
|
||||
? fieldsStream.length()
|
||||
: indexReader.getStartPointer(docBase + chunkDocs);
|
||||
out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -140,14 +140,6 @@ public abstract class CompressionMode {
|
|||
bytes.length = length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
|
||||
final int copied = LZ4.copyCompressedData(in, originalLength, out);
|
||||
if (copied != originalLength) {
|
||||
throw new CorruptIndexException("Currupted compressed stream: expected " + originalLength + " bytes, but got at least" + copied);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Decompressor clone() {
|
||||
return this;
|
||||
|
@ -224,13 +216,6 @@ public abstract class CompressionMode {
|
|||
bytes.length = length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
|
||||
final int compressedLength = in.readVInt();
|
||||
out.writeVInt(compressedLength);
|
||||
out.copyBytes(in, compressedLength);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Decompressor clone() {
|
||||
return new DeflateDecompressor();
|
||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.lucene.codecs.compressing;
|
|||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.store.DataInput;
|
||||
import org.apache.lucene.store.DataOutput;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
/**
|
||||
|
@ -47,10 +46,6 @@ public abstract class Decompressor implements Cloneable {
|
|||
*/
|
||||
public abstract void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;
|
||||
|
||||
/** Copy a compressed stream whose original length is
|
||||
* <code>originalLength</code> from <code>in</code> to <code>out</code>. */
|
||||
public abstract void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException;
|
||||
|
||||
@Override
|
||||
public abstract Decompressor clone();
|
||||
|
||||
|
|
|
@ -506,51 +506,4 @@ class LZ4 {
|
|||
encodeLastLiterals(src, anchor, srcEnd - anchor, out);
|
||||
}
|
||||
|
||||
/** Copy bytes from <code>in</code> to <code>out</code> where
|
||||
* <code>in</code> is a LZ4-encoded stream. This method copies enough bytes
|
||||
* so that <code>out</code> can be used later on to restore the first
|
||||
* <code>length</code> bytes of the stream. This method always reads at
|
||||
* least one byte from <code>in</code> so make sure not to call this method
|
||||
* if <code>in</code> reached the end of the stream, even if
|
||||
* <code>length=0</code>. */
|
||||
public static int copyCompressedData(DataInput in, int length, DataOutput out) throws IOException {
|
||||
int n = 0;
|
||||
do {
|
||||
// literals
|
||||
final byte token = in.readByte();
|
||||
out.writeByte(token);
|
||||
int literalLen = (token & 0xFF) >>> 4;
|
||||
if (literalLen == 0x0F) {
|
||||
byte len;
|
||||
while ((len = in.readByte()) == (byte) 0xFF) {
|
||||
literalLen += 0xFF;
|
||||
out.writeByte(len);
|
||||
}
|
||||
literalLen += len & 0xFF;
|
||||
out.writeByte(len);
|
||||
}
|
||||
out.copyBytes(in, literalLen);
|
||||
n += literalLen;
|
||||
if (n >= length) {
|
||||
break;
|
||||
}
|
||||
|
||||
// matchs
|
||||
out.copyBytes(in, 2); // match dec
|
||||
int matchLen = token & 0x0F;
|
||||
if (matchLen == 0x0F) {
|
||||
byte len;
|
||||
while ((len = in.readByte()) == (byte) 0xFF) {
|
||||
matchLen += 0xFF;
|
||||
out.writeByte(len);
|
||||
}
|
||||
matchLen += len & 0xFF;
|
||||
out.writeByte(len);
|
||||
}
|
||||
matchLen += MIN_MATCH;
|
||||
n += matchLen;
|
||||
} while (n < length);
|
||||
return n;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -80,16 +80,6 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
|
|||
return Arrays.copyOfRange(bytes.bytes, bytes.offset, bytes.offset + bytes.length);
|
||||
}
|
||||
|
||||
static byte[] copyCompressedData(Decompressor decompressor, byte[] compressed, int originalLength) throws IOException {
|
||||
GrowableByteArrayDataOutput out = new GrowableByteArrayDataOutput(compressed.length);
|
||||
decompressor.copyCompressedData(new ByteArrayDataInput(compressed), originalLength, out);
|
||||
return Arrays.copyOf(out.bytes, out.length);
|
||||
}
|
||||
|
||||
byte[] copyCompressedData(byte[] compressed, int originalLength) throws IOException {
|
||||
return copyCompressedData(mode.newDecompressor(), compressed, originalLength);
|
||||
}
|
||||
|
||||
public void testDecompress() throws IOException {
|
||||
final int iterations = atLeast(10);
|
||||
for (int i = 0; i < iterations; ++i) {
|
||||
|
@ -117,17 +107,10 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
public void testCopyCompressedData() throws IOException {
|
||||
final byte[] decompressed = randomArray();
|
||||
final byte[] compressed = compress(decompressed);
|
||||
assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
|
||||
}
|
||||
|
||||
public byte[] test(byte[] decompressed) throws IOException {
|
||||
final byte[] compressed = compress(decompressed);
|
||||
final byte[] restored = decompress(compressed, decompressed.length);
|
||||
assertEquals(decompressed.length, restored.length);
|
||||
assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
|
||||
return compressed;
|
||||
}
|
||||
|
||||
|
|
|
@ -66,11 +66,6 @@ public class DummyCompressingCodec extends CompressingCodec {
|
|||
bytes.length = length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
|
||||
out.copyBytes(in, originalLength);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Decompressor clone() {
|
||||
return this;
|
||||
|
|
Loading…
Reference in New Issue