honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Stored Compression: failure to fetch document in certain cases (read failure, index compression works), closes #2055.

This commit is contained in:
Shay Banon 2012-06-26 01:54:19 +02:00
parent 6c74f99ed3
commit 12a644c89b
2 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java
View File

@ -40,19 +40,16 @@ public abstract class CompressedIndexInput extends IndexInput {
protected byte[] uncompressed;
private int position = 0;
private int valid = 0;
private long headerLength;
private int currentOffsetIdx;
private long currentOffset;
private long currentOffsetFilePointer;
private long metaDataPosition;
private long currentUncompressedChunkPointer;
public CompressedIndexInput(IndexInput in) throws IOException {
super("compressed(" + in.toString() + ")");
this.in = in;
readHeader(in);
this.version = in.readInt();
metaDataPosition = in.readLong();
headerLength = in.getFilePointer();
long metaDataPosition = in.readLong();
long headerLength = in.getFilePointer();
in.seek(metaDataPosition);
this.uncompressedLength = in.readVLong();
int size = in.readVInt();
@ -61,8 +58,7 @@ public abstract class CompressedIndexInput extends IndexInput {
offsets.set(i, in.readVLong());
}
this.currentOffsetIdx = -1;
this.currentOffset = 0;
this.currentOffsetFilePointer = 0;
this.currentUncompressedChunkPointer = 0;
in.seek(headerLength);
}
@ -132,7 +128,7 @@ public abstract class CompressedIndexInput extends IndexInput {
@Override
public long getFilePointer() {
return currentOffsetFilePointer + position;
return currentUncompressedChunkPointer + position;
}
@Override
@ -147,8 +143,8 @@ public abstract class CompressedIndexInput extends IndexInput {
}
// TODO: optimize so we won't have to readyBuffer on seek, can keep the position around, and set it on readyBuffer in this case
long pointer = offsets.get(idx);
if (pointer != currentOffset) {
if (idx != currentOffsetIdx) {
long pointer = offsets.get(idx);
in.seek(pointer);
position = 0;
valid = 0;
@ -191,8 +187,7 @@ public abstract class CompressedIndexInput extends IndexInput {
return false;
}
currentOffsetIdx++;
currentOffset = offsets.get(currentOffsetIdx);
currentOffsetFilePointer = currentOffset - headerLength;
currentUncompressedChunkPointer = currentOffsetIdx * uncompressed.length;
position = 0;
return (position < valid);
}

16
src/test/java/org/elasticsearch/test/unit/common/compress/CompressIndexInputOutputTests.java
View File

@ -127,6 +127,16 @@ public class CompressIndexInputOutputTests {
long pos3 = out.getFilePointer();
out.writeVInt(4);
out.writeInt(4);
int size = 50;
long[] positions = new long[size];
String[] data = new String[size];
for (int i = 0; i < 50; i++) {
positions[i] = out.getFilePointer();
data[i] = RandomStringGenerator.random(12345);
out.writeString(data[i]);
}
out.close();
//IndexInput in = dir.openInput("test");
@ -138,6 +148,12 @@ public class CompressIndexInputOutputTests {
in.seek(in.getFilePointer() + numBytes);
assertThat(in.readVInt(), equalTo(4));
assertThat(in.readInt(), equalTo(4));
for (int i = 0; i < size; i++) {
in.seek(positions[i]);
assertThat(in.getFilePointer(), equalTo(positions[i]));
assertThat(in.readString(), equalTo(data[i]));
}
}
@Test