honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13848 from s1monw/verify_written_checksum 

Verify actually written checksum in VerifyingIndexOutput
This commit is contained in:
Simon Willnauer 2015-09-30 15:36:46 +02:00
parent 77c2f151d2 c185a12ef8
commit f526754d0e
2 changed files with 106 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
core/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@ -1259,27 +1260,42 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
private long writtenBytes;
private final long checksumPosition;
private String actualChecksum;
private final byte[] footerChecksum = new byte[8]; // this holds the actual footer checksum data written by to this output
LuceneVerifyingIndexOutput(StoreFileMetaData metadata, IndexOutput out) {
super(out);
this.metadata = metadata;
checksumPosition = metadata.length() - 8; // the last 8 bytes are the checksum
checksumPosition = metadata.length() - 8; // the last 8 bytes are the checksum - we store it in footerChecksum
}
@Override
public void verify() throws IOException {
String footerDigest = null;
if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) {
return;
ByteArrayIndexInput indexInput = new ByteArrayIndexInput("checksum", this.footerChecksum);
footerDigest = digestToString(indexInput.readLong());
if (metadata.checksum().equals(footerDigest)) {
return;
}
}
throw new CorruptIndexException("verification failed (hardware problem?) : expected=" + metadata.checksum() +
" actual=" + actualChecksum + " writtenLength=" + writtenBytes + " expectedLength=" + metadata.length() +
" actual=" + actualChecksum + " footer=" + footerDigest +" writtenLength=" + writtenBytes + " expectedLength=" + metadata.length() +
" (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")");
}
@Override
public void writeByte(byte b) throws IOException {
if (writtenBytes++ == checksumPosition) {
final long writtenBytes = this.writtenBytes++;
if (writtenBytes == checksumPosition) {
readAndCompareChecksum();
} else if (writtenBytes > checksumPosition) { // we are writing parts of the checksum....
final int index = Math.toIntExact(writtenBytes - checksumPosition);
if (index < footerChecksum.length) {
footerChecksum[index] = b;
} else {
verify(); // fail if we write more than expected
throw new AssertionError("write past EOF expected length: " + metadata.length() + " writtenBytes: " + writtenBytes);
}
}
out.writeByte(b);
}
@ -1295,17 +1311,23 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
if (writtenBytes + length > checksumPosition && actualChecksum == null) {
assert writtenBytes <= checksumPosition;
final int bytesToWrite = (int) (checksumPosition - writtenBytes);
out.writeBytes(b, offset, bytesToWrite);
readAndCompareChecksum();
offset += bytesToWrite;
length -= bytesToWrite;
writtenBytes += bytesToWrite;
if (writtenBytes + length > checksumPosition) {
if (actualChecksum == null) {
assert writtenBytes <= checksumPosition;
final int bytesToWrite = (int) (checksumPosition - writtenBytes);
out.writeBytes(b, offset, bytesToWrite);
readAndCompareChecksum();
offset += bytesToWrite;
length -= bytesToWrite;
writtenBytes += bytesToWrite;
}
for (int i = 0; i < length; i++) {
writeByte(b[offset+i]);
}
} else {
out.writeBytes(b, offset, length);
writtenBytes += length;
}
out.writeBytes(b, offset, length);
writtenBytes += length;
}
}

71
core/src/test/java/org/elasticsearch/index/store/StoreTests.java
View File

@ -149,16 +149,85 @@ public class StoreTests extends ESTestCase {
}
}
Store.verify(verifyingOutput);
verifyingOutput.writeByte((byte) 0x0);
try {
appendRandomData(verifyingOutput);
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
try {
Store.verify(verifyingOutput);
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(indexInput, verifyingOutput, dir);
}
public void testChecksumCorrupted() throws IOException {
Directory dir = newDirectory();
IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
output.writeInt(CodecUtil.FOOTER_MAGIC);
output.writeInt(0);
String checksum = Store.digestToString(output.getChecksum());
output.writeLong(output.getChecksum() + 1); // write a wrong checksum to the file
output.close();
IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
indexInput.seek(0);
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long length = indexInput.length();
IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetaData("foo1.bar", length, checksum), dir.createOutput("foo1.bar", IOContext.DEFAULT));
while (length > 0) {
if (random().nextInt(10) == 0) {
verifyingOutput.writeByte(indexInput.readByte());
length--;
} else {
int min = (int) Math.min(length, ref.bytes.length);
indexInput.readBytes(ref.bytes, ref.offset, min);
verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
length -= min;
}
}
try {
if (randomBoolean()) {
appendRandomData(verifyingOutput);
} else {
Store.verify(verifyingOutput);
}
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(indexInput, verifyingOutput, dir);
}
private void appendRandomData(IndexOutput output) throws IOException {
int numBytes = randomIntBetween(1, 1024);
final BytesRef ref = new BytesRef(scaledRandomIntBetween(1, numBytes));
ref.length = ref.bytes.length;
while (numBytes > 0) {
if (random().nextInt(10) == 0) {
output.writeByte(randomByte());
numBytes--;
} else {
for (int i = 0; i<ref.length; i++) {
ref.bytes[i] = randomByte();
}
final int min = Math.min(numBytes, ref.bytes.length);
output.writeBytes(ref.bytes, ref.offset, min);
numBytes -= min;
}
}
}
@Test
public void testVerifyingIndexOutputWithBogusInput() throws IOException {
Directory dir = newDirectory();