honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Verify Checksum once it has been fully written to fail as soon as possible 

Today we are relying on calling Store.verify on the closed stream to validate the
checksum. This is still necessary to catch file truncation but for an actually corrupted
file or checksum we can fail early and check the checksum against the actual metadata
once it's been fully written to the VerifyingIndexOutput.
This commit is contained in:
Simon Willnauer 2015-10-02 01:24:18 +02:00
parent 5fbf3494fe
commit 04e892634e
3 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -1292,6 +1292,9 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
final int index = Math.toIntExact(writtenBytes - checksumPosition);
if (index < footerChecksum.length) {
footerChecksum[index] = b;
if (index == footerChecksum.length-1) {
verify();// we have recorded the entire checksum
}
} else {
verify(); // fail if we write more than expected
throw new AssertionError("write past EOF expected length: " + metadata.length() + " writtenBytes: " + writtenBytes);

1
core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
View File

@ -674,7 +674,6 @@ public class RecoverySourceHandler {
try (final OutputStream outputStream = outputStreamFactory.apply(md);
final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStream);
Store.verify(indexInput);
}
return null;
});

11
core/src/test/java/org/elasticsearch/index/store/StoreTests.java
View File

@ -184,6 +184,7 @@ public class StoreTests extends ESTestCase {
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long length = indexInput.length();
IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetaData("foo1.bar", length, checksum), dir.createOutput("foo1.bar", IOContext.DEFAULT));
length -= 8; // we write the checksum in the try / catch block below
while (length > 0) {
if (random().nextInt(10) == 0) {
verifyingOutput.writeByte(indexInput.readByte());
@ -197,6 +198,16 @@ public class StoreTests extends ESTestCase {
}
try {
BytesRef checksumBytes = new BytesRef(8);
checksumBytes.length = 8;
indexInput.readBytes(checksumBytes.bytes, checksumBytes.offset, checksumBytes.length);
if (randomBoolean()) {
verifyingOutput.writeBytes(checksumBytes.bytes, checksumBytes.offset, checksumBytes.length);
} else {
for (int i = 0; i < checksumBytes.length; i++) {
verifyingOutput.writeByte(checksumBytes.bytes[i]);
}
}
if (randomBoolean()) {
appendRandomData(verifyingOutput);
} else {