Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6920, SOLR-6640: When we so not have a checksum for a file, always download files under 100kb and other small improvements. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1658078 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2015-02-07 17:00:08 +00:00
parent 376256316b
commit 704626ac47
2 changed files with 33 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -449,12 +449,17 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
// add the segments_N file
Map<String,Object> fileMeta = new HashMap<>();
fileMeta.put(NAME, infos.getSegmentsFileName());
fileMeta.put(SIZE, dir.fileLength(infos.getSegmentsFileName()));
if (infos.getId() != null) {
try (final IndexInput in = dir.openInput(infos.getSegmentsFileName(), IOContext.READONCE)) {
fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in));
try {
fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in));
} catch(Exception e) {
LOG.warn("Could not read checksum from index file.", e);
}
}
}
result.add(fileMeta);

37
solr/core/src/java/org/apache/solr/handler/SnapPuller.java
View File

@ -802,10 +802,10 @@ public class SnapPuller {
}
for (Map<String,Object> file : filesToDownload) {
String filename = (String) file.get(NAME);
CompareResult compareResult = compareFile(indexDir, filename, (Long) file.get(SIZE), (Long) file.get(CHECKSUM));
long size = (Long) file.get(SIZE);
CompareResult compareResult = compareFile(indexDir, filename, size, (Long) file.get(CHECKSUM));
if (!compareResult.equal || downloadCompleteIndex
|| (!compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv")
|| filename.startsWith("segments_")))) {
|| filesToAlwaysDownloadIfChecksumFails(filename, size, compareResult)) {
dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file,
(String) file.get(NAME), false, latestGeneration);
currentFile = file;
@ -817,6 +817,14 @@ public class SnapPuller {
}
}
}
private boolean filesToAlwaysDownloadIfChecksumFails(String filename,
long size, CompareResult compareResult) {
// without checksums to compare, we always download .si, .liv, segments_N,
// and any file under 100kb
return !compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv")
|| filename.startsWith("segments_") || size < 100000);
}
static class CompareResult {
boolean equal = false;
@ -830,24 +838,33 @@ public class SnapPuller {
long indexFileLen = indexInput.length();
long indexFileChecksum = 0;
try {
indexFileChecksum = CodecUtil.retrieveChecksum(indexInput);
compareResult.checkSummed = true;
} catch (Exception e) {
LOG.warn("Could not retrieve checksum from file.", e);
if (backupIndexFileChecksum != null) {
try {
indexFileChecksum = CodecUtil.retrieveChecksum(indexInput);
compareResult.checkSummed = true;
} catch (Exception e) {
LOG.warn("Could not retrieve checksum from file.", e);
}
}
if (!compareResult.checkSummed) {
// we don't have checksums to compare
if (indexFileLen == backupIndexFileLen) {
compareResult.equal = true;
return compareResult;
} else {
LOG.warn("File {} did not match. expected checksum is {} and actual is checksum {}. " +
"expected length is {} and actual length is {}", filename, backupIndexFileChecksum, indexFileChecksum,
LOG.warn(
"File {} did not match. " + "expected length is {} and actual length is {}",
filename, backupIndexFileChecksum, indexFileChecksum,
backupIndexFileLen, indexFileLen);
compareResult.equal = false;
return compareResult;
}
}
// we have checksums to compare
if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) {
compareResult.equal = true;
return compareResult;