mirror of https://github.com/apache/lucene.git
SOLR-5213: Log when shard splitting unexpectedly leads to documents going to zero or multiple sub-shards
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1676075 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
40dda65fb2
commit
0c7715812e
|
@ -228,6 +228,9 @@ Other Changes
|
|||
* SOLR-7391: Use a time based expiration cache for one off HDFS FileSystem instances.
|
||||
(Mark Miller)
|
||||
|
||||
* SOLR-5213: Log when shard splitting unexpectedly leads to documents going to
|
||||
no or multiple shards (Christine Poerschke, Ramkumar Aiyengar)
|
||||
|
||||
================== 5.1.0 ==================
|
||||
|
||||
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
|
||||
|
|
|
@ -168,6 +168,12 @@ public class SolrIndexSplitter {
|
|||
BytesRef term = null;
|
||||
PostingsEnum postingsEnum = null;
|
||||
|
||||
int[] docsMatchingRanges = null;
|
||||
if (ranges != null) {
|
||||
// +1 because documents can belong to *zero*, one, several or all ranges in rangesArr
|
||||
docsMatchingRanges = new int[rangesArr.length+1];
|
||||
}
|
||||
|
||||
CharsRefBuilder idRef = new CharsRefBuilder();
|
||||
for (;;) {
|
||||
term = termsEnum.next();
|
||||
|
@ -203,11 +209,37 @@ public class SolrIndexSplitter {
|
|||
docSets[currPartition].set(doc);
|
||||
currPartition = (currPartition + 1) % numPieces;
|
||||
} else {
|
||||
int matchingRangesCount = 0;
|
||||
for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
|
||||
if (rangesArr[i].includes(hash)) {
|
||||
docSets[i].set(doc);
|
||||
++matchingRangesCount;
|
||||
}
|
||||
}
|
||||
docsMatchingRanges[matchingRangesCount]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (docsMatchingRanges != null) {
|
||||
for (int ii = 0; ii < docsMatchingRanges.length; ii++) {
|
||||
if (0 == docsMatchingRanges[ii]) continue;
|
||||
switch (ii) {
|
||||
case 0:
|
||||
// document loss
|
||||
log.error("Splitting {}: {} documents belong to no shards and will be dropped",
|
||||
reader, docsMatchingRanges[ii]);
|
||||
break;
|
||||
case 1:
|
||||
// normal case, each document moves to one of the sub-shards
|
||||
log.info("Splitting {}: {} documents will move into a sub-shard",
|
||||
reader, docsMatchingRanges[ii]);
|
||||
break;
|
||||
default:
|
||||
// document duplication
|
||||
log.error("Splitting {}: {} documents will be moved to multiple ({}) sub-shards",
|
||||
reader, docsMatchingRanges[ii], ii);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue