Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5214: Reduce memory usage for shard splitting by merging segments one at a time 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1553644 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2013-12-27 08:36:23 +00:00
parent ca4acb2bb7
commit 49209e1d16
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -236,6 +236,9 @@ Optimizations
* SOLR-2960: fix DIH XPathEntityProcessor to add the correct number of "null"
placeholders for multi-valued fields (Michael Watts via James Dyer)
* SOLR-5214: Reduce memory usage for shard splitting by merging segments one
at a time. (Christine Poerschke via shalin)
Other Changes
---------------------

15
solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
View File

@ -105,13 +105,8 @@ public class SolrIndexSplitter {
// - need to worry about if IW.addIndexes does a sync or not...
// - would be more efficient on the read side, but prob less efficient merging
IndexReader[] subReaders = new IndexReader[leaves.size()];
for (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {
log.info("SolrIndexSplitter: partition #" + partitionNumber + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
for (int segmentNumber = 0; segmentNumber<subReaders.length; segmentNumber++) {
subReaders[segmentNumber] = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
}
log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
boolean success = false;
@ -130,8 +125,12 @@ public class SolrIndexSplitter {
}
try {
// This merges the subreaders and will thus remove deletions (i.e. no optimize needed)
iw.addIndexes(subReaders);
// This removes deletions but optimize might still be needed because sub-shards will have the same number of segments as the parent shard.
for (int segmentNumber = 0; segmentNumber<leaves.size(); segmentNumber++) {
log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : "") + " segment #"+segmentNumber + " segmentCount=" + leaves.size());
IndexReader subReader = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
iw.addIndexes(subReader);
}
success = true;
} finally {
if (iwRef != null) {