Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@805774 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul 2009-08-19 12:21:22 +00:00
parent 462571235e
commit 27b491ade5
9 changed files with 123 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES.txt
View File

@ -267,7 +267,9 @@ New Features
HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, HTMLStripCharFilter can be used
with an arbitrary Tokenizer. (koji)
68. SOLR-1367: Added callback mechanism for converting DocList to SolrDocumentList in SolrPluginUtils (gsingers)
68. SOLR-1367: Added callback mechanism for converting DocList to SolrDocumentList in SolrPluginUtils (gsingers)
68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
Optimizations

2
src/common/org/apache/solr/common/params/UpdateParams.java
View File

@ -49,4 +49,6 @@ public interface UpdateParams
* If optimizing, set the maximum number of segments left in the index after optimization. 1 is the default (and is equivalent to calling IndexWriter.optimize() in Lucene).
*/
public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
public static final String EXPUNGE_DELETES = "expungeDeletes";
}

2
src/java/org/apache/solr/handler/RequestHandlerUtils.java
View File

@ -68,6 +68,7 @@ public class RequestHandlerUtils
CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, cmd.waitFlush );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes);
cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
req.getCore().getUpdateHandler().commit( cmd );
@ -101,6 +102,7 @@ public class RequestHandlerUtils
CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, cmd.waitFlush );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes);
cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
processor.processCommit( cmd );
return true;

2
src/java/org/apache/solr/handler/XMLLoader.java
View File

@ -155,6 +155,8 @@ class XMLLoader extends ContentStreamLoader {
sawWaitSearcher = true;
} else if (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
} else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
} else {
XmlUpdateRequestHandler.log.warn("unexpected attribute commit/@" + attrName);
}

2
src/java/org/apache/solr/update/CommitUpdateCommand.java
View File

@ -23,6 +23,7 @@ public class CommitUpdateCommand extends UpdateCommand {
public boolean optimize;
public boolean waitFlush;
public boolean waitSearcher=true;
public boolean expungeDeletes = false;
/**
* During optimize, optimize down to <= this many segments. Must be >= 1
@ -39,6 +40,7 @@ public class CommitUpdateCommand extends UpdateCommand {
return "commit(optimize="+optimize
+",waitFlush="+waitFlush
+",waitSearcher="+waitSearcher
+",expungeDeletes="+expungeDeletes
+')';
}
}

5
src/java/org/apache/solr/update/DirectUpdateHandler.java
View File

@ -241,9 +241,10 @@ public class DirectUpdateHandler extends UpdateHandler {
synchronized (this) {
pset.clear();
closeSearcher(); // flush any deletes
if (cmd.optimize) {
if (cmd.optimize || cmd.expungeDeletes) {
openWriter(); // writer needs to be open to optimize
writer.optimize(cmd.maxOptimizeSegments);
if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes);
}
closeWriter();

7
src/java/org/apache/solr/update/DirectUpdateHandler2.java
View File

@ -130,6 +130,7 @@ public class DirectUpdateHandler2 extends UpdateHandler {
AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
AtomicLong deleteByQueryCommands= new AtomicLong();
AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
AtomicLong expungeDeleteCommands = new AtomicLong();
AtomicLong mergeIndexesCommands = new AtomicLong();
AtomicLong commitCommands= new AtomicLong();
AtomicLong optimizeCommands= new AtomicLong();
@ -382,6 +383,8 @@ public class DirectUpdateHandler2 extends UpdateHandler {
if (cmd.optimize) {
optimizeCommands.incrementAndGet();
} else if (cmd.expungeDeletes) {
expungeDeleteCommands.incrementAndGet();
} else {
commitCommands.incrementAndGet();
}
@ -402,6 +405,10 @@ public class DirectUpdateHandler2 extends UpdateHandler {
}
closeWriter();
if (!cmd.optimize && cmd.expungeDeletes) {
openWriter();
writer.expungeDeletes();
}
callPostCommitCallbacks();
if (cmd.optimize) {

6
src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
View File

@ -153,6 +153,12 @@ public class UpdateRequest extends SolrRequest
return this;
}
public UpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
setAction(action, waitFlush, waitSearcher,maxSegments) ;
params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
return this;
}
/**
* @since Solr 1.4
*/

96
src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
View File

@ -17,20 +17,32 @@
package org.apache.solr.update;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermEnum;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexReader;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.RefCounted;
/**
*
@ -247,6 +259,90 @@ public class DirectUpdateHandlerTest extends AbstractSolrTestCase {
);
}
public void testExpungeDeletes() throws Exception {
for (int x = 0; x < 3000; x++) {
addSimpleDoc(x + "");
}
SolrCore core = h.getCore();
UpdateHandler updater = core.getUpdateHandler();
CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
cmtCmd.waitSearcher = true;
updater.commit(cmtCmd);
List<String> todelete = new ArrayList<String>();
Set<String> segsdel = new HashSet<String>();
SegmentReader[] sirs = getSegmentReaders(core);
assertTrue(sirs.length > 6);
todelete.add(getNthIDTerm(2, sirs[0]));
segsdel.add(sirs[0].getSegmentName());
todelete.add(getNthIDTerm(7, sirs[2]));
segsdel.add(sirs[2].getSegmentName());
todelete.add(getNthIDTerm(4, sirs[5]));
segsdel.add(sirs[5].getSegmentName());
for (String id : todelete) {
deleteSimpleDoc(id);
}
// commit the deletes
cmtCmd = new CommitUpdateCommand(false);
cmtCmd.waitSearcher = true;
updater.commit(cmtCmd);
// expunge deletes
cmtCmd = new CommitUpdateCommand(false);
cmtCmd.waitSearcher = true;
cmtCmd.expungeDeletes = true;
updater.commit(cmtCmd);
// we'll have fewer segments
SegmentReader[] sirs2 = getSegmentReaders(core);
assertTrue(sirs.length > sirs2.length);
// check the actual segment names
for (SegmentReader sr : sirs2) {
assertTrue(!segsdel.contains(sr.getSegmentName()));
}
}
SegmentReader[] getSegmentReaders(SolrCore core) throws IOException {
RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, null);
SolrIndexSearcher is = ref.get();
SegmentReader[] segmentReaders = null;
try {
SolrIndexReader reader = is.getReader();
IndexReader[] subreaders = reader.getSequentialSubReaders();
segmentReaders = new SegmentReader[subreaders.length];
for (int x = 0; x < subreaders.length; x++) {
assert subreaders[x] instanceof SolrIndexReader;
SolrIndexReader sir = (SolrIndexReader) subreaders[x];
SegmentReader sr = (SegmentReader) sir.getWrappedReader();
segmentReaders[x] = sr;
}
} finally {
ref.decref();
}
return segmentReaders;
}
private String getNthIDTerm(int n, IndexReader r) throws IOException {
TermEnum te = r.terms(new Term("id", ""));
try {
int x = 0;
do {
if (x >= n) {
return te.term().text();
}
x++;
} while (te.next());
} finally {
te.close();
}
return null;
}
private void addSimpleDoc(String id) throws Exception {
SolrCore core = h.getCore();