Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2540: CommitWithin as an Update Request parameter 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1165754 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jan Høydahl 2011-09-06 16:27:20 +00:00
parent bf9cde01f1
commit ab554500bd
12 changed files with 91 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -340,6 +340,9 @@ Upgrading from Solr 3.3
New Features
----------------------
* SOLR-2540: CommitWithin as an Update Request parameter
You can now specify &commitWithin=N (ms) on the update request (janhoy)
* SOLR-2458: post.jar enhanced to handle JSON, CSV and <optimize> (janhoy)
* LUCENE-3234: add a new parameter hl.phraseLimit for FastVectorHighlighter speed up.

3
solr/contrib/extraction/CHANGES.txt
View File

@ -30,6 +30,9 @@ $Id$
================== Release 3.4.0-dev ==============
* SOLR-2540: CommitWithin as an Update Request parameter
You can now specify &commitWithin=N (ms) on the update request (janhoy)
* SOLR-2743: Remove commons logging. (koji)
================== Release 3.3.0 ==================

1
solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
View File

@ -97,6 +97,7 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
//this is lightweight
autoDetectParser = new AutoDetectParser(config);

21
solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
View File

@ -285,6 +285,27 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {
assertQ(req("extractedContent:Apache"), "//*[@numFound='1']");
}
@Test
public void testCommitWithin() throws Exception {
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
// Load plain text specifying filename
loadLocal("extraction/version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"literal.id", "one",
"fmap.language", "extractedLanguage",
"fmap.content", "extractedContent",
ExtractingParams.RESOURCE_NAME, "extraction/version_control.txt",
"commitWithin", "200"
);
assertQ(req("id:one"), "//*[@numFound='0']");
// TODO: Find better way of testing commitWithin without sleeping?
Thread.sleep(1000);
assertQ(req("id:one"), "//*[@numFound='1']");
}
// Note: If you load a plain text file specifying neither MIME type nor filename, extraction will silently fail. This is because Tika's
// automatic MIME type detection will fail, and it will default to using an empty-string-returning default parser

3
solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
View File

@ -24,7 +24,6 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import static org.apache.solr.handler.XmlUpdateRequestHandler.COMMIT_WITHIN;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
@ -97,7 +96,7 @@ public class BinaryUpdateRequestHandler extends ContentStreamHandlerBase {
AddUpdateCommand addCmd = new AddUpdateCommand(req);
addCmd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
addCmd.commitWithin = params.getInt(COMMIT_WITHIN, -1);
addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
return addCmd;
}

4
solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
View File

@ -22,6 +22,7 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.schema.IndexSchema;
@ -202,7 +203,8 @@ abstract class CSVLoader extends ContentStreamLoader {
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite=params.getBool(OVERWRITE,true);
templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
String sep = params.get(SEPARATOR);
if (sep!=null) {

5
solr/core/src/java/org/apache/solr/handler/JsonLoader.java
View File

@ -26,6 +26,7 @@ import org.apache.noggit.JSONParser;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@ -54,8 +55,8 @@ class JsonLoader extends ContentStreamLoader {
this.processor = processor;
this.req = req;
commitWithin = req.getParams().getInt(XmlUpdateRequestHandler.COMMIT_WITHIN, -1);
overwrite = req.getParams().getBool(XmlUpdateRequestHandler.OVERWRITE, true);
commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
}
@Override

17
solr/core/src/java/org/apache/solr/handler/XMLLoader.java
View File

@ -28,6 +28,7 @@ import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.commons.io.IOUtils;
@ -94,6 +95,7 @@ class XMLLoader extends ContentStreamLoader {
InstantiationException, IllegalAccessException,
TransformerConfigurationException {
AddUpdateCommand addCmd = null;
SolrParams params = req.getParams();
while (true) {
int event = parser.next();
switch (event) {
@ -108,6 +110,9 @@ class XMLLoader extends ContentStreamLoader {
addCmd = new AddUpdateCommand(req);
// First look for commitWithin parameter on the request, will be overwritten for individual <add>'s
addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
for (int i = 0; i < parser.getAttributeCount(); i++) {
String attrName = parser.getAttributeLocalName(i);
String attrVal = parser.getAttributeValue(i);
@ -121,10 +126,14 @@ class XMLLoader extends ContentStreamLoader {
}
} else if ("doc".equals(currTag)) {
XmlUpdateRequestHandler.log.trace("adding doc...");
addCmd.clear();
addCmd.solrDoc = readDoc(parser);
processor.processAdd(addCmd);
if(addCmd != null) {
XmlUpdateRequestHandler.log.trace("adding doc...");
addCmd.clear();
addCmd.solrDoc = readDoc(parser);
processor.processAdd(addCmd);
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unexpected <doc> tag without an <add> tag surrounding it.");
}
} else if (XmlUpdateRequestHandler.COMMIT.equals(currTag) || XmlUpdateRequestHandler.OPTIMIZE.equals(currTag)) {
XmlUpdateRequestHandler.log.trace("parsing " + currTag);

2
solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
View File

@ -45,6 +45,8 @@ public class XmlUpdateRequestHandler extends ContentStreamHandlerBase {
public static final String SOFT_COMMIT = "softCommit";
public static final String OVERWRITE = "overwrite";
// NOTE: This constant is for use with the <add> XML tag, not the HTTP param with same name
public static final String COMMIT_WITHIN = "commitWithin";

9
solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
View File

@ -113,6 +113,15 @@ public class TestCSVLoader extends SolrTestCaseJ4 {
assertQ(req("id:[100 TO 110]"),"//*[@numFound='3']");
}
@Test
public void testCommitWithin() throws Exception {
makeFile("id\n100\n101\n102");
loadLocal("stream.file",filename,"commitWithin","200");
assertQ(req("id:[100 TO 110]"),"//*[@numFound='0']");
Thread.sleep(1000);
assertQ(req("id:[100 TO 110]"),"//*[@numFound='3']");
}
@Test
public void testCommitTrue() throws Exception {
makeFile("id\n100\n101\n102");

29
solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
View File

@ -24,6 +24,10 @@ import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamReader;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
import org.junit.BeforeClass;
import org.junit.Test;
@ -73,4 +77,29 @@ public class XmlUpdateRequestHandlerTest extends SolrTestCaseJ4 {
assertEquals( 3, out.size() );
assertEquals( "[aaa, bbb, bbb]", out.toString() );
}
@Test
public void testCommitWithin() throws Exception
{
String xml =
"<add>" +
" <doc>" +
" <field name=\"id\">12345</field>" +
" <field name=\"name\">kitten</field>" +
" </doc>" +
"</add>";
SolrQueryRequest req = req("commitWithin","100");
SolrQueryResponse rsp = new SolrQueryResponse();
BufferingRequestProcessor p = new BufferingRequestProcessor(null);
XMLLoader loader = new XMLLoader(p, inputFactory);
loader.load(req, rsp, new ContentStreamBase.StringStream(xml));
AddUpdateCommand add = p.addCommands.get(0);
assertEquals(add.commitWithin, 100);
req.close();
}
}

3
solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java
View File

@ -37,6 +37,9 @@ public interface UpdateParams
/** Commit everything after the command completes */
public static String COMMIT = "commit";
/** Commit within a certain time period (in ms) */
public static String COMMIT_WITHIN = "commitWithin";
/** Optimize the index and commit everything after the command completes */
public static String OPTIMIZE = "optimize";