From 97e369ace89549d6b2397f0edac61b51269e9d39 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 14 Oct 2008 00:24:04 +0000 Subject: [PATCH] SOLR-793: Add 'commitWithin' argument to the update add command. This behaves similar to the global autoCommit maxTime argument except that it is set for each request. git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@704288 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 4 + SOLR-620.patch | 287 ++++++++ SOLR-657.patch | 675 ++++++++++++++++++ SOLR-793-commitWithin.patch | 221 ++++++ SOLR-793-commitWithin2.patch | 221 ++++++ .../client/solrj/request/UpdateRequest.java | 16 +- .../solr/client/solrj/SolrExampleTests.java | 37 +- example/solr/conf/velocity/default.vm | 40 ++ lib/commons-collections-3.2.jar | 2 + lib/commons-lang-2.4.jar | 2 + lib/velocity-1.6-dev.jar | 2 + .../solr/handler/XmlUpdateRequestHandler.java | 10 + .../solr/request/VelocityResponseWriter.java | 222 ++++++ .../apache/solr/update/AddUpdateCommand.java | 4 + .../solr/update/DirectUpdateHandler2.java | 29 +- 15 files changed, 1750 insertions(+), 22 deletions(-) create mode 100644 SOLR-620.patch create mode 100644 SOLR-657.patch create mode 100644 SOLR-793-commitWithin.patch create mode 100644 SOLR-793-commitWithin2.patch create mode 100644 example/solr/conf/velocity/default.vm create mode 100644 lib/commons-collections-3.2.jar create mode 100644 lib/commons-lang-2.4.jar create mode 100644 lib/velocity-1.6-dev.jar create mode 100644 src/java/org/apache/solr/request/VelocityResponseWriter.java diff --git a/CHANGES.txt b/CHANGES.txt index 9ddb903100c..9ab44f47624 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -49,6 +49,10 @@ New Features 4. SOLR-658: Allow Solr to load index from arbitrary directory in dataDir (Noble Paul, Akshay Ukey via shalin) + 5. SOLR-793: Add 'commitWithin' argument to the update add command. This behaves + similar to the global autoCommit maxTime argument except that it is set for + each request. (ryan) + Optimizations ---------------------- diff --git a/SOLR-620.patch b/SOLR-620.patch new file mode 100644 index 00000000000..bb7ff828c92 --- /dev/null +++ b/SOLR-620.patch @@ -0,0 +1,287 @@ +Index: src/java/org/apache/solr/request/VelocityResponseWriter.java +=================================================================== +--- src/java/org/apache/solr/request/VelocityResponseWriter.java (Revision 0) ++++ src/java/org/apache/solr/request/VelocityResponseWriter.java (Revision 0) +@@ -0,0 +1,222 @@ ++/** ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.solr.request; ++ ++import java.io.File; ++import java.io.IOException; ++import java.io.StringWriter; ++import java.io.Writer; ++import java.util.Iterator; ++ ++import org.apache.lucene.document.Document; ++import org.apache.lucene.index.CorruptIndexException; ++import org.apache.lucene.search.Searcher; ++import org.apache.solr.client.solrj.SolrResponse; ++import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; ++import org.apache.solr.common.SolrDocument; ++import org.apache.solr.common.util.NamedList; ++import org.apache.solr.search.DocSlice; ++import org.apache.solr.update.DocumentBuilder; ++import org.apache.velocity.Template; ++import org.apache.velocity.VelocityContext; ++import org.apache.velocity.app.VelocityEngine; ++ ++/** ++ *

A response writer that uses velocity template for response creation. ++ * Possible request parameters:

++ * ++ */ ++public class VelocityResponseWriter implements QueryResponseWriter { ++ ++ private static final String PARAMETER_TEMPLATE="vl.template"; ++ private static final String PARAMETER_JSON="vl.json"; ++ private static final String PARAMETER_RESPONSE="vl.response"; ++ private static final String PARAMETER_CONTENT_TYPE="vl.content"; ++ ++ public void write(Writer writer, SolrQueryRequest request, ++ SolrQueryResponse response) throws IOException { ++ ++ // init velocity and get template ++ VelocityEngine engine = new VelocityEngine(); ++ File baseDir = new File(request.getCore().getResourceLoader().getConfigDir(), "velocity"); ++ engine.setProperty(VelocityEngine.FILE_RESOURCE_LOADER_PATH, baseDir.getAbsolutePath()); ++ engine.setProperty(VelocityEngine.RESOURCE_LOADER, "file"); ++ Template template; ++ try { ++ template = engine.getTemplate(request.getParams().get(PARAMETER_TEMPLATE, "default") + ".vm"); ++ } catch (Exception e) { ++ throw new IOException(e.getMessage()); ++ } ++ ++ // put raw response into context ++ VelocityContext context = new VelocityContext(); ++ context.put("rawResponse", new RawResponseHelper(request, response)); ++ ++ // convert response if a class is specified ++ if (request.getParams().get(PARAMETER_RESPONSE) != null) { ++ String className = request.getParams().get(PARAMETER_RESPONSE); ++ ++ // create SolrResponse using reflection ++ SolrResponse solrResponse; ++ Object object; ++ try { ++ object = request.getCore().getResourceLoader().newInstance(className, "client.solrj.response."); ++ } catch (RuntimeException e) { ++ throw new IOException("Unable to resolve response class \"" + className + "\": " + e.getMessage()); ++ } ++ if (!(object instanceof SolrResponse)) { ++ throw new IOException("Class \"" + className + "\" doesn't implement SolrResponse!"); ++ } ++ solrResponse = (SolrResponse) object; ++ ++ // inject the request into the response ++ solrResponse.setResponse(new EmbeddedSolrServer(request.getCore()).getParsedResponse(request, response)); ++ ++ // put it into the context ++ context.put("response", solrResponse); ++ } ++ ++ // create output, optionally wrap it into a json object ++ if (isWrappedResponse(request)) { ++ StringWriter stringWriter = new StringWriter(); ++ template.merge(context, stringWriter); ++ writer.write(request.getParams().get(PARAMETER_JSON) + "("); ++ writer.write(getJSONWrap(stringWriter.toString())); ++ writer.write(')'); ++ } else { ++ template.merge(context, writer); ++ } ++ ++ } ++ ++ public String getContentType(SolrQueryRequest request, ++ SolrQueryResponse response) { ++ if (request.getParams().get(PARAMETER_CONTENT_TYPE) != null) { ++ return request.getParams().get(PARAMETER_CONTENT_TYPE); ++ } ++ if (isWrappedResponse(request)) { ++ return JSONResponseWriter.CONTENT_TYPE_JSON_UTF8; ++ } ++ return "text/html"; ++ } ++ ++ public void init(NamedList args) { ++ // TODO ++ } ++ ++ private boolean isWrappedResponse(SolrQueryRequest request) { ++ return request.getParams().get(PARAMETER_JSON) != null; ++ } ++ ++ public String getJSONWrap(String xmlResult) { ++ // escape the double quotes and backslashes ++ String replace1 = xmlResult.replaceAll("\\\\", "\\\\\\\\"); ++ replace1 = replace1.replaceAll("\\n", "\\\\n"); ++ replace1 = replace1.replaceAll("\\r", "\\\\r"); ++ String replaced = replace1.replaceAll("\"", "\\\\\""); ++ // wrap it in a JSON object ++ return "{\"result\":\"" + replaced + "\"}"; ++ } ++ ++ /** ++ * A helper class that provides convenient methods for the raw solr response. ++ */ ++ public class RawResponseHelper { ++ ++ private Searcher searcher; ++ private SolrQueryResponse response; ++ private SolrQueryRequest request; ++ ++ public RawResponseHelper(SolrQueryRequest request, ++ SolrQueryResponse response) { ++ this.searcher = request.getSearcher(); ++ this.response = response; ++ this.request = request; ++ } ++ ++ public Iterator getResultIterator() { ++ final Iterator iterator = ((DocSlice) response.getValues() ++ .get("response")).iterator(); ++ return new Iterator() { ++ ++ public boolean hasNext() { ++ return iterator.hasNext(); ++ } ++ ++ public SolrDocument next() { ++ Document document = null; ++ SolrDocument solrDocument = new SolrDocument(); ++ try { ++ document = searcher.doc(iterator.next()); ++ new DocumentBuilder(request.getSchema()).loadStoredFields(solrDocument, document); ++ } catch (CorruptIndexException e) { ++ throw new RuntimeException("Error converting lucene document into solr document!"); ++ } catch (IOException e) { ++ throw new RuntimeException("Error converting lucene document into solr document!"); ++ } ++ ++ return solrDocument; ++ } ++ ++ public void remove() { ++ ++ } ++ ++ }; ++ } ++ ++ public String getRequestParameter(String param) { ++ return request.getParams().get(param); ++ } ++ ++ public SolrQueryRequest getRequest() { ++ return request; ++ } ++ ++ public SolrQueryResponse getResponse() { ++ return response; ++ } ++ ++ ++ } ++ ++ ++} +Index: example/solr/conf/solrconfig.xml +=================================================================== +--- example/solr/conf/solrconfig.xml (Revision 679745) ++++ example/solr/conf/solrconfig.xml (Arbeitskopie) +@@ -726,7 +726,9 @@ + --> + + 5 +- ++ ++ ++ + + + +Index: example/solr/conf/velocity/default.vm +=================================================================== +--- example/solr/conf/velocity/default.vm (Revision 0) ++++ example/solr/conf/velocity/default.vm (Revision 0) +@@ -0,0 +1,40 @@ ++#set($iterator=$rawResponse.getResultIterator()) ++#set($request=$rawResponse.request) ++#set($core=$request.core) ++#set($schema=$core.schema) ++ ++ ++ Default VelocityResponseWriter Response ++ ++ ++

++ A default response using only the raw solr response. The "rawResponse" object is a helper that provides access to the ++ SolrQueryRequest, SolrQueryResponse, as well as convenience methods like getResultIterator() and getRequestParam() ++

++

++ If you want to use an instance of SolrResponse of solrj inside a template, use eg. vl.response=QueryResponse. ++ This object then is avalaible as "response" in the context. ++

++
++ ++ #foreach($document in $iterator) ++ #foreach($field in $document.getFieldNames()) ++

$field: $document.getFieldValue($field)

++ #end ++
++ #end ++ ++ 
++      header = $rawResponse.$response.responseHeader
++
++      values = $rawResponse.$response.values
++
++      request = $request
++
++      core = $core
++
++      schema = $schema
++
++ ++ ++ diff --git a/SOLR-657.patch b/SOLR-657.patch new file mode 100644 index 00000000000..909ac9c7bf2 --- /dev/null +++ b/SOLR-657.patch @@ -0,0 +1,675 @@ +diff --git a/src/java/org/apache/solr/analysis/SynonymFilterFactory.java b/src/java/org/apache/solr/analysis/SynonymFilterFactory.java +index 41cee1f..6cc6c4a 100644 +--- a/src/java/org/apache/solr/analysis/SynonymFilterFactory.java ++++ b/src/java/org/apache/solr/analysis/SynonymFilterFactory.java +@@ -134,7 +134,7 @@ public class SynonymFilterFactory extends BaseTokenFilterFactory implements Reso + List tokList = new ArrayList(); + try { + for( Token token = ts.next(); token != null; token = ts.next() ){ +- String text = token.termText(); ++ String text = new String(token.termBuffer(), 0, token.termLength()); + if( text.length() > 0 ) + tokList.add( text ); + } +diff --git a/src/java/org/apache/solr/core/QuerySenderListener.java b/src/java/org/apache/solr/core/QuerySenderListener.java +index 78777cf..f24c532 100644 +--- a/src/java/org/apache/solr/core/QuerySenderListener.java ++++ b/src/java/org/apache/solr/core/QuerySenderListener.java +@@ -20,6 +20,7 @@ package org.apache.solr.core; + import org.apache.solr.search.SolrIndexSearcher; + import org.apache.solr.search.DocList; + import org.apache.solr.search.DocIterator; ++import org.apache.solr.common.params.CommonParams; + import org.apache.solr.common.util.NamedList; + import org.apache.solr.request.LocalSolrQueryRequest; + import org.apache.solr.request.SolrQueryResponse; +@@ -47,7 +48,7 @@ class QuerySenderListener extends AbstractSolrEventListener { + }; + + SolrQueryResponse rsp = new SolrQueryResponse(); +- core.execute(req,rsp); ++ core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); + + // Retrieve the Document instances (not just the ids) to warm + // the OS disk cache, and any Solr document cache. Only the top +diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java +index 0f1e656..c2f2169 100644 +--- a/src/java/org/apache/solr/core/SolrCore.java ++++ b/src/java/org/apache/solr/core/SolrCore.java +@@ -38,7 +38,7 @@ import org.apache.solr.schema.IndexSchema; + import org.apache.solr.search.QParserPlugin; + import org.apache.solr.search.SolrIndexSearcher; + import org.apache.solr.search.ValueSourceParser; +-import org.apache.solr.update.DirectUpdateHandler; ++import org.apache.solr.update.DirectUpdateHandler2; + import org.apache.solr.update.SolrIndexWriter; + import org.apache.solr.update.UpdateHandler; + import org.apache.solr.update.processor.LogUpdateProcessorFactory; +@@ -470,7 +470,7 @@ public final class SolrCore implements SolrInfoMBean { + getSearcher(false,false,null); + + updateHandler = createUpdateHandler( +- solrConfig.get("updateHandler/@class", DirectUpdateHandler.class.getName()) ++ solrConfig.get("updateHandler/@class", DirectUpdateHandler2.class.getName()) + ); + + infoRegistry.put("updateHandler", updateHandler); +@@ -1190,8 +1190,8 @@ public final class SolrCore implements SolrInfoMBean { + + public void execute(SolrRequestHandler handler, SolrQueryRequest req, SolrQueryResponse rsp) { + if (handler==null) { +- log.warning(logid+"Null Request Handler '" + req.getQueryType() +"' :" + req); +- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Null Request Handler '" + req.getQueryType() + "'", true); ++ log.warning(logid+"Null Request Handler '" + req.getParams().get(CommonParams.QT) +"' :" + req); ++ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Null Request Handler '" + req.getParams().get(CommonParams.QT) + "'", true); + } + // setup response header and handle request + final NamedList responseHeader = new SimpleOrderedMap(); +@@ -1320,7 +1320,7 @@ public final class SolrCore implements SolrInfoMBean { + * 'wt' parameter, attempts to find that one; otherwise return the default writer. + */ + public final QueryResponseWriter getQueryResponseWriter(SolrQueryRequest request) { +- return getQueryResponseWriter(request.getParam("wt")); ++ return getQueryResponseWriter(request.getParams().get(CommonParams.WT)); + } + + private final Map qParserPlugins = new HashMap(); +diff --git a/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +index 046cbc2..ead5c65 100644 +--- a/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java ++++ b/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +@@ -376,7 +376,7 @@ public class LukeRequestHandler extends RequestHandlerBase + finfo.add("fields", fields); + finfo.add("dynamicFields", dynamicFields); + finfo.add("uniqueKeyField", uniqueField.getName()); +- finfo.add("defaultSearchField", schema.getDefaultSearchFieldName()); ++ finfo.add("defaultSearchField", schema.getSolrQueryParser(null).getField()); + finfo.add("types", types); + return finfo; + } +diff --git a/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java b/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java +index ba2adda..51ad696 100644 +--- a/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java ++++ b/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java +@@ -73,7 +73,7 @@ public class SystemInfoHandler extends RequestHandlerBase + SimpleOrderedMap info = new SimpleOrderedMap(); + + IndexSchema schema = core.getSchema(); +- info.add( "schema", schema != null ? schema.getName():"no schema!" ); ++ info.add( "schema", schema != null ? schema.getSchemaName():"no schema!" ); + + // Host + InetAddress addr = InetAddress.getLocalHost(); +diff --git a/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +index 5cbd5f6..a12d140 100644 +--- a/src/java/org/apache/solr/handler/component/QueryElevationComponent.java ++++ b/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +@@ -299,7 +299,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore + TokenStream tokens = analyzer.tokenStream( null, new StringReader( query ) ); + Token token = tokens.next(); + while( token != null ) { +- norm.append( token.termText() ); ++ norm.append( new String(token.termBuffer(), 0, token.termLength()) ); + token = tokens.next(); + } + return norm.toString(); +diff --git a/src/java/org/apache/solr/request/TextResponseWriter.java b/src/java/org/apache/solr/request/TextResponseWriter.java +index f7a21c6..b377650 100644 +--- a/src/java/org/apache/solr/request/TextResponseWriter.java ++++ b/src/java/org/apache/solr/request/TextResponseWriter.java +@@ -18,6 +18,7 @@ + package org.apache.solr.request; + + import org.apache.lucene.document.Document; ++import org.apache.solr.common.params.CommonParams; + import org.apache.solr.common.util.NamedList; + import org.apache.solr.common.util.FastWriter; + import org.apache.solr.common.SolrDocument; +@@ -54,7 +55,7 @@ public abstract class TextResponseWriter { + this.schema = req.getSchema(); + this.req = req; + this.rsp = rsp; +- String indent = req.getParam("indent"); ++ String indent = req.getParams().get("indent"); + if (indent != null && !"".equals(indent) && !"off".equals(indent)) { + doIndent=true; + } +diff --git a/src/java/org/apache/solr/request/XMLWriter.java b/src/java/org/apache/solr/request/XMLWriter.java +index 1735159..c6f5a05 100644 +--- a/src/java/org/apache/solr/request/XMLWriter.java ++++ b/src/java/org/apache/solr/request/XMLWriter.java +@@ -19,6 +19,7 @@ package org.apache.solr.request; + + import org.apache.solr.common.SolrDocument; + import org.apache.solr.common.SolrDocumentList; ++import org.apache.solr.common.params.CommonParams; + import org.apache.solr.common.util.NamedList; + import org.apache.solr.common.util.XML; + import org.apache.solr.search.SolrIndexSearcher; +@@ -61,18 +62,18 @@ final public class XMLWriter { + + public static void writeResponse(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + +- String ver = req.getParam("version"); ++ String ver = req.getParams().get(CommonParams.VERSION); + + writer.write(XML_START1); + +- String stylesheet = req.getParam("stylesheet"); ++ String stylesheet = req.getParams().get("stylesheet"); + if (stylesheet != null && stylesheet.length() > 0) { + writer.write(XML_STYLESHEET); + writer.write(stylesheet); + writer.write(XML_STYLESHEET_END); + } + +- String noSchema = req.getParam("noSchema"); ++ String noSchema = req.getParams().get("noSchema"); + // todo - change when schema becomes available? + if (false && noSchema == null) + writer.write(XML_START2_SCHEMA); +@@ -87,7 +88,7 @@ final public class XMLWriter { + XMLWriter xw = new XMLWriter(writer, req.getSchema(), req, ver); + xw.defaultFieldList = rsp.getReturnFields(); + +- String indent = req.getParam("indent"); ++ String indent = req.getParams().get("indent"); + if (indent != null) { + if ("".equals(indent) || "off".equals(indent)) { + xw.setIndent(false); +diff --git a/src/java/org/apache/solr/search/FieldQParserPlugin.java b/src/java/org/apache/solr/search/FieldQParserPlugin.java +index ed3c5ba..1dca87a 100644 +--- a/src/java/org/apache/solr/search/FieldQParserPlugin.java ++++ b/src/java/org/apache/solr/search/FieldQParserPlugin.java +@@ -97,7 +97,7 @@ public class FieldQParserPlugin extends QParserPlugin { + return null; + else if (lst.size() == 1) { + t = lst.get(0); +- return new TermQuery(new Term(field, t.termText())); ++ return new TermQuery(new Term(field, new String(t.termBuffer(), 0, t.termLength()))); + } else { + if (severalTokensAtSamePosition) { + if (positionCount == 1) { +@@ -106,7 +106,7 @@ public class FieldQParserPlugin extends QParserPlugin { + for (int i = 0; i < lst.size(); i++) { + t = (org.apache.lucene.analysis.Token) lst.get(i); + TermQuery currentQuery = new TermQuery( +- new Term(field, t.termText())); ++ new Term(field, new String(t.termBuffer(), 0, t.termLength()))); + q.add(currentQuery, BooleanClause.Occur.SHOULD); + } + return q; +@@ -122,7 +122,7 @@ public class FieldQParserPlugin extends QParserPlugin { + mpq.add((Term[])multiTerms.toArray(new Term[0])); + multiTerms.clear(); + } +- multiTerms.add(new Term(field, t.termText())); ++ multiTerms.add(new Term(field, new String(t.termBuffer(), 0, t.termLength()))); + } + mpq.add((Term[])multiTerms.toArray(new Term[0])); + return mpq; +@@ -132,7 +132,8 @@ public class FieldQParserPlugin extends QParserPlugin { + PhraseQuery q = new PhraseQuery(); + q.setSlop(phraseSlop); + for (int i = 0; i < lst.size(); i++) { +- q.add(new Term(field, lst.get(i).termText())); ++ Token token = lst.get(i); ++ q.add(new Term(field, new String(token.termBuffer(), 0, token.termLength()))); + } + return q; + } +diff --git a/src/java/org/apache/solr/search/LuceneQParserPlugin.java b/src/java/org/apache/solr/search/LuceneQParserPlugin.java +index 33a1fbe..5ae9cd6 100755 +--- a/src/java/org/apache/solr/search/LuceneQParserPlugin.java ++++ b/src/java/org/apache/solr/search/LuceneQParserPlugin.java +@@ -61,7 +61,7 @@ class LuceneQParser extends QParser { + + String defaultField = getParam(CommonParams.DF); + if (defaultField==null) { +- defaultField = getReq().getSchema().getDefaultSearchFieldName(); ++ defaultField = getReq().getSchema().getSolrQueryParser(null).getField(); + } + lparser = new SolrQueryParser(this, defaultField); + +@@ -71,7 +71,7 @@ class LuceneQParser extends QParser { + lparser.setDefaultOperator("AND".equals(opParam) ? QueryParser.Operator.AND : QueryParser.Operator.OR); + } else { + // try to get default operator from schema +- String operator = getReq().getSchema().getQueryParserDefaultOperator(); ++ String operator = getReq().getSchema().getSolrQueryParser(null).getField(); + lparser.setDefaultOperator("AND".equals(operator) ? + QueryParser.Operator.AND : QueryParser.Operator.OR); + } +diff --git a/src/java/org/apache/solr/search/LuceneQueryOptimizer.java b/src/java/org/apache/solr/search/LuceneQueryOptimizer.java +index df9020f..758c351 100644 +--- a/src/java/org/apache/solr/search/LuceneQueryOptimizer.java ++++ b/src/java/org/apache/solr/search/LuceneQueryOptimizer.java +@@ -97,7 +97,7 @@ if (c.query instanceof TermQuery) { + filter = (Filter)cache.get(filterQuery); + } + if (filter == null) { // miss +- filter = new QueryFilter(filterQuery); // construct new entry ++ filter = new CachingWrapperFilter(new QueryWrapperFilter(filterQuery)); // construct new entry + synchronized (cache) { + cache.put(filterQuery, filter); // cache it + } +diff --git a/src/java/org/apache/solr/util/HighlightingUtils.java b/src/java/org/apache/solr/util/HighlightingUtils.java +index b2a5aae..04ac761 100644 +--- a/src/java/org/apache/solr/util/HighlightingUtils.java ++++ b/src/java/org/apache/solr/util/HighlightingUtils.java +@@ -325,7 +325,7 @@ class MultiValueTokenStream extends TokenStream { + } + // create an modified token which is the offset into the concatenated + // string of all values +- Token offsetToken = new Token(nextToken.termText(), ++ Token offsetToken = new Token(new String(nextToken.termBuffer(), 0, nextToken.termLength()), + nextToken.startOffset() + curOffset, + nextToken.endOffset() + curOffset); + offsetToken.setPositionIncrement(nextToken.getPositionIncrement() + extra*10); +diff --git a/src/java/org/apache/solr/util/SolrPluginUtils.java b/src/java/org/apache/solr/util/SolrPluginUtils.java +index c0b5b0b..eef0904 100644 +--- a/src/java/org/apache/solr/util/SolrPluginUtils.java ++++ b/src/java/org/apache/solr/util/SolrPluginUtils.java +@@ -180,7 +180,7 @@ public class SolrPluginUtils { + public static int setReturnFields(SolrQueryRequest req, + SolrQueryResponse res) { + +- return setReturnFields(req.getParam(FL), res); ++ return setReturnFields(req.getParams().get(org.apache.solr.common.params.CommonParams.FL), res); + } + + /** +@@ -382,14 +382,14 @@ public class SolrPluginUtils { + DocList results) + throws IOException { + +- String debug = req.getParam(org.apache.solr.common.params.CommonParams.DEBUG_QUERY); ++ String debug = req.getParams().get(org.apache.solr.common.params.CommonParams.DEBUG_QUERY); + + NamedList dbg = null; + if (debug!=null) { + dbg = new SimpleOrderedMap(); + + /* userQuery may have been pre-processes .. expose that */ +- dbg.add("rawquerystring", req.getQueryString()); ++ dbg.add("rawquerystring", req.getParams().get(org.apache.solr.common.params.CommonParams.Q)); + dbg.add("querystring", userQuery); + + /* QueryParsing.toString isn't perfect, use it to see converted +@@ -401,7 +401,7 @@ public class SolrPluginUtils { + + dbg.add("explain", getExplainList + (query, results, req.getSearcher(), req.getSchema())); +- String otherQueryS = req.getParam("explainOther"); ++ String otherQueryS = req.getParams().get(org.apache.solr.common.params.CommonParams.EXPLAIN_OTHER); + if (otherQueryS != null && otherQueryS.length() > 0) { + DocList otherResults = doSimpleQuery + (otherQueryS,req.getSearcher(), req.getSchema(),0,10); +@@ -804,7 +804,7 @@ public class SolrPluginUtils { + */ + public static Sort getSort(SolrQueryRequest req) { + +- String sort = req.getParam(org.apache.solr.common.params.CommonParams.SORT); ++ String sort = req.getParams().get(org.apache.solr.common.params.CommonParams.SORT); + if (null == sort || sort.equals("")) { + return null; + } +diff --git a/src/java/org/apache/solr/util/SuggestMissingFactories.java b/src/java/org/apache/solr/util/SuggestMissingFactories.java +index 9556c1e..b8d0625 100644 +--- a/src/java/org/apache/solr/util/SuggestMissingFactories.java ++++ b/src/java/org/apache/solr/util/SuggestMissingFactories.java +@@ -131,7 +131,7 @@ class FindClasses { + try { + for (int i =0; i < jars.length; i++) { + jarFiles[i] = new JarFile(jars[i]); +- urls[i] = jars[i].toURL(); ++ urls[i] = jars[i].toURI().toURL(); + } + } catch (MalformedURLException e) { + throw new RuntimeException +diff --git a/src/java/org/apache/solr/util/TestHarness.java b/src/java/org/apache/solr/util/TestHarness.java +index 35ed671..7a81034 100644 +--- a/src/java/org/apache/solr/util/TestHarness.java ++++ b/src/java/org/apache/solr/util/TestHarness.java +@@ -17,6 +17,7 @@ + + package org.apache.solr.util; + ++import org.apache.solr.common.params.CommonParams; + import org.apache.solr.common.util.NamedList; + import org.apache.solr.common.util.XML; + import org.apache.solr.core.SolrConfig; +@@ -117,7 +118,7 @@ public class TestHarness { + public TestHarness( String dataDirectory, + SolrConfig solrConfig, + String schemaFile) { +- this( dataDirectory, solrConfig, new IndexSchema(solrConfig, schemaFile)); ++ this( dataDirectory, solrConfig, new IndexSchema(solrConfig, schemaFile, null)); + } + /** + * @param dataDirectory path for index data, will not be cleaned up +@@ -300,7 +301,7 @@ public class TestHarness { + * @see LocalSolrQueryRequest + */ + public String query(SolrQueryRequest req) throws IOException, Exception { +- return query(req.getQueryType(), req); ++ return query(req.getParams().get(CommonParams.QT), req); + } + + /** +diff --git a/src/test/org/apache/solr/BasicFunctionalityTest.java b/src/test/org/apache/solr/BasicFunctionalityTest.java +index cf91a02..ce45cd2 100644 +--- a/src/test/org/apache/solr/BasicFunctionalityTest.java ++++ b/src/test/org/apache/solr/BasicFunctionalityTest.java +@@ -21,6 +21,7 @@ import org.apache.lucene.document.*; + import org.apache.lucene.search.Query; + import org.apache.lucene.search.BooleanQuery; + import org.apache.solr.common.params.AppendedSolrParams; ++import org.apache.solr.common.params.CommonParams; + import org.apache.solr.common.params.DefaultSolrParams; + import org.apache.solr.common.params.MapSolrParams; + import org.apache.solr.common.params.SolrParams; +@@ -297,14 +298,14 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase { + args.put("string", "string value"); + args.put("array", new String[] {"array", "value"}); + SolrQueryRequest req = new LocalSolrQueryRequest(null, null, null, 0, 20, args); +- assertEquals("string value", req.getParam("string")); +- assertEquals("array", req.getParam("array")); ++ assertEquals("string value", req.getParams().get("string")); ++ assertEquals("array", req.getParams().get("array")); + +- String[] stringParams = req.getParams("string"); ++ String[] stringParams = req.getParams().getParams("string"); + assertEquals(1, stringParams.length); + assertEquals("string value", stringParams[0]); + +- String[] arrayParams = req.getParams("array"); ++ String[] arrayParams = req.getParams().getParams("array"); + assertEquals(2, arrayParams.length); + assertEquals("array", arrayParams[0]); + assertEquals("value", arrayParams[1]); +@@ -337,7 +338,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase { + + public void testTermVectorFields() { + +- IndexSchema ischema = new IndexSchema(solrConfig, getSchemaFile()); ++ IndexSchema ischema = new IndexSchema(solrConfig, getSchemaFile(), null); + SchemaField f; // Solr field type + Field luf; // Lucene field + +@@ -506,7 +507,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase { + } + public void testCompressableFieldType() { + +- IndexSchema ischema = new IndexSchema(solrConfig, getSchemaFile()); ++ IndexSchema ischema = new IndexSchema(solrConfig, getSchemaFile(), null); + SchemaField f; // Solr field type + Field luf; // Lucene field + +@@ -538,7 +539,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase { + + SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title,test_hlt"); + SolrQueryResponse rsp = new SolrQueryResponse(); +- core.execute(req, rsp); ++ core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); + + DocList dl = (DocList) rsp.getValues().get("response"); + org.apache.lucene.document.Document d = req.getSearcher().doc(dl.iterator().nextDoc()); +@@ -558,7 +559,7 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase { + + SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title"); + SolrQueryResponse rsp = new SolrQueryResponse(); +- core.execute(req, rsp); ++ core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); + + DocList dl = (DocList) rsp.getValues().get("response"); + DocIterator di = dl.iterator(); +diff --git a/src/test/org/apache/solr/analysis/BaseTokenTestCase.java b/src/test/org/apache/solr/analysis/BaseTokenTestCase.java +index 2487ca3..322b19f 100644 +--- a/src/test/org/apache/solr/analysis/BaseTokenTestCase.java ++++ b/src/test/org/apache/solr/analysis/BaseTokenTestCase.java +@@ -37,10 +37,10 @@ public abstract class BaseTokenTestCase extends AnalysisTestCase + StringBuffer out = new StringBuffer(); + Token t = in.next(); + if (null != t) +- out.append(t.termText()); ++ out.append(new String(t.termBuffer(), 0, t.termLength())); + + for (t = in.next(); null != t; t = in.next()) { +- out.append(" ").append(t.termText()); ++ out.append(" ").append(new String(t.termBuffer(), 0, t.termLength())); + } + in.close(); + return out.toString(); +@@ -49,7 +49,7 @@ public abstract class BaseTokenTestCase extends AnalysisTestCase + public List tok2str(Iterable tokLst) { + ArrayList lst = new ArrayList(); + for ( Token t : tokLst ) { +- lst.add( t.termText()); ++ lst.add( new String(t.termBuffer(), 0, t.termLength())); + } + return lst; + } +@@ -70,7 +70,7 @@ public abstract class BaseTokenTestCase extends AnalysisTestCase + for (Iterator iter = a.iterator(); iter.hasNext();) { + Token tok = (Token)iter.next(); + pos += tok.getPositionIncrement(); +- if (!tokAt(b, tok.termText(), pos ++ if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos + , checkOff ? tok.startOffset() : -1 + , checkOff ? tok.endOffset() : -1 + )) +@@ -85,7 +85,7 @@ public abstract class BaseTokenTestCase extends AnalysisTestCase + for (Iterator iter = lst.iterator(); iter.hasNext();) { + Token tok = (Token)iter.next(); + pos += tok.getPositionIncrement(); +- if (pos==tokPos && tok.termText().equals(val) ++ if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val) + && (startOff==-1 || tok.startOffset()==startOff) + && (endOff ==-1 || tok.endOffset() ==endOff ) + ) +diff --git a/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java b/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java +index 6226a10..b24c2a4 100644 +--- a/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java ++++ b/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java +@@ -33,9 +33,9 @@ public class TestBufferedTokenStream extends BaseTokenTestCase { + public static class AB_Q_Stream extends BufferedTokenStream { + public AB_Q_Stream(TokenStream input) {super(input);} + protected Token process(Token t) throws IOException { +- if ("A".equals(t.termText())) { ++ if ("A".equals(new String(t.termBuffer(), 0, t.termLength()))) { + Token t2 = read(); +- if (t2!=null && "B".equals(t2.termText())) t.setTermText("Q"); ++ if (t2!=null && "B".equals(new String(t2.termBuffer(), 0, t2.termLength()))) t.setTermText("Q"); + if (t2!=null) pushBack(t2); + } + return t; +@@ -46,7 +46,8 @@ public class TestBufferedTokenStream extends BaseTokenTestCase { + public static class AB_AAB_Stream extends BufferedTokenStream { + public AB_AAB_Stream(TokenStream input) {super(input);} + protected Token process(Token t) throws IOException { +- if ("A".equals(t.termText()) && "B".equals(peek(1).termText())) ++ if ("A".equals(new String(t.termBuffer(), 0, t.termLength())) && ++ "B".equals(new String(peek(1).termBuffer(), 0, peek(1).termLength()))) + write(t); + return t; + } +diff --git a/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java b/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java +index 4ddeaf3..4727a6a 100644 +--- a/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java ++++ b/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java +@@ -65,7 +65,7 @@ public class TestPatternTokenizerFactory extends AnalysisTestCase + int i=0; + for( Token t = stream.next(); null != t; t = stream.next() ) + { +- assertEquals( "split: "+test[1] + " "+i, split[i++], t.termText() ); ++ assertEquals( "split: "+test[1] + " "+i, split[i++], new String(t.termBuffer(), 0, t.termLength()) ); + } + } + } +diff --git a/src/test/org/apache/solr/analysis/TestSynonymMap.java b/src/test/org/apache/solr/analysis/TestSynonymMap.java +index fb627db..e0bd7c8 100644 +--- a/src/test/org/apache/solr/analysis/TestSynonymMap.java ++++ b/src/test/org/apache/solr/analysis/TestSynonymMap.java +@@ -259,7 +259,7 @@ public class TestSynonymMap extends AnalysisTestCase { + Token[] tokens = ((SynonymMap)map.submap.get( src )).synonyms; + boolean inc = false; + for( Token token : tokens ){ +- if( exp.equals( token.termText() ) ) ++ if( exp.equals( new String(token.termBuffer(), 0, token.termLength()) ) ) + inc = true; + } + assertTrue( inc ); +diff --git a/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java b/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java +index 0b34fdc..1b8084e 100644 +--- a/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java ++++ b/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java +@@ -147,17 +147,18 @@ public class TestWordDelimiterFilter extends AbstractSolrTestCase { + + int i=0; + for(Token t; (t=wdf.next())!=null;) { +- if (t.termText().equals("foo")) { ++ String termText = new String(t.termBuffer(), 0, t.termLength()); ++ if (termText.equals("foo")) { + assertEquals(5, t.startOffset()); + assertEquals(8, t.endOffset()); + i++; + } +- if (t.termText().equals("bar")) { ++ if (termText.equals("bar")) { + assertEquals(9, t.startOffset()); + assertEquals(12, t.endOffset()); + i++; + } +- if (t.termText().equals("foobar")) { ++ if (termText.equals("foobar")) { + assertEquals(5, t.startOffset()); + assertEquals(12, t.endOffset()); + i++; +diff --git a/src/test/org/apache/solr/highlight/HighlighterConfigTest.java b/src/test/org/apache/solr/highlight/HighlighterConfigTest.java +index a0e58db..e38ff63 100644 +--- a/src/test/org/apache/solr/highlight/HighlighterConfigTest.java ++++ b/src/test/org/apache/solr/highlight/HighlighterConfigTest.java +@@ -35,7 +35,7 @@ public class HighlighterConfigTest extends AbstractSolrTestCase { + + public void testConfig() + { +- SolrHighlighter highlighter = SolrCore.getSolrCore().getHighlighter(); ++ SolrHighlighter highlighter = h.getCore().getHighlighter(); + System.out.println( "highlighter" ); + + assertTrue( highlighter instanceof DummyHighlighter ); +diff --git a/src/test/org/apache/solr/highlight/HighlighterTest.java b/src/test/org/apache/solr/highlight/HighlighterTest.java +index fab092a..2ebeb9f 100755 +--- a/src/test/org/apache/solr/highlight/HighlighterTest.java ++++ b/src/test/org/apache/solr/highlight/HighlighterTest.java +@@ -54,7 +54,7 @@ public class HighlighterTest extends AbstractSolrTestCase { + + public void testConfig() + { +- SolrHighlighter highlighter = SolrCore.getSolrCore().getHighlighter(); ++ SolrHighlighter highlighter = h.getCore().getHighlighter(); + System.out.println( "highlighter" ); + + // Make sure we loaded the one formatter +diff --git a/src/test/org/apache/solr/schema/DateFieldTest.java b/src/test/org/apache/solr/schema/DateFieldTest.java +index 785db52..d672023 100644 +--- a/src/test/org/apache/solr/schema/DateFieldTest.java ++++ b/src/test/org/apache/solr/schema/DateFieldTest.java +@@ -99,11 +99,10 @@ public class DateFieldTest extends LegacyDateFieldTest { + } + + public void testFormatter() { +- DateFormat fmt = f.getThreadLocalDateFormat(); +- assertEquals("1970-01-01T00:00:00.005", fmt.format(new Date(5))); +- assertEquals("1970-01-01T00:00:00", fmt.format(new Date(0))); +- assertEquals("1970-01-01T00:00:00.37", fmt.format(new Date(370))); +- assertEquals("1970-01-01T00:00:00.9", fmt.format(new Date(900))); ++ assertEquals("1970-01-01T00:00:00.005", f.formatDate(new Date(5))); ++ assertEquals("1970-01-01T00:00:00", f.formatDate(new Date(0))); ++ assertEquals("1970-01-01T00:00:00.37", f.formatDate(new Date(370))); ++ assertEquals("1970-01-01T00:00:00.9", f.formatDate(new Date(900))); + + } + +diff --git a/src/test/org/apache/solr/schema/LegacyDateFieldTest.java b/src/test/org/apache/solr/schema/LegacyDateFieldTest.java +index 201db56..4d3e6bb 100644 +--- a/src/test/org/apache/solr/schema/LegacyDateFieldTest.java ++++ b/src/test/org/apache/solr/schema/LegacyDateFieldTest.java +@@ -95,11 +95,10 @@ public class LegacyDateFieldTest extends TestCase { + assertItoR("1995-12-31T23:59:59Z", "1995-12-31T23:59:59"); + } + public void testFormatter() { +- DateFormat fmt = f.getThreadLocalDateFormat(); +- assertEquals("1970-01-01T00:00:00.005", fmt.format(new Date(5))); ++ assertEquals("1970-01-01T00:00:00.005", f.formatDate(new Date(5))); + // all of this is broken behavior +- assertEquals("1970-01-01T00:00:00.000", fmt.format(new Date(0))); +- assertEquals("1970-01-01T00:00:00.370", fmt.format(new Date(370))); +- assertEquals("1970-01-01T00:00:00.900", fmt.format(new Date(900))); ++ assertEquals("1970-01-01T00:00:00.000", f.formatDate(new Date(0))); ++ assertEquals("1970-01-01T00:00:00.370", f.formatDate(new Date(370))); ++ assertEquals("1970-01-01T00:00:00.900", f.formatDate(new Date(900))); + } + } +diff --git a/src/test/org/apache/solr/servlet/SolrRequestParserTest.java b/src/test/org/apache/solr/servlet/SolrRequestParserTest.java +index f2f2ac7..ad9bc47 100644 +--- a/src/test/org/apache/solr/servlet/SolrRequestParserTest.java ++++ b/src/test/org/apache/solr/servlet/SolrRequestParserTest.java +@@ -56,7 +56,7 @@ public class SolrRequestParserTest extends AbstractSolrTestCase { + String body2 = "qwertasdfgzxcvb"; + String body3 = "1234567890"; + +- SolrCore core = SolrCore.getSolrCore(); ++ SolrCore core = h.getCore(); + + Map args = new HashMap(); + args.put( CommonParams.STREAM_BODY, new String[] {body1} ); +@@ -110,7 +110,7 @@ public class SolrRequestParserTest extends AbstractSolrTestCase { + return; + } + +- SolrCore core = SolrCore.getSolrCore(); ++ SolrCore core = h.getCore(); + + Map args = new HashMap(); + args.put( CommonParams.STREAM_URL, new String[] {url} ); +diff --git a/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java b/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java +index 93d2921..eeee6a7 100644 +--- a/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java ++++ b/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java +@@ -37,7 +37,7 @@ public class DirectUpdateHandlerTest extends AbstractSolrTestCase { + + public void testRequireUniqueKey() throws Exception + { +- SolrCore core = SolrCore.getSolrCore(); ++ SolrCore core = h.getCore(); + + UpdateHandler updater = core.getUpdateHandler(); + +diff --git a/src/test/org/apache/solr/update/DocumentBuilderTest.java b/src/test/org/apache/solr/update/DocumentBuilderTest.java +index 0174501..45f14a3 100644 +--- a/src/test/org/apache/solr/update/DocumentBuilderTest.java ++++ b/src/test/org/apache/solr/update/DocumentBuilderTest.java +@@ -51,7 +51,7 @@ public class DocumentBuilderTest extends AbstractSolrTestCase { + + public void testNullField() + { +- SolrCore core = SolrCore.getSolrCore(); ++ SolrCore core = h.getCore(); + + // make sure a null value is not indexed + SolrInputDocument doc = new SolrInputDocument(); +diff --git a/src/test/org/apache/solr/util/SolrPluginUtilsTest.java b/src/test/org/apache/solr/util/SolrPluginUtilsTest.java +index 4bbc2fd..0098635 100644 +--- a/src/test/org/apache/solr/util/SolrPluginUtilsTest.java ++++ b/src/test/org/apache/solr/util/SolrPluginUtilsTest.java +@@ -133,7 +133,7 @@ public class SolrPluginUtilsTest extends AbstractSolrTestCase { + assertTrue(t+" sanity test isn't TermQuery: " + out.getClass(), + out instanceof TermQuery); + assertEquals(t+" sanity test is wrong field", +- h.getCore().getSchema().getDefaultSearchFieldName(), ++ h.getCore().getSchema().getSolrQueryParser(null).getField(), + ((TermQuery)out).getTerm().field()); + + t = "subject:XXXXXXXX"; diff --git a/SOLR-793-commitWithin.patch b/SOLR-793-commitWithin.patch new file mode 100644 index 00000000000..3ea271e6381 --- /dev/null +++ b/SOLR-793-commitWithin.patch @@ -0,0 +1,221 @@ +Index: src/java/org/apache/solr/update/AddUpdateCommand.java +=================================================================== +--- src/java/org/apache/solr/update/AddUpdateCommand.java (revision 701732) ++++ src/java/org/apache/solr/update/AddUpdateCommand.java (working copy) +@@ -42,6 +42,10 @@ + public boolean allowDups; + public boolean overwritePending; + public boolean overwriteCommitted; ++ ++ ++ public int commitWithin = -1; ++ + + /** Reset state to reuse this object with a different document in the same request */ + public void clear() { +Index: src/java/org/apache/solr/update/DirectUpdateHandler2.java +=================================================================== +--- src/java/org/apache/solr/update/DirectUpdateHandler2.java (revision 701732) ++++ src/java/org/apache/solr/update/DirectUpdateHandler2.java (working copy) +@@ -198,7 +198,7 @@ + synchronized (this) { + // adding document -- prep writer + openWriter(); +- tracker.addedDocument(); ++ tracker.addedDocument( cmd.commitWithin ); + } // end synchronized block + + // this is the only unsynchronized code in the iwAccess block, which +@@ -424,7 +424,7 @@ + SolrCore.log.info("AutoCommit: " + this); + } + +- /** schedeule individual commits */ ++ /** schedule individual commits */ + public synchronized void scheduleCommitWithin(long commitMaxTime) + { + // Check if there is a commit already scheduled for longer then this time +@@ -443,30 +443,22 @@ + + /** Indicate that documents have been added + */ +- public void addedDocument() { ++ public void addedDocument( int commitWithin ) { + docsSinceCommit++; + lastAddedTime = System.currentTimeMillis(); + // maxDocs-triggered autoCommit + if( docsUpperBound > 0 && (docsSinceCommit > docsUpperBound) ) { +- if (pending != null && +- pending.getDelay(TimeUnit.MILLISECONDS) > DOC_COMMIT_DELAY_MS) { +- // another commit is pending, but too far away (probably due to +- // maxTime) +- pending.cancel(false); +- pending = null; +- } +- if (pending == null) { +- // 1/4 second seems fast enough for anyone using maxDocs +- pending = scheduler.schedule(this, DOC_COMMIT_DELAY_MS, +- TimeUnit.MILLISECONDS); +- } ++ scheduleCommitWithin( DOC_COMMIT_DELAY_MS ); + } ++ + // maxTime-triggered autoCommit +- if( pending == null && timeUpperBound > 0 ) { +- // Don't start a new event if one is already waiting +- pending = scheduler.schedule( this, timeUpperBound, TimeUnit.MILLISECONDS ); ++ long ctime = timeUpperBound; ++ if( commitWithin > 0 && (ctime < 0 || commitWithin < ctime) ) { ++ ctime = commitWithin; + } +- ++ if( ctime > 0 ) { ++ scheduleCommitWithin( ctime ); ++ } + } + + /** Inform tracker that a commit has occurred, cancel any pending commits */ +Index: src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java +=================================================================== +--- src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (revision 701732) ++++ src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (working copy) +@@ -19,6 +19,7 @@ + + import java.io.IOException; + import java.io.Reader; ++import java.io.StringReader; + import java.io.Writer; + import java.io.File; + import java.util.HashMap; +@@ -72,6 +73,7 @@ + public static final String WAIT_FLUSH = "waitFlush"; + + public static final String OVERWRITE = "overwrite"; ++ public static final String COMMIT_WITHIN = "commitWithin"; + public static final String OVERWRITE_COMMITTED = "overwriteCommitted"; // @Deprecated + public static final String OVERWRITE_PENDING = "overwritePending"; // @Deprecated + public static final String ALLOW_DUPS = "allowDups"; +@@ -120,6 +122,12 @@ + for( ContentStream stream : req.getContentStreams() ) { + Reader reader = stream.getReader(); + try { ++ if( log.isTraceEnabled() ) { ++ String body = IOUtils.toString( reader ); ++ log.trace( "body", body ); ++ reader = new StringReader( body ); ++ } ++ + XMLStreamReader parser = inputFactory.createXMLStreamReader(reader); + this.processUpdate( processor, parser ); + } +@@ -169,6 +177,8 @@ + overwrite = StrUtils.parseBoolean(attrVal); + } else if (ALLOW_DUPS.equals(attrName)) { + overwrite = !StrUtils.parseBoolean(attrVal); ++ } else if ( COMMIT_WITHIN.equals(attrName) ) { ++ addCmd.commitWithin = Integer.parseInt( attrVal ); + } else if ( OVERWRITE_PENDING.equals(attrName) ) { + overwritePending = StrUtils.parseBoolean(attrVal); + } else if ( OVERWRITE_COMMITTED.equals(attrName) ) { +Index: client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java +=================================================================== +--- client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java (revision 701732) ++++ client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java (working copy) +@@ -29,6 +29,7 @@ + import org.apache.solr.client.solrj.request.DirectXmlRequest; + import org.apache.solr.client.solrj.request.LukeRequest; + import org.apache.solr.client.solrj.request.SolrPing; ++import org.apache.solr.client.solrj.request.UpdateRequest; + import org.apache.solr.client.solrj.response.LukeResponse; + import org.apache.solr.client.solrj.response.QueryResponse; + import org.apache.solr.client.solrj.response.FacetField; +@@ -149,8 +150,8 @@ + Assert.assertEquals(2, response.getResults().getNumFound() ); + Assert.assertFalse(query.getFilterQueries() == query2.getFilterQueries()); + } +- + ++ + /** + * query the example + */ +@@ -195,8 +196,40 @@ + rsp = server.query( query ); + Assert.assertEquals( 2, rsp.getResults().getNumFound() ); + System.out.println( rsp.getResults() ); ++ + } + ++ /** ++ * query the example ++ */ ++ public void testCommitWithin() throws Exception ++ { ++ // make sure it is empty... ++ SolrServer server = getSolrServer(); ++ QueryResponse rsp = server.query( new SolrQuery( "*:*") ); ++ Assert.assertEquals( 0, rsp.getResults().getNumFound() ); ++ ++ // Now try a timed commit... ++ SolrInputDocument doc3 = new SolrInputDocument(); ++ doc3.addField( "id", "id3", 1.0f ); ++ doc3.addField( "name", "doc3", 1.0f ); ++ doc3.addField( "price", 10 ); ++ UpdateRequest up = new UpdateRequest(); ++ up.add( doc3 ); ++ up.setCommitWithin( 5 ); ++ up.process( server ); ++ ++ rsp = server.query( new SolrQuery( "*:*") ); ++ Assert.assertEquals( 0, rsp.getResults().getNumFound() ); ++ ++ Thread.sleep( 500 ); // wait 1/2 seconds... ++ ++ // now check that it comes out... ++ rsp = server.query( new SolrQuery( "id:id3") ); ++ Assert.assertEquals( 1, rsp.getResults().getNumFound() ); ++ } ++ ++ + protected void assertNumFound( String query, int num ) throws SolrServerException, IOException + { + QueryResponse rsp = getSolrServer().query( new SolrQuery( query ) ); +Index: client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java +=================================================================== +--- client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java (revision 701732) ++++ client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java (working copy) +@@ -51,6 +51,7 @@ + private List deleteQuery = null; + + private ModifiableSolrParams params; ++ private int commitWithin = -1; + + public UpdateRequest() + { +@@ -163,7 +164,12 @@ + public String getXML() throws IOException { + StringWriter writer = new StringWriter(); + if( documents != null && documents.size() > 0 ) { +- writer.write(""); ++ if( commitWithin > 0 ) { ++ writer.write(""); ++ } ++ else { ++ writer.write(""); ++ } + for (SolrInputDocument doc : documents ) { + if( doc != null ) { + ClientUtils.writeXML( doc, writer ); +@@ -245,4 +251,12 @@ + public void setWaitSearcher(boolean waitSearcher) { + setParam( UpdateParams.WAIT_SEARCHER, waitSearcher+"" ); + } ++ ++ public int getCommitWithin() { ++ return commitWithin; ++ } ++ ++ public void setCommitWithin(int commitWithin) { ++ this.commitWithin = commitWithin; ++ } + } diff --git a/SOLR-793-commitWithin2.patch b/SOLR-793-commitWithin2.patch new file mode 100644 index 00000000000..3ea271e6381 --- /dev/null +++ b/SOLR-793-commitWithin2.patch @@ -0,0 +1,221 @@ +Index: src/java/org/apache/solr/update/AddUpdateCommand.java +=================================================================== +--- src/java/org/apache/solr/update/AddUpdateCommand.java (revision 701732) ++++ src/java/org/apache/solr/update/AddUpdateCommand.java (working copy) +@@ -42,6 +42,10 @@ + public boolean allowDups; + public boolean overwritePending; + public boolean overwriteCommitted; ++ ++ ++ public int commitWithin = -1; ++ + + /** Reset state to reuse this object with a different document in the same request */ + public void clear() { +Index: src/java/org/apache/solr/update/DirectUpdateHandler2.java +=================================================================== +--- src/java/org/apache/solr/update/DirectUpdateHandler2.java (revision 701732) ++++ src/java/org/apache/solr/update/DirectUpdateHandler2.java (working copy) +@@ -198,7 +198,7 @@ + synchronized (this) { + // adding document -- prep writer + openWriter(); +- tracker.addedDocument(); ++ tracker.addedDocument( cmd.commitWithin ); + } // end synchronized block + + // this is the only unsynchronized code in the iwAccess block, which +@@ -424,7 +424,7 @@ + SolrCore.log.info("AutoCommit: " + this); + } + +- /** schedeule individual commits */ ++ /** schedule individual commits */ + public synchronized void scheduleCommitWithin(long commitMaxTime) + { + // Check if there is a commit already scheduled for longer then this time +@@ -443,30 +443,22 @@ + + /** Indicate that documents have been added + */ +- public void addedDocument() { ++ public void addedDocument( int commitWithin ) { + docsSinceCommit++; + lastAddedTime = System.currentTimeMillis(); + // maxDocs-triggered autoCommit + if( docsUpperBound > 0 && (docsSinceCommit > docsUpperBound) ) { +- if (pending != null && +- pending.getDelay(TimeUnit.MILLISECONDS) > DOC_COMMIT_DELAY_MS) { +- // another commit is pending, but too far away (probably due to +- // maxTime) +- pending.cancel(false); +- pending = null; +- } +- if (pending == null) { +- // 1/4 second seems fast enough for anyone using maxDocs +- pending = scheduler.schedule(this, DOC_COMMIT_DELAY_MS, +- TimeUnit.MILLISECONDS); +- } ++ scheduleCommitWithin( DOC_COMMIT_DELAY_MS ); + } ++ + // maxTime-triggered autoCommit +- if( pending == null && timeUpperBound > 0 ) { +- // Don't start a new event if one is already waiting +- pending = scheduler.schedule( this, timeUpperBound, TimeUnit.MILLISECONDS ); ++ long ctime = timeUpperBound; ++ if( commitWithin > 0 && (ctime < 0 || commitWithin < ctime) ) { ++ ctime = commitWithin; + } +- ++ if( ctime > 0 ) { ++ scheduleCommitWithin( ctime ); ++ } + } + + /** Inform tracker that a commit has occurred, cancel any pending commits */ +Index: src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java +=================================================================== +--- src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (revision 701732) ++++ src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (working copy) +@@ -19,6 +19,7 @@ + + import java.io.IOException; + import java.io.Reader; ++import java.io.StringReader; + import java.io.Writer; + import java.io.File; + import java.util.HashMap; +@@ -72,6 +73,7 @@ + public static final String WAIT_FLUSH = "waitFlush"; + + public static final String OVERWRITE = "overwrite"; ++ public static final String COMMIT_WITHIN = "commitWithin"; + public static final String OVERWRITE_COMMITTED = "overwriteCommitted"; // @Deprecated + public static final String OVERWRITE_PENDING = "overwritePending"; // @Deprecated + public static final String ALLOW_DUPS = "allowDups"; +@@ -120,6 +122,12 @@ + for( ContentStream stream : req.getContentStreams() ) { + Reader reader = stream.getReader(); + try { ++ if( log.isTraceEnabled() ) { ++ String body = IOUtils.toString( reader ); ++ log.trace( "body", body ); ++ reader = new StringReader( body ); ++ } ++ + XMLStreamReader parser = inputFactory.createXMLStreamReader(reader); + this.processUpdate( processor, parser ); + } +@@ -169,6 +177,8 @@ + overwrite = StrUtils.parseBoolean(attrVal); + } else if (ALLOW_DUPS.equals(attrName)) { + overwrite = !StrUtils.parseBoolean(attrVal); ++ } else if ( COMMIT_WITHIN.equals(attrName) ) { ++ addCmd.commitWithin = Integer.parseInt( attrVal ); + } else if ( OVERWRITE_PENDING.equals(attrName) ) { + overwritePending = StrUtils.parseBoolean(attrVal); + } else if ( OVERWRITE_COMMITTED.equals(attrName) ) { +Index: client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java +=================================================================== +--- client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java (revision 701732) ++++ client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java (working copy) +@@ -29,6 +29,7 @@ + import org.apache.solr.client.solrj.request.DirectXmlRequest; + import org.apache.solr.client.solrj.request.LukeRequest; + import org.apache.solr.client.solrj.request.SolrPing; ++import org.apache.solr.client.solrj.request.UpdateRequest; + import org.apache.solr.client.solrj.response.LukeResponse; + import org.apache.solr.client.solrj.response.QueryResponse; + import org.apache.solr.client.solrj.response.FacetField; +@@ -149,8 +150,8 @@ + Assert.assertEquals(2, response.getResults().getNumFound() ); + Assert.assertFalse(query.getFilterQueries() == query2.getFilterQueries()); + } +- + ++ + /** + * query the example + */ +@@ -195,8 +196,40 @@ + rsp = server.query( query ); + Assert.assertEquals( 2, rsp.getResults().getNumFound() ); + System.out.println( rsp.getResults() ); ++ + } + ++ /** ++ * query the example ++ */ ++ public void testCommitWithin() throws Exception ++ { ++ // make sure it is empty... ++ SolrServer server = getSolrServer(); ++ QueryResponse rsp = server.query( new SolrQuery( "*:*") ); ++ Assert.assertEquals( 0, rsp.getResults().getNumFound() ); ++ ++ // Now try a timed commit... ++ SolrInputDocument doc3 = new SolrInputDocument(); ++ doc3.addField( "id", "id3", 1.0f ); ++ doc3.addField( "name", "doc3", 1.0f ); ++ doc3.addField( "price", 10 ); ++ UpdateRequest up = new UpdateRequest(); ++ up.add( doc3 ); ++ up.setCommitWithin( 5 ); ++ up.process( server ); ++ ++ rsp = server.query( new SolrQuery( "*:*") ); ++ Assert.assertEquals( 0, rsp.getResults().getNumFound() ); ++ ++ Thread.sleep( 500 ); // wait 1/2 seconds... ++ ++ // now check that it comes out... ++ rsp = server.query( new SolrQuery( "id:id3") ); ++ Assert.assertEquals( 1, rsp.getResults().getNumFound() ); ++ } ++ ++ + protected void assertNumFound( String query, int num ) throws SolrServerException, IOException + { + QueryResponse rsp = getSolrServer().query( new SolrQuery( query ) ); +Index: client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java +=================================================================== +--- client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java (revision 701732) ++++ client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java (working copy) +@@ -51,6 +51,7 @@ + private List deleteQuery = null; + + private ModifiableSolrParams params; ++ private int commitWithin = -1; + + public UpdateRequest() + { +@@ -163,7 +164,12 @@ + public String getXML() throws IOException { + StringWriter writer = new StringWriter(); + if( documents != null && documents.size() > 0 ) { +- writer.write(""); ++ if( commitWithin > 0 ) { ++ writer.write(""); ++ } ++ else { ++ writer.write(""); ++ } + for (SolrInputDocument doc : documents ) { + if( doc != null ) { + ClientUtils.writeXML( doc, writer ); +@@ -245,4 +251,12 @@ + public void setWaitSearcher(boolean waitSearcher) { + setParam( UpdateParams.WAIT_SEARCHER, waitSearcher+"" ); + } ++ ++ public int getCommitWithin() { ++ return commitWithin; ++ } ++ ++ public void setCommitWithin(int commitWithin) { ++ this.commitWithin = commitWithin; ++ } + } diff --git a/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java b/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java index 7c9911d1d9c..71432fb01d7 100644 --- a/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java +++ b/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java @@ -51,6 +51,7 @@ public class UpdateRequest extends SolrRequest private List deleteQuery = null; private ModifiableSolrParams params; + private int commitWithin = -1; public UpdateRequest() { @@ -163,7 +164,12 @@ public class UpdateRequest extends SolrRequest public String getXML() throws IOException { StringWriter writer = new StringWriter(); if( documents != null && documents.size() > 0 ) { - writer.write(""); + if( commitWithin > 0 ) { + writer.write(""); + } + else { + writer.write(""); + } for (SolrInputDocument doc : documents ) { if( doc != null ) { ClientUtils.writeXML( doc, writer ); @@ -245,4 +251,12 @@ public class UpdateRequest extends SolrRequest public void setWaitSearcher(boolean waitSearcher) { setParam( UpdateParams.WAIT_SEARCHER, waitSearcher+"" ); } + + public int getCommitWithin() { + return commitWithin; + } + + public void setCommitWithin(int commitWithin) { + this.commitWithin = commitWithin; + } } diff --git a/client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java b/client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java index 87b5748c048..3bec75224d1 100644 --- a/client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/client/java/solrj/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -29,6 +29,7 @@ import junit.framework.Assert; import org.apache.solr.client.solrj.request.DirectXmlRequest; import org.apache.solr.client.solrj.request.LukeRequest; import org.apache.solr.client.solrj.request.SolrPing; +import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.LukeResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.FacetField; @@ -149,7 +150,7 @@ abstract public class SolrExampleTests extends SolrExampleTestBase Assert.assertEquals(2, response.getResults().getNumFound() ); Assert.assertFalse(query.getFilterQueries() == query2.getFilterQueries()); } - + /** * query the example @@ -195,8 +196,42 @@ abstract public class SolrExampleTests extends SolrExampleTestBase rsp = server.query( query ); Assert.assertEquals( 2, rsp.getResults().getNumFound() ); System.out.println( rsp.getResults() ); + } + /** + * query the example + */ + public void testCommitWithin() throws Exception + { + // make sure it is empty... + SolrServer server = getSolrServer(); + server.deleteByQuery( "*:*" );// delete everything! + server.commit(); + QueryResponse rsp = server.query( new SolrQuery( "*:*") ); + Assert.assertEquals( 0, rsp.getResults().getNumFound() ); + + // Now try a timed commit... + SolrInputDocument doc3 = new SolrInputDocument(); + doc3.addField( "id", "id3", 1.0f ); + doc3.addField( "name", "doc3", 1.0f ); + doc3.addField( "price", 10 ); + UpdateRequest up = new UpdateRequest(); + up.add( doc3 ); + up.setCommitWithin( 10 ); + up.process( server ); + + rsp = server.query( new SolrQuery( "*:*") ); + Assert.assertEquals( 0, rsp.getResults().getNumFound() ); + + Thread.sleep( 500 ); // wait 1/2 seconds... + + // now check that it comes out... + rsp = server.query( new SolrQuery( "id:id3") ); + Assert.assertEquals( 1, rsp.getResults().getNumFound() ); + } + + protected void assertNumFound( String query, int num ) throws SolrServerException, IOException { QueryResponse rsp = getSolrServer().query( new SolrQuery( query ) ); diff --git a/example/solr/conf/velocity/default.vm b/example/solr/conf/velocity/default.vm new file mode 100644 index 00000000000..f48fb3823f8 --- /dev/null +++ b/example/solr/conf/velocity/default.vm @@ -0,0 +1,40 @@ +#set($iterator=$rawResponse.getResultIterator()) +#set($request=$rawResponse.request) +#set($core=$request.core) +#set($schema=$core.schema) + + + Default VelocityResponseWriter Response + + +

+ A default response using only the raw solr response. The "rawResponse" object is a helper that provides access to the + SolrQueryRequest, SolrQueryResponse, as well as convenience methods like getResultIterator() and getRequestParam() +

+

+ If you want to use an instance of SolrResponse of solrj inside a template, use eg. vl.response=QueryResponse. + This object then is avalaible as "response" in the context. +

+
+ + #foreach($document in $iterator) + #foreach($field in $document.getFieldNames()) +

$field: $document.getFieldValue($field)

+ #end +
+ #end + + 
+      header = $rawResponse.$response.responseHeader
+
+      values = $rawResponse.$response.values
+
+      request = $request
+
+      core = $core
+
+      schema = $schema
+
+ + + diff --git a/lib/commons-collections-3.2.jar b/lib/commons-collections-3.2.jar new file mode 100644 index 00000000000..8071d63ea7a --- /dev/null +++ b/lib/commons-collections-3.2.jar @@ -0,0 +1,2 @@ +AnyObjectId[75580be255065727b20b41c2d338b14792bb35cd] was removed in git history. +Apache SVN contains full history. \ No newline at end of file diff --git a/lib/commons-lang-2.4.jar b/lib/commons-lang-2.4.jar new file mode 100644 index 00000000000..63550c90560 --- /dev/null +++ b/lib/commons-lang-2.4.jar @@ -0,0 +1,2 @@ +AnyObjectId[ce0ca22c8d29a9be736d775fe50bfdc6ce770186] was removed in git history. +Apache SVN contains full history. \ No newline at end of file diff --git a/lib/velocity-1.6-dev.jar b/lib/velocity-1.6-dev.jar new file mode 100644 index 00000000000..0db2980363f --- /dev/null +++ b/lib/velocity-1.6-dev.jar @@ -0,0 +1,2 @@ +AnyObjectId[1454e77a8a5b60bb95b316d5e991783f940c4d56] was removed in git history. +Apache SVN contains full history. \ No newline at end of file diff --git a/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java b/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java index fb4d03f1e49..838852d2427 100644 --- a/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java +++ b/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java @@ -19,6 +19,7 @@ package org.apache.solr.handler; import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import java.io.Writer; import java.io.File; import java.util.HashMap; @@ -72,6 +73,7 @@ public class XmlUpdateRequestHandler extends RequestHandlerBase public static final String WAIT_FLUSH = "waitFlush"; public static final String OVERWRITE = "overwrite"; + public static final String COMMIT_WITHIN = "commitWithin"; public static final String OVERWRITE_COMMITTED = "overwriteCommitted"; // @Deprecated public static final String OVERWRITE_PENDING = "overwritePending"; // @Deprecated public static final String ALLOW_DUPS = "allowDups"; @@ -120,6 +122,12 @@ public class XmlUpdateRequestHandler extends RequestHandlerBase for( ContentStream stream : req.getContentStreams() ) { Reader reader = stream.getReader(); try { + if( log.isTraceEnabled() ) { + String body = IOUtils.toString( reader ); + log.trace( "body", body ); + reader = new StringReader( body ); + } + XMLStreamReader parser = inputFactory.createXMLStreamReader(reader); this.processUpdate( processor, parser ); } @@ -169,6 +177,8 @@ public class XmlUpdateRequestHandler extends RequestHandlerBase overwrite = StrUtils.parseBoolean(attrVal); } else if (ALLOW_DUPS.equals(attrName)) { overwrite = !StrUtils.parseBoolean(attrVal); + } else if ( COMMIT_WITHIN.equals(attrName) ) { + addCmd.commitWithin = Integer.parseInt( attrVal ); } else if ( OVERWRITE_PENDING.equals(attrName) ) { overwritePending = StrUtils.parseBoolean(attrVal); } else if ( OVERWRITE_COMMITTED.equals(attrName) ) { diff --git a/src/java/org/apache/solr/request/VelocityResponseWriter.java b/src/java/org/apache/solr/request/VelocityResponseWriter.java new file mode 100644 index 00000000000..6f7dab1043b --- /dev/null +++ b/src/java/org/apache/solr/request/VelocityResponseWriter.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.request; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Iterator; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.search.Searcher; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.search.DocSlice; +import org.apache.solr.update.DocumentBuilder; +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; + +/** + *

A response writer that uses velocity template for response creation. + * Possible request parameters:

+ *
    + *
  • + * vl.template: + * The name of the template file without .vm suffix.
    • + *
  • + * vl.json: + * A name of a Javascript method. If set, the response is wrapped into this object. + * useful for JSON requests. + *
    • + *
  • + * vl.content: + * Specify a custom content type for the response. Default is "text/html" for standard requests, + * and "text/x-json" for JSON requests. + *
    • + *
  • + * vl.response: + * To provide an implementation of {@link SolrResponse} inside the template, + * specify the class name of the implementation. For convenience, it looks + * inside the package "org.apache.solr.client.solrj.response", so you only + * have to choose eg. QueryResponse, LikeResponse, MultiCoreResponse. + * Custom classes may be accessed using the full qualified class name, + * eg. my.custom.package.CustomResponse + *
    • + *
+ */ +public class VelocityResponseWriter implements QueryResponseWriter { + + private static final String PARAMETER_TEMPLATE="vl.template"; + private static final String PARAMETER_JSON="vl.json"; + private static final String PARAMETER_RESPONSE="vl.response"; + private static final String PARAMETER_CONTENT_TYPE="vl.content"; + + public void write(Writer writer, SolrQueryRequest request, + SolrQueryResponse response) throws IOException { + + // init velocity and get template + VelocityEngine engine = new VelocityEngine(); + File baseDir = new File(request.getCore().getResourceLoader().getConfigDir(), "velocity"); + engine.setProperty(VelocityEngine.FILE_RESOURCE_LOADER_PATH, baseDir.getAbsolutePath()); + engine.setProperty(VelocityEngine.RESOURCE_LOADER, "file"); + Template template; + try { + template = engine.getTemplate(request.getParams().get(PARAMETER_TEMPLATE, "default") + ".vm"); + } catch (Exception e) { + throw new IOException(e.getMessage()); + } + + // put raw response into context + VelocityContext context = new VelocityContext(); + context.put("rawResponse", new RawResponseHelper(request, response)); + + // convert response if a class is specified + if (request.getParams().get(PARAMETER_RESPONSE) != null) { + String className = request.getParams().get(PARAMETER_RESPONSE); + + // create SolrResponse using reflection + SolrResponse solrResponse; + Object object; + try { + object = request.getCore().getResourceLoader().newInstance(className, "client.solrj.response."); + } catch (RuntimeException e) { + throw new IOException("Unable to resolve response class \"" + className + "\": " + e.getMessage()); + } + if (!(object instanceof SolrResponse)) { + throw new IOException("Class \"" + className + "\" doesn't implement SolrResponse!"); + } + solrResponse = (SolrResponse) object; + + // inject the request into the response + solrResponse.setResponse(new EmbeddedSolrServer(request.getCore()).getParsedResponse(request, response)); + + // put it into the context + context.put("response", solrResponse); + } + + // create output, optionally wrap it into a json object + if (isWrappedResponse(request)) { + StringWriter stringWriter = new StringWriter(); + template.merge(context, stringWriter); + writer.write(request.getParams().get(PARAMETER_JSON) + "("); + writer.write(getJSONWrap(stringWriter.toString())); + writer.write(')'); + } else { + template.merge(context, writer); + } + + } + + public String getContentType(SolrQueryRequest request, + SolrQueryResponse response) { + if (request.getParams().get(PARAMETER_CONTENT_TYPE) != null) { + return request.getParams().get(PARAMETER_CONTENT_TYPE); + } + if (isWrappedResponse(request)) { + return JSONResponseWriter.CONTENT_TYPE_JSON_UTF8; + } + return "text/html"; + } + + public void init(NamedList args) { + // TODO + } + + private boolean isWrappedResponse(SolrQueryRequest request) { + return request.getParams().get(PARAMETER_JSON) != null; + } + + public String getJSONWrap(String xmlResult) { + // escape the double quotes and backslashes + String replace1 = xmlResult.replaceAll("\\\\", "\\\\\\\\"); + replace1 = replace1.replaceAll("\\n", "\\\\n"); + replace1 = replace1.replaceAll("\\r", "\\\\r"); + String replaced = replace1.replaceAll("\"", "\\\\\""); + // wrap it in a JSON object + return "{\"result\":\"" + replaced + "\"}"; + } + + /** + * A helper class that provides convenient methods for the raw solr response. + */ + public class RawResponseHelper { + + private Searcher searcher; + private SolrQueryResponse response; + private SolrQueryRequest request; + + public RawResponseHelper(SolrQueryRequest request, + SolrQueryResponse response) { + this.searcher = request.getSearcher(); + this.response = response; + this.request = request; + } + + public Iterator getResultIterator() { + final Iterator iterator = ((DocSlice) response.getValues() + .get("response")).iterator(); + return new Iterator() { + + public boolean hasNext() { + return iterator.hasNext(); + } + + public SolrDocument next() { + Document document = null; + SolrDocument solrDocument = new SolrDocument(); + try { + document = searcher.doc(iterator.next()); + new DocumentBuilder(request.getSchema()).loadStoredFields(solrDocument, document); + } catch (CorruptIndexException e) { + throw new RuntimeException("Error converting lucene document into solr document!"); + } catch (IOException e) { + throw new RuntimeException("Error converting lucene document into solr document!"); + } + + return solrDocument; + } + + public void remove() { + + } + + }; + } + + public String getRequestParameter(String param) { + return request.getParams().get(param); + } + + public SolrQueryRequest getRequest() { + return request; + } + + public SolrQueryResponse getResponse() { + return response; + } + + + } + + +} diff --git a/src/java/org/apache/solr/update/AddUpdateCommand.java b/src/java/org/apache/solr/update/AddUpdateCommand.java index dcfd06f3831..d719b5f0068 100644 --- a/src/java/org/apache/solr/update/AddUpdateCommand.java +++ b/src/java/org/apache/solr/update/AddUpdateCommand.java @@ -42,6 +42,10 @@ public class AddUpdateCommand extends UpdateCommand { public boolean allowDups; public boolean overwritePending; public boolean overwriteCommitted; + + + public int commitWithin = -1; + /** Reset state to reuse this object with a different document in the same request */ public void clear() { diff --git a/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/src/java/org/apache/solr/update/DirectUpdateHandler2.java index a1fe5451230..1fcef351a98 100644 --- a/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -198,7 +198,7 @@ public class DirectUpdateHandler2 extends UpdateHandler { synchronized (this) { // adding document -- prep writer openWriter(); - tracker.addedDocument(); + tracker.addedDocument( cmd.commitWithin ); } // end synchronized block // this is the only unsynchronized code in the iwAccess block, which @@ -424,7 +424,7 @@ public class DirectUpdateHandler2 extends UpdateHandler { SolrCore.log.info("AutoCommit: " + this); } - /** schedeule individual commits */ + /** schedule individual commits */ public synchronized void scheduleCommitWithin(long commitMaxTime) { // Check if there is a commit already scheduled for longer then this time @@ -443,30 +443,19 @@ public class DirectUpdateHandler2 extends UpdateHandler { /** Indicate that documents have been added */ - public void addedDocument() { + public void addedDocument( int commitWithin ) { docsSinceCommit++; lastAddedTime = System.currentTimeMillis(); // maxDocs-triggered autoCommit if( docsUpperBound > 0 && (docsSinceCommit > docsUpperBound) ) { - if (pending != null && - pending.getDelay(TimeUnit.MILLISECONDS) > DOC_COMMIT_DELAY_MS) { - // another commit is pending, but too far away (probably due to - // maxTime) - pending.cancel(false); - pending = null; - } - if (pending == null) { - // 1/4 second seems fast enough for anyone using maxDocs - pending = scheduler.schedule(this, DOC_COMMIT_DELAY_MS, - TimeUnit.MILLISECONDS); - } - } - // maxTime-triggered autoCommit - if( pending == null && timeUpperBound > 0 ) { - // Don't start a new event if one is already waiting - pending = scheduler.schedule( this, timeUpperBound, TimeUnit.MILLISECONDS ); + scheduleCommitWithin( DOC_COMMIT_DELAY_MS ); } + // maxTime-triggered autoCommit + long ctime = (commitWithin>0) ? commitWithin : timeUpperBound; + if( ctime > 0 ) { + scheduleCommitWithin( ctime ); + } } /** Inform tracker that a commit has occurred, cancel any pending commits */