From 876dd28846aff89d330f5a6a2ee2edd30b2662ed Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Mon, 26 Mar 2007 18:21:26 +0000 Subject: [PATCH] lazy loading request handlers: SOLR-182 git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@522584 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + .../org/apache/solr/core/RequestHandlers.java | 319 +++++++++++++++--- src/java/org/apache/solr/core/SolrCore.java | 120 +++++-- .../apache/solr/core/RequestHandlersTest.java | 65 ++++ .../org/apache/solr/core/SolrCoreTest.java | 63 ++++ src/test/test-files/solr/conf/solrconfig.xml | 9 + 6 files changed, 493 insertions(+), 86 deletions(-) create mode 100755 src/test/org/apache/solr/core/RequestHandlersTest.java create mode 100755 src/test/org/apache/solr/core/SolrCoreTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 373e3337dcb..ba06b878128 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -125,6 +125,9 @@ New Features the Lucene contrib. (Otis Gospodnetic and Adam Hiatt) +17. SOLR-182: allow lazy loading of request handlers on first request. + (Ryan McKinley via yonik) + Changes in runtime behavior 1. Highlighting using DisMax will only pick up terms from the main user query, not boost or filter queries (klaas). diff --git a/src/java/org/apache/solr/core/RequestHandlers.java b/src/java/org/apache/solr/core/RequestHandlers.java index 5dce8c0c20c..bd540b5e5f1 100644 --- a/src/java/org/apache/solr/core/RequestHandlers.java +++ b/src/java/org/apache/solr/core/RequestHandlers.java @@ -17,76 +17,293 @@ package org.apache.solr.core; -import org.apache.solr.util.DOMUtil; -import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.request.StandardRequestHandler; -import org.w3c.dom.NodeList; -import org.w3c.dom.Node; +import java.net.URL; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.logging.Logger; import javax.xml.xpath.XPathConstants; -import java.util.logging.Logger; -import java.util.HashMap; + +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.StandardRequestHandler; +import org.apache.solr.util.DOMUtil; +import org.apache.solr.util.NamedList; +import org.apache.solr.util.SimpleOrderedMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; /** * @author yonik */ final class RequestHandlers { - public static Logger log = Logger.getLogger(org.apache.solr.core.RequestHandlers.class.getName()); + public static Logger log = Logger.getLogger(RequestHandlers.class.getName()); public static final String DEFAULT_HANDLER_NAME="standard"; - final HashMap map = new HashMap(); - - public RequestHandlers(Config config) { - NodeList nodes = (NodeList)config.evaluate("requestHandler", XPathConstants.NODESET); - - if (nodes!=null) { - for (int i=0; i handlers = Collections.synchronizedMap( + new HashMap() ); + /** + * @return the RequestHandler registered at the given name + */ public SolrRequestHandler get(String handlerName) { - return map.get(handlerName); + return handlers.get(handlerName); } - public void put(String handlerName, SolrRequestHandler handler) { - map.put(handlerName, handler); + /** + * Handlers must be initalized before calling this function. As soon as this is + * called, the handler can immediatly accept requests. + * + * This call is thread safe. + * + * @return the previous handler at the given path or null + */ + public SolrRequestHandler register( String handlerName, SolrRequestHandler handler ) { + if( handler == null ) { + return handlers.remove( handlerName ); + } + SolrRequestHandler old = handlers.put(handlerName, handler); if (handlerName != null && handlerName != "") { if (handler instanceof SolrInfoMBean) { SolrInfoRegistry.getRegistry().put(handlerName, (SolrInfoMBean)handler); } } + return old; } + /** + * Returns an unmodifieable Map containing the registered handlers + */ + public Map getRequestHandlers() { + return Collections.unmodifiableMap( handlers ); + } + + + /** + * Read solrconfig.xml and register the appropriate handlers + * + * This function should only be called from the SolrCore constructor. It is + * not intended as a public API. + * + * While the normal runtime registration contract is that handlers MUST be initalizad + * before they are registered, this function does not do that exactly. + * + * This funciton registers all handlers first and then calls init() for each one. + * + * This is OK because this function is only called at startup and there is no chance that + * a handler could be asked to handle a request before it is initalized. + * + * The advantage to this approach is that handlers can know what path they are registered + * to and what other handlers are avaliable at startup. + * + * Handlers will be registered and initalized in the order they appear in solrconfig.xml + */ + @SuppressWarnings("unchecked") + void initHandlersFromConfig( Config config ) + { + NodeList nodes = (NodeList)config.evaluate("requestHandler", XPathConstants.NODESET); + + if (nodes !=null ) { + // make sure it only once/handler and that that handlers get initalized in the + // order they were defined + Map> names = new LinkedHashMap>(); + for (int i=0; i args = DOMUtil.childNodesToNamedList(node); + + // Perhaps lazy load the request handler with a wrapper + SolrRequestHandler handler = null; + if( "lazy".equals( startup ) ) { + log.info("adding lazy requestHandler: " + name + "=" + className); + handler = new LazyRequestHandlerWrapper( className, args ); + } + else { + Class clazz = Config.findClass( className, new String[]{} ); + log.info("adding requestHandler: " + name + "=" + className); + handler = clazz.newInstance(); + } + + SolrRequestHandler old = register( name, handler ); + if( old != null ) { + // TODO: SOLR-179? + log.warning( "multiple handlers registered on the same path! ignoring: "+old ); + } + names.put( name, args ); + } + catch (Exception e) { + // TODO: SOLR-179 + SolrException.logOnce(log,null,e); + } + } + + // Call init() on each handler after they have all been registered + for( Map.Entry> reg : names.entrySet() ) { + try { + handlers.get( reg.getKey() ).init( reg.getValue() ); + } + catch( Exception e ) { + // TODO: SOLR-179 + SolrException.logOnce(log,null,e); + } + } + } + + // + // Get the default handler and add it in the map under null and empty + // to act as the default. + // + SolrRequestHandler handler = get(RequestHandlers.DEFAULT_HANDLER_NAME); + if (handler == null) { + handler = new StandardRequestHandler(); + register(RequestHandlers.DEFAULT_HANDLER_NAME, handler); + } + register(null, handler); + register("", handler); + } + + + /** + * The LazyRequestHandlerWrapper wraps any {@link SolrRequestHandler}. + * Rather then instanciate and initalize the handler on startup, this wrapper waits + * untill it is actually called. This should only be used for handlers that are + * unlikely to be used in the normal lifecycle. + * + * You can enable lazy loading in solrconfig.xml using: + * + * 
+   *  <requestHandler name="..." class="..." startup="lazy">
+   *    ...
+   *  </requestHandler>
+   *
+ * + * This is a private class - if there is a real need for it to be public, it could + * move + * + * @author ryan + * @version $Id$ + * @since solr 1.2 + */ + private static final class LazyRequestHandlerWrapper implements SolrRequestHandler, SolrInfoMBean + { + private String _className; + private NamedList _args; + private SolrRequestHandler _handler; + + public LazyRequestHandlerWrapper( String className, NamedList args ) + { + _className = className; + _args = args; + _handler = null; // don't initalize + } + + /** + * In normal use, this function will not be called + */ + public void init(NamedList args) { + // do nothing + } + + /** + * Wait for the first request before initalizing the wrapped handler + */ + public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { + getWrappedHandler().handleRequest( req, rsp ); + } + + public synchronized SolrRequestHandler getWrappedHandler() + { + if( _handler == null ) { + try { + Class clazz = Config.findClass( _className, new String[]{} ); + _handler = (SolrRequestHandler)clazz.newInstance(); + _handler.init( _args ); + } + catch( Exception ex ) { + throw new SolrException( 500, "lazy loading error", ex ); + } + } + return _handler; + } + + public String getHandlerClass() + { + return _className; + } + + //////////////////////// SolrInfoMBeans methods ////////////////////// + + public String getName() { + return "Lazy["+_className+"]"; + } + + public String getDescription() + { + if( _handler == null ) { + return getName(); + } + return _handler.getDescription(); + } + + public String getVersion() { + String rev = "$Revision$"; + if( _handler != null ) { + rev += " :: " + _handler.getVersion(); + } + return rev; + } + + public String getSourceId() { + String rev = "$Id$"; + if( _handler != null ) { + rev += " :: " + _handler.getSourceId(); + } + return rev; + } + + public String getSource() { + String rev = "$URL$"; + if( _handler != null ) { + rev += "\n" + _handler.getSource(); + } + return rev; + } + + public URL[] getDocs() { + if( _handler == null ) { + return null; + } + return _handler.getDocs(); + } + + public Category getCategory() + { + return Category.QUERYHANDLER; + } + + public NamedList getStatistics() { + if( _handler != null ) { + return _handler.getStatistics(); + } + NamedList lst = new SimpleOrderedMap(); + lst.add("note", "not initaized yet" ); + return lst; + } + } } + + + diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java index dccdcb4a77a..281dfb58952 100644 --- a/src/java/org/apache/solr/core/SolrCore.java +++ b/src/java/org/apache/solr/core/SolrCore.java @@ -17,39 +17,46 @@ package org.apache.solr.core; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.solr.handler.XmlUpdateRequestHandler; -import org.apache.solr.request.*; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.SchemaField; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.update.*; -import org.apache.solr.util.*; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xmlpull.v1.XmlPullParser; -import org.xmlpull.v1.XmlPullParserException; -import org.xmlpull.v1.XmlPullParserFactory; - -import javax.xml.xpath.XPathConstants; import java.io.File; import java.io.IOException; -import java.io.PrintWriter; -import java.io.Reader; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.logging.Logger; +import javax.xml.xpath.XPathConstants; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.solr.request.JSONResponseWriter; +import org.apache.solr.request.PythonResponseWriter; +import org.apache.solr.request.QueryResponseWriter; +import org.apache.solr.request.RubyResponseWriter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.XMLResponseWriter; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.DirectUpdateHandler; +import org.apache.solr.update.SolrIndexConfig; +import org.apache.solr.update.SolrIndexWriter; +import org.apache.solr.update.UpdateHandler; +import org.apache.solr.util.DOMUtil; +import org.apache.solr.util.NamedList; +import org.apache.solr.util.RefCounted; +import org.apache.solr.util.SimpleOrderedMap; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + /** * @author yonik @@ -67,6 +74,7 @@ public final class SolrCore { private final String index_path; private final UpdateHandler updateHandler; private static final long startTime = System.currentTimeMillis(); + private final RequestHandlers reqHandlers = new RequestHandlers(); public long getStartTime() { return startTime; } @@ -105,13 +113,6 @@ public final class SolrCore { public String getDataDir() { return dataDir; } public String getIndexDir() { return index_path; } - private final RequestHandlers reqHandlers = new RequestHandlers(SolrConfig.config); - - public SolrRequestHandler getRequestHandler(String handlerName) { - return reqHandlers.get(handlerName); - } - - // gets a non-caching searcher public SolrIndexSearcher newSearcher(String name) throws IOException { return new SolrIndexSearcher(schema, name,getIndexDir(),false); @@ -202,6 +203,8 @@ public final class SolrCore { initIndex(); initWriters(); + + reqHandlers.initHandlersFromConfig( SolrConfig.config ); try { // Open the searcher *before* the handler so we don't end up opening @@ -218,7 +221,6 @@ public final class SolrCore { } } - public void close() { log.info("CLOSING SolrCore!"); try { @@ -241,6 +243,50 @@ public final class SolrCore { void finalizer() { close(); } + + //////////////////////////////////////////////////////////////////////////////// + // Request Handler + //////////////////////////////////////////////////////////////////////////////// + + /** + * Get the request handler registered to a given name. + * + * This function is thread safe. + */ + public SolrRequestHandler getRequestHandler(String handlerName) { + return reqHandlers.get(handlerName); + } + + /** + * Returns an unmodifieable Map containing the registered handlers + */ + public Map getRequestHandlers() { + return reqHandlers.getRequestHandlers(); + } + + /** + * Registers a handler at the specified location. If one exists there, it will be replaced. + * To remove a handler, register null at its path + * + * Once registered the handler can be accessed through: + * 
+   *   http://${host}:${port}/${context}/${handlerName}
+   * or:  
+   *   http://${host}:${port}/${context}/select?qt=${handlerName}
+   *
+ * + * Handlers must be initalized before getting registered. Registered + * handlers can immediatly accept requests. + * + * This call is thread safe. + * + * @return the previous SolrRequestHandler registered to this name null if none. + */ + public SolrRequestHandler registerRequestHandler(String handlerName, SolrRequestHandler handler) { + return reqHandlers.register(handlerName,handler); + } + + //////////////////////////////////////////////////////////////////////////////// // Update Handler //////////////////////////////////////////////////////////////////////////////// @@ -251,7 +297,7 @@ public final class SolrCore { */ public UpdateHandler getUpdateHandler() { - return updateHandler; + return updateHandler; } //////////////////////////////////////////////////////////////////////////////// @@ -610,7 +656,7 @@ public final class SolrCore { log.info(req.getContext().get("path") + " " + req.getParamString()+ " 0 "+ - (int)(rsp.getEndTime() - req.getStartTime())); + (int)(rsp.getEndTime() - req.getStartTime())); } @Deprecated @@ -719,3 +765,7 @@ public final class SolrCore { + + + + diff --git a/src/test/org/apache/solr/core/RequestHandlersTest.java b/src/test/org/apache/solr/core/RequestHandlersTest.java new file mode 100755 index 00000000000..80f05a643e2 --- /dev/null +++ b/src/test/org/apache/solr/core/RequestHandlersTest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.StandardRequestHandler; +import org.apache.solr.util.AbstractSolrTestCase; + +public class RequestHandlersTest extends AbstractSolrTestCase { + + public String getSchemaFile() { return "schema.xml"; } + public String getSolrConfigFile() { return "solrconfig.xml"; } + + + public void testLazyLoading() { + SolrCore core = SolrCore.getSolrCore(); + SolrRequestHandler handler = core.getRequestHandler( "lazy" ); + assertFalse( handler instanceof StandardRequestHandler ); + + // But it should behave just like the 'defaults' request handler above + assertU(adoc("id", "42", + "name", "Zapp Brannigan")); + assertU(adoc("id", "43", + "title", "Democratic Order of Planets")); + assertU(adoc("id", "44", + "name", "The Zapper")); + assertU(adoc("id", "45", + "title", "25 star General")); + assertU(adoc("id", "46", + "subject", "Defeated the pacifists of the Gandhi nebula")); + assertU(adoc("id", "47", + "text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!")); + assertU(commit()); + + assertQ("lazy request handler returns all matches", + req("id:[42 TO 47]"), + "*[count(//doc)=6]" + ); + + assertQ("lazy handler returns fewer matches", + req("q", "id:[42 TO 47]", "qt","defaults"), + "*[count(//doc)=4]" + ); + + assertQ("lazy handler includes highlighting", + req("q", "name:Zapp OR title:General", "qt","defaults"), + "//lst[@name='highlighting']" + ); + } +} diff --git a/src/test/org/apache/solr/core/SolrCoreTest.java b/src/test/org/apache/solr/core/SolrCoreTest.java new file mode 100755 index 00000000000..f70ad94e00f --- /dev/null +++ b/src/test/org/apache/solr/core/SolrCoreTest.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.util.AbstractSolrTestCase; + +public class SolrCoreTest extends AbstractSolrTestCase { + + public String getSchemaFile() { return "schema.xml"; } + public String getSolrConfigFile() { return "solrconfig.xml"; } + + public void testRequestHandlerRegistry() { + // property values defined in build.xml + SolrCore core = SolrCore.getSolrCore(); + + EmptyRequestHandler handler1 = new EmptyRequestHandler(); + EmptyRequestHandler handler2 = new EmptyRequestHandler(); + + String path = "/this/is A path /that won't be registered!"; + SolrRequestHandler old = core.registerRequestHandler( path, handler1 ); + assertNull( old ); // should not be anything... + assertEquals( core.getRequestHandlers().get( path ), handler1 ); + old = core.registerRequestHandler( path, handler2 ); + assertEquals( old, handler1 ); // should pop out the old one + assertEquals( core.getRequestHandlers().get( path ), handler2 ); + } +} + + +/** + * An empty handler for testing + */ +class EmptyRequestHandler extends RequestHandlerBase +{ + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + // nothing! + } + + @Override public String getDescription() { return null; } + @Override public String getSource() { return null; } + @Override public String getSourceId() { return null; } + @Override public String getVersion() { return null; } +} diff --git a/src/test/test-files/solr/conf/solrconfig.xml b/src/test/test-files/solr/conf/solrconfig.xml index c0d6d544f70..edda9913989 100644 --- a/src/test/test-files/solr/conf/solrconfig.xml +++ b/src/test/test-files/solr/conf/solrconfig.xml @@ -253,6 +253,15 @@ text,name,subject,title,whitetok + + + + + 4 + true + text,name,subject,title,whitetok + +