lazy loading request handlers: SOLR-182

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@522584 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2007-03-26 18:21:26 +00:00
parent ff396cac79
commit 876dd28846
6 changed files with 493 additions and 86 deletions

View File

@ -125,6 +125,9 @@ New Features
the Lucene contrib.
(Otis Gospodnetic and Adam Hiatt)
17. SOLR-182: allow lazy loading of request handlers on first request.
(Ryan McKinley via yonik)
Changes in runtime behavior
1. Highlighting using DisMax will only pick up terms from the main
user query, not boost or filter queries (klaas).

View File

@ -17,76 +17,293 @@
package org.apache.solr.core;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.StandardRequestHandler;
import org.w3c.dom.NodeList;
import org.w3c.dom.Node;
import java.net.URL;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.logging.Logger;
import javax.xml.xpath.XPathConstants;
import java.util.logging.Logger;
import java.util.HashMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.StandardRequestHandler;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.NamedList;
import org.apache.solr.util.SimpleOrderedMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/**
* @author yonik
*/
final class RequestHandlers {
public static Logger log = Logger.getLogger(org.apache.solr.core.RequestHandlers.class.getName());
public static Logger log = Logger.getLogger(RequestHandlers.class.getName());
public static final String DEFAULT_HANDLER_NAME="standard";
final HashMap<String, SolrRequestHandler> map = new HashMap<String,SolrRequestHandler>();
public RequestHandlers(Config config) {
NodeList nodes = (NodeList)config.evaluate("requestHandler", XPathConstants.NODESET);
if (nodes!=null) {
for (int i=0; i<nodes.getLength(); i++) {
Node node = nodes.item(i);
// We can tolerate an error in some request handlers, still load the
// others, and have a working system.
try {
String name = DOMUtil.getAttr(node,"name","requestHandler config");
String className = DOMUtil.getAttr(node,"class","requestHandler config");
log.info("adding requestHandler " + name + "=" + className);
SolrRequestHandler handler = (SolrRequestHandler) Config.newInstance(className);
handler.init(DOMUtil.childNodesToNamedList(node));
put(name, handler);
} catch (Exception e) {
SolrException.logOnce(log,null,e);
}
}
}
//
// Get the default handler and add it in the map under null and empty
// to act as the default.
//
SolrRequestHandler handler = get(DEFAULT_HANDLER_NAME);
if (handler == null) {
handler = new StandardRequestHandler();
put(DEFAULT_HANDLER_NAME, handler);
}
put(null, handler);
put("", handler);
}
// Use a synchronized map - since the handlers can be changed at runtime,
// the map implementaion should be thread safe
private final Map<String, SolrRequestHandler> handlers = Collections.synchronizedMap(
new HashMap<String,SolrRequestHandler>() );
/**
* @return the RequestHandler registered at the given name
*/
public SolrRequestHandler get(String handlerName) {
return map.get(handlerName);
return handlers.get(handlerName);
}
public void put(String handlerName, SolrRequestHandler handler) {
map.put(handlerName, handler);
/**
* Handlers must be initalized before calling this function. As soon as this is
* called, the handler can immediatly accept requests.
*
* This call is thread safe.
*
* @return the previous handler at the given path or null
*/
public SolrRequestHandler register( String handlerName, SolrRequestHandler handler ) {
if( handler == null ) {
return handlers.remove( handlerName );
}
SolrRequestHandler old = handlers.put(handlerName, handler);
if (handlerName != null && handlerName != "") {
if (handler instanceof SolrInfoMBean) {
SolrInfoRegistry.getRegistry().put(handlerName, (SolrInfoMBean)handler);
}
}
return old;
}
/**
* Returns an unmodifieable Map containing the registered handlers
*/
public Map<String,SolrRequestHandler> getRequestHandlers() {
return Collections.unmodifiableMap( handlers );
}
/**
* Read solrconfig.xml and register the appropriate handlers
*
* This function should <b>only</b> be called from the SolrCore constructor. It is
* not intended as a public API.
*
* While the normal runtime registration contract is that handlers MUST be initalizad
* before they are registered, this function does not do that exactly.
*
* This funciton registers all handlers first and then calls init() for each one.
*
* This is OK because this function is only called at startup and there is no chance that
* a handler could be asked to handle a request before it is initalized.
*
* The advantage to this approach is that handlers can know what path they are registered
* to and what other handlers are avaliable at startup.
*
* Handlers will be registered and initalized in the order they appear in solrconfig.xml
*/
@SuppressWarnings("unchecked")
void initHandlersFromConfig( Config config )
{
NodeList nodes = (NodeList)config.evaluate("requestHandler", XPathConstants.NODESET);
if (nodes !=null ) {
// make sure it only once/handler and that that handlers get initalized in the
// order they were defined
Map<String,NamedList<Object>> names = new LinkedHashMap<String,NamedList<Object>>();
for (int i=0; i<nodes.getLength(); i++) {
Node node = nodes.item(i);
// In a production environment, we can tolerate an error in some request handlers,
// still load the others, and have a working system.
try {
String name = DOMUtil.getAttr(node,"name","requestHandler config");
String className = DOMUtil.getAttr(node,"class","requestHandler config");
String startup = DOMUtil.getAttr(node,"startup", null );
NamedList<Object> args = DOMUtil.childNodesToNamedList(node);
// Perhaps lazy load the request handler with a wrapper
SolrRequestHandler handler = null;
if( "lazy".equals( startup ) ) {
log.info("adding lazy requestHandler: " + name + "=" + className);
handler = new LazyRequestHandlerWrapper( className, args );
}
else {
Class<? extends SolrRequestHandler> clazz = Config.findClass( className, new String[]{} );
log.info("adding requestHandler: " + name + "=" + className);
handler = clazz.newInstance();
}
SolrRequestHandler old = register( name, handler );
if( old != null ) {
// TODO: SOLR-179?
log.warning( "multiple handlers registered on the same path! ignoring: "+old );
}
names.put( name, args );
}
catch (Exception e) {
// TODO: SOLR-179
SolrException.logOnce(log,null,e);
}
}
// Call init() on each handler after they have all been registered
for( Map.Entry<String, NamedList<Object>> reg : names.entrySet() ) {
try {
handlers.get( reg.getKey() ).init( reg.getValue() );
}
catch( Exception e ) {
// TODO: SOLR-179
SolrException.logOnce(log,null,e);
}
}
}
//
// Get the default handler and add it in the map under null and empty
// to act as the default.
//
SolrRequestHandler handler = get(RequestHandlers.DEFAULT_HANDLER_NAME);
if (handler == null) {
handler = new StandardRequestHandler();
register(RequestHandlers.DEFAULT_HANDLER_NAME, handler);
}
register(null, handler);
register("", handler);
}
/**
* The <code>LazyRequestHandlerWrapper</core> wraps any {@link SolrRequestHandler}.
* Rather then instanciate and initalize the handler on startup, this wrapper waits
* untill it is actually called. This should only be used for handlers that are
* unlikely to be used in the normal lifecycle.
*
* You can enable lazy loading in solrconfig.xml using:
*
* <pre>
* &lt;requestHandler name="..." class="..." startup="lazy"&gt;
* ...
* &lt;/requestHandler&gt;
* </pre>
*
* This is a private class - if there is a real need for it to be public, it could
* move
*
* @author ryan
* @version $Id$
* @since solr 1.2
*/
private static final class LazyRequestHandlerWrapper implements SolrRequestHandler, SolrInfoMBean
{
private String _className;
private NamedList _args;
private SolrRequestHandler _handler;
public LazyRequestHandlerWrapper( String className, NamedList args )
{
_className = className;
_args = args;
_handler = null; // don't initalize
}
/**
* In normal use, this function will not be called
*/
public void init(NamedList args) {
// do nothing
}
/**
* Wait for the first request before initalizing the wrapped handler
*/
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
getWrappedHandler().handleRequest( req, rsp );
}
public synchronized SolrRequestHandler getWrappedHandler()
{
if( _handler == null ) {
try {
Class clazz = Config.findClass( _className, new String[]{} );
_handler = (SolrRequestHandler)clazz.newInstance();
_handler.init( _args );
}
catch( Exception ex ) {
throw new SolrException( 500, "lazy loading error", ex );
}
}
return _handler;
}
public String getHandlerClass()
{
return _className;
}
//////////////////////// SolrInfoMBeans methods //////////////////////
public String getName() {
return "Lazy["+_className+"]";
}
public String getDescription()
{
if( _handler == null ) {
return getName();
}
return _handler.getDescription();
}
public String getVersion() {
String rev = "$Revision$";
if( _handler != null ) {
rev += " :: " + _handler.getVersion();
}
return rev;
}
public String getSourceId() {
String rev = "$Id$";
if( _handler != null ) {
rev += " :: " + _handler.getSourceId();
}
return rev;
}
public String getSource() {
String rev = "$URL$";
if( _handler != null ) {
rev += "\n" + _handler.getSource();
}
return rev;
}
public URL[] getDocs() {
if( _handler == null ) {
return null;
}
return _handler.getDocs();
}
public Category getCategory()
{
return Category.QUERYHANDLER;
}
public NamedList getStatistics() {
if( _handler != null ) {
return _handler.getStatistics();
}
NamedList<String> lst = new SimpleOrderedMap<String>();
lst.add("note", "not initaized yet" );
return lst;
}
}
}

View File

@ -17,39 +17,46 @@
package org.apache.solr.core;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.solr.handler.XmlUpdateRequestHandler;
import org.apache.solr.request.*;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.*;
import org.apache.solr.util.*;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlPullParserFactory;
import javax.xml.xpath.XPathConstants;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Logger;
import javax.xml.xpath.XPathConstants;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.solr.request.JSONResponseWriter;
import org.apache.solr.request.PythonResponseWriter;
import org.apache.solr.request.QueryResponseWriter;
import org.apache.solr.request.RubyResponseWriter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.XMLResponseWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.DirectUpdateHandler;
import org.apache.solr.update.SolrIndexConfig;
import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.NamedList;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.SimpleOrderedMap;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/**
* @author yonik
@ -67,6 +74,7 @@ public final class SolrCore {
private final String index_path;
private final UpdateHandler updateHandler;
private static final long startTime = System.currentTimeMillis();
private final RequestHandlers reqHandlers = new RequestHandlers();
public long getStartTime() { return startTime; }
@ -105,13 +113,6 @@ public final class SolrCore {
public String getDataDir() { return dataDir; }
public String getIndexDir() { return index_path; }
private final RequestHandlers reqHandlers = new RequestHandlers(SolrConfig.config);
public SolrRequestHandler getRequestHandler(String handlerName) {
return reqHandlers.get(handlerName);
}
// gets a non-caching searcher
public SolrIndexSearcher newSearcher(String name) throws IOException {
return new SolrIndexSearcher(schema, name,getIndexDir(),false);
@ -202,6 +203,8 @@ public final class SolrCore {
initIndex();
initWriters();
reqHandlers.initHandlersFromConfig( SolrConfig.config );
try {
// Open the searcher *before* the handler so we don't end up opening
@ -218,7 +221,6 @@ public final class SolrCore {
}
}
public void close() {
log.info("CLOSING SolrCore!");
try {
@ -241,6 +243,50 @@ public final class SolrCore {
void finalizer() { close(); }
////////////////////////////////////////////////////////////////////////////////
// Request Handler
////////////////////////////////////////////////////////////////////////////////
/**
* Get the request handler registered to a given name.
*
* This function is thread safe.
*/
public SolrRequestHandler getRequestHandler(String handlerName) {
return reqHandlers.get(handlerName);
}
/**
* Returns an unmodifieable Map containing the registered handlers
*/
public Map<String,SolrRequestHandler> getRequestHandlers() {
return reqHandlers.getRequestHandlers();
}
/**
* Registers a handler at the specified location. If one exists there, it will be replaced.
* To remove a handler, register <code>null</code> at its path
*
* Once registered the handler can be accessed through:
* <pre>
* http://${host}:${port}/${context}/${handlerName}
* or:
* http://${host}:${port}/${context}/select?qt=${handlerName}
* </pre>
*
* Handlers <em>must</em> be initalized before getting registered. Registered
* handlers can immediatly accept requests.
*
* This call is thread safe.
*
* @return the previous <code>SolrRequestHandler</code> registered to this name <code>null</code> if none.
*/
public SolrRequestHandler registerRequestHandler(String handlerName, SolrRequestHandler handler) {
return reqHandlers.register(handlerName,handler);
}
////////////////////////////////////////////////////////////////////////////////
// Update Handler
////////////////////////////////////////////////////////////////////////////////
@ -251,7 +297,7 @@ public final class SolrCore {
*/
public UpdateHandler getUpdateHandler()
{
return updateHandler;
return updateHandler;
}
////////////////////////////////////////////////////////////////////////////////
@ -610,7 +656,7 @@ public final class SolrCore {
log.info(req.getContext().get("path") + " "
+ req.getParamString()+ " 0 "+
(int)(rsp.getEndTime() - req.getStartTime()));
(int)(rsp.getEndTime() - req.getStartTime()));
}
@Deprecated
@ -719,3 +765,7 @@ public final class SolrCore {

View File

@ -0,0 +1,65 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.StandardRequestHandler;
import org.apache.solr.util.AbstractSolrTestCase;
public class RequestHandlersTest extends AbstractSolrTestCase {
public String getSchemaFile() { return "schema.xml"; }
public String getSolrConfigFile() { return "solrconfig.xml"; }
public void testLazyLoading() {
SolrCore core = SolrCore.getSolrCore();
SolrRequestHandler handler = core.getRequestHandler( "lazy" );
assertFalse( handler instanceof StandardRequestHandler );
// But it should behave just like the 'defaults' request handler above
assertU(adoc("id", "42",
"name", "Zapp Brannigan"));
assertU(adoc("id", "43",
"title", "Democratic Order of Planets"));
assertU(adoc("id", "44",
"name", "The Zapper"));
assertU(adoc("id", "45",
"title", "25 star General"));
assertU(adoc("id", "46",
"subject", "Defeated the pacifists of the Gandhi nebula"));
assertU(adoc("id", "47",
"text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!"));
assertU(commit());
assertQ("lazy request handler returns all matches",
req("id:[42 TO 47]"),
"*[count(//doc)=6]"
);
assertQ("lazy handler returns fewer matches",
req("q", "id:[42 TO 47]", "qt","defaults"),
"*[count(//doc)=4]"
);
assertQ("lazy handler includes highlighting",
req("q", "name:Zapp OR title:General", "qt","defaults"),
"//lst[@name='highlighting']"
);
}
}

View File

@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.util.AbstractSolrTestCase;
public class SolrCoreTest extends AbstractSolrTestCase {
public String getSchemaFile() { return "schema.xml"; }
public String getSolrConfigFile() { return "solrconfig.xml"; }
public void testRequestHandlerRegistry() {
// property values defined in build.xml
SolrCore core = SolrCore.getSolrCore();
EmptyRequestHandler handler1 = new EmptyRequestHandler();
EmptyRequestHandler handler2 = new EmptyRequestHandler();
String path = "/this/is A path /that won't be registered!";
SolrRequestHandler old = core.registerRequestHandler( path, handler1 );
assertNull( old ); // should not be anything...
assertEquals( core.getRequestHandlers().get( path ), handler1 );
old = core.registerRequestHandler( path, handler2 );
assertEquals( old, handler1 ); // should pop out the old one
assertEquals( core.getRequestHandlers().get( path ), handler2 );
}
}
/**
* An empty handler for testing
*/
class EmptyRequestHandler extends RequestHandlerBase
{
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
// nothing!
}
@Override public String getDescription() { return null; }
@Override public String getSource() { return null; }
@Override public String getSourceId() { return null; }
@Override public String getVersion() { return null; }
}

View File

@ -253,6 +253,15 @@
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<!-- test query parameter defaults -->
<requestHandler name="lazy" class="solr.StandardRequestHandler" startup="lazy">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<!-- enable streaming for testing... -->
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />