From cbc9fae327920cd0f7b912dc0ead45631481c6d9 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 9 Jun 2009 05:50:57 +0000 Subject: [PATCH] SOLR-1198 moved updateHandler, updateprocessor to solrconfig git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@782885 13f79535-47bb-0310-9956-ffa450edef68 --- src/java/org/apache/solr/core/SolrConfig.java | 134 ++++++++++++++---- src/java/org/apache/solr/core/SolrCore.java | 117 ++++++--------- .../solr/update/DirectUpdateHandler2.java | 4 +- 3 files changed, 154 insertions(+), 101 deletions(-) diff --git a/src/java/org/apache/solr/core/SolrConfig.java b/src/java/org/apache/solr/core/SolrConfig.java index 79810146781..755465daff2 100644 --- a/src/java/org/apache/solr/core/SolrConfig.java +++ b/src/java/org/apache/solr/core/SolrConfig.java @@ -19,6 +19,7 @@ package org.apache.solr.core; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.DOMUtil; +import org.apache.solr.common.SolrException; import org.apache.solr.handler.PingRequestHandler; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -33,10 +34,13 @@ import org.slf4j.LoggerFactory; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.w3c.dom.NamedNodeMap; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; import java.util.*; import java.util.regex.Pattern; @@ -169,6 +173,8 @@ public class SolrConfig extends Config { maxWarmingSearchers = getInt("query/maxWarmingSearchers",Integer.MAX_VALUE); loadPluginInfo(); + updateProcessorChainInfo = loadUpdateProcessorInfo(); + updateHandlerInfo = loadUpdatehandlerInfo(); Config.log.info("Loaded SolrConfig: " + name); @@ -176,27 +182,62 @@ public class SolrConfig extends Config { config = this; } - protected void loadPluginInfo() { - List reqFields = Arrays.asList("name","class"); - reqHandlerInfo = loadPluginInfo("requestHandler",reqFields); - respWriterInfo = loadPluginInfo("queryResponseWriter",reqFields); - valueSourceParserInfo = loadPluginInfo("valueSourceParser",reqFields); - queryParserInfo = loadPluginInfo("queryParser",reqFields); - searchComponentInfo = loadPluginInfo("searchComponent",reqFields); - List plugins = loadPluginInfo("directoryFactory",reqFields); - directoryfactoryInfo = plugins.isEmpty() ? null:plugins.get(0); - reqFields = Arrays.asList("class"); - plugins = loadPluginInfo("mainIndex/deletionPolicy",reqFields); - deletionPolicyInfo = plugins.isEmpty() ? null : plugins.get(0); - firstSearcherListenerInfo = loadPluginInfo("//listener[@event='firstSearcher']",reqFields); - newSearcherListenerInfo = loadPluginInfo("//listener[@event='newSearcher']",reqFields); + protected UpdateHandlerInfo loadUpdatehandlerInfo() { + return new UpdateHandlerInfo(get("updateHandler/@class",null), + getInt("updateHandler/autoCommit/maxDocs",-1), + getInt("updateHandler/autoCommit/maxTime",-1), + getInt("updateHandler/commitIntervalLowerBound",-1)); } - private List loadPluginInfo(String tag, List reqFields) { + protected void loadPluginInfo() { + reqHandlerInfo = loadPluginInfo("requestHandler",true); + respWriterInfo = loadPluginInfo("queryResponseWriter",true); + valueSourceParserInfo = loadPluginInfo("valueSourceParser",true); + queryParserInfo = loadPluginInfo("queryParser",true); + searchComponentInfo = loadPluginInfo("searchComponent",true); + List plugins = loadPluginInfo("directoryFactory",true); + directoryfactoryInfo = plugins.isEmpty() ? null:plugins.get(0); + plugins = loadPluginInfo("mainIndex/deletionPolicy",false); + deletionPolicyInfo = plugins.isEmpty() ? null : plugins.get(0); + firstSearcherListenerInfo = loadPluginInfo("//listener[@event='firstSearcher']",false); + newSearcherListenerInfo = loadPluginInfo("//listener[@event='newSearcher']",false); + } + + protected Map> loadUpdateProcessorInfo() { + HashMap> chains = new HashMap>(); + NodeList nodes = (NodeList) evaluate("updateRequestProcessorChain", XPathConstants.NODESET); + if (nodes != null) { + boolean requireName = nodes.getLength() > 1; + for (int i = 0; i < nodes.getLength(); i++) { + Node node = nodes.item(i); + String name = DOMUtil.getAttr(node,"name", requireName ? "[solrconfig.xml] updateRequestProcessorChain":null); + boolean isDefault = "true".equals( DOMUtil.getAttr(node,"default", null ) ); + XPath xpath = getXPath(); + try { + NodeList nl = (NodeList) xpath.evaluate("processor",node, XPathConstants.NODESET); + if((nl.getLength() <1)) { + throw new RuntimeException( "updateRequestProcessorChain require at least one processor"); + } + ArrayList result = new ArrayList(); + for (int j=0; j loadPluginInfo(String tag, boolean requireName) { ArrayList result = new ArrayList(); NodeList nodes = (NodeList) evaluate(tag, XPathConstants.NODESET); for (int i=0; i newSearcherListenerInfo; protected PluginInfo directoryfactoryInfo; + protected Map> updateProcessorChainInfo ; + protected UpdateHandlerInfo updateHandlerInfo ; public final int maxWarmingSearchers; public final boolean unlockOnStartup; @@ -372,32 +415,59 @@ public class SolrConfig extends Config { public LastModFrom getLastModFrom() { return lastModFrom; } } + public static class UpdateHandlerInfo{ + public final String className; + public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound; + + /** + * @param className + * @param autoCommmitMaxDocs set -1 as default + * @param autoCommmitMaxTime set -1 as default + * @param commitIntervalLowerBound set -1 as default + */ + public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, int commitIntervalLowerBound) { + this.className = className; + this.autoCommmitMaxDocs = autoCommmitMaxDocs; + this.autoCommmitMaxTime = autoCommmitMaxTime; + this.commitIntervalLowerBound = commitIntervalLowerBound; + } + } + public static class PluginInfo { final String startup, name, className; final boolean isDefault; final NamedList initArgs; + final Map otherAttributes; public PluginInfo(String startup, String name, String className, - boolean isdefault, NamedList initArgs) { + boolean isdefault, NamedList initArgs, Map otherAttrs) { this.startup = startup; this.name = name; this.className = className; this.isDefault = isdefault; this.initArgs = initArgs; + otherAttributes = otherAttrs == null ? Collections.emptyMap(): otherAttrs; } - public PluginInfo(Node node, String err, List reqFields) { - startup = getVal( node, "startup",reqFields,err); - name = getVal(node, "name", reqFields,err); - className = getVal(node, "class",reqFields,err); - isDefault = Boolean.parseBoolean(getVal(node,"default",reqFields,err)); + public PluginInfo(Node node, String err, boolean requireName) { + name = DOMUtil.getAttr(node, "name", requireName ? err : null); + className = DOMUtil.getAttr(node, "class", err ); + isDefault = Boolean.parseBoolean(DOMUtil.getAttr(node, "default", null)); + startup = DOMUtil.getAttr(node, "startup",null); initArgs = DOMUtil.childNodesToNamedList(node); - } + Map m = new HashMap(); + NamedNodeMap nnm = node.getAttributes(); + for (int i = 0; i < nnm.getLength(); i++) { + String name= nnm.item(i).getNodeName(); + if(knownAttrs.contains(name)) continue; + m.put(name, nnm.item(i).getNodeValue()); + } + otherAttributes = m.isEmpty() ? + Collections.emptyMap(): + Collections.unmodifiableMap(m); - private String getVal(Node node, String name, List required, String err) { - return DOMUtil.getAttr(node, name, required.contains(name) ? err : null); - } + } @Override public String toString() { @@ -405,10 +475,18 @@ public class SolrConfig extends Config { if(name != null) sb.append("name = "+name +","); if(className != null) sb.append("class = "+className +","); if(isDefault) sb.append("default = "+isDefault +","); + if(startup != null) sb.append("startup = "+startup +","); if(initArgs.size() >0) sb.append("args = "+initArgs); sb.append("}"); return sb.toString(); } + private static final Set knownAttrs = new HashSet(); + static { + knownAttrs.add("name"); + knownAttrs.add("class"); + knownAttrs.add("startup"); + knownAttrs.add("default"); + } } public List getReqHandlerInfo() { return reqHandlerInfo; } @@ -428,4 +506,8 @@ public class SolrConfig extends Config { public PluginInfo getDirectoryfactoryInfo() { return directoryfactoryInfo; } public PluginInfo getDeletionPolicyInfo() { return deletionPolicyInfo; } + + public Map> getUpdateProcessorChainInfo() { return updateProcessorChainInfo; } + + public UpdateHandlerInfo getUpdateHandlerInfo() { return updateHandlerInfo; } } diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java index dfb85be8210..e3449eb524c 100644 --- a/src/java/org/apache/solr/core/SolrCore.java +++ b/src/java/org/apache/solr/core/SolrCore.java @@ -86,6 +86,7 @@ public final class SolrCore implements SolrInfoMBean { private final CoreDescriptor coreDescriptor; private final SolrConfig solrConfig; + private final SolrResourceLoader resourceLoader; private final IndexSchema schema; private final String dataDir; private final UpdateHandler updateHandler; @@ -126,7 +127,7 @@ public final class SolrCore implements SolrInfoMBean { * @since solr 1.3 */ public SolrResourceLoader getResourceLoader() { - return solrConfig.getResourceLoader(); + return resourceLoader; } /** @@ -504,13 +505,13 @@ public final class SolrCore implements SolrInfoMBean { public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd) { coreDescriptor = cd; this.setName( name ); - SolrResourceLoader loader = config.getResourceLoader(); + resourceLoader = config.getResourceLoader(); if (dataDir == null) dataDir = config.get("dataDir",cd.getDataDir()); dataDir = SolrResourceLoader.normalizeDir(dataDir); - log.info(logid+"Opening new SolrCore at " + loader.getInstanceDir() + ", dataDir="+dataDir); + log.info(logid+"Opening new SolrCore at " + resourceLoader.getInstanceDir() + ", dataDir="+dataDir); if (schema==null) { schema = new IndexSchema(config, IndexSchema.DEFAULT_SCHEMA_FILE, null); @@ -542,7 +543,7 @@ public final class SolrCore implements SolrInfoMBean { initQParsers(); initValueSourceParsers(); - this.searchComponents = loadSearchComponents( config ); + this.searchComponents = loadSearchComponents(); // Processors initialized before the handlers updateProcessorChains = loadUpdateProcessorChains(); @@ -574,16 +575,15 @@ public final class SolrCore implements SolrInfoMBean { // one in the middle. // With lockless commits in Lucene now, this probably shouldn't be an issue anymore getSearcher(false,false,null); - - updateHandler = createUpdateHandler( - solrConfig.get("updateHandler/@class", DirectUpdateHandler2.class.getName()) - ); + String updateHandlerClass = solrConfig.getUpdateHandlerInfo().className; + + updateHandler = createUpdateHandler(updateHandlerClass == null ? DirectUpdateHandler2.class.getName():updateHandlerClass); infoRegistry.put("updateHandler", updateHandler); // Finally tell anyone who wants to know - loader.inform( loader ); - loader.inform( this ); + resourceLoader.inform( resourceLoader ); + resourceLoader.inform( this ); instance = this; // set singleton for backwards compatibility } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); @@ -597,79 +597,50 @@ public final class SolrCore implements SolrInfoMBean { /** - * Load the request processors configured in solrconfig.xml + * Load the request processors */ - private Map loadUpdateProcessorChains() { - final Map map = new HashMap(); - - final String parsingErrorText = "Parsing Update Request Processor Chain"; + private Map loadUpdateProcessorChains() { + final Map map = new HashMap(); UpdateRequestProcessorChain def = null; - - // This is kinda ugly, but at least it keeps the xpath logic in one place - // away from the Processors themselves. - XPath xpath = solrConfig.getXPath(); - NodeList nodes = (NodeList)solrConfig.evaluate("updateRequestProcessorChain", XPathConstants.NODESET); - boolean requireName = nodes.getLength() > 1; - if (nodes !=null ) { - for (int i=0; i> infos = solrConfig.getUpdateProcessorChainInfo(); + System.out.println(infos); + if (!infos.isEmpty()) { + boolean defaultProcessed = false; + List defProcessorChainInfo = infos.get(null);// this is the default one + for (Map.Entry> e : solrConfig.getUpdateProcessorChainInfo().entrySet()) { + List processorsInfo = e.getValue(); + if (processorsInfo == defProcessorChainInfo && defaultProcessed) { + map.put(e.getKey(), def); + continue; } - if( links == null || links.getLength() < 1 ) { - throw new RuntimeException( "updateRequestProcessorChain require at least one processor"); + UpdateRequestProcessorFactory[] chain = new UpdateRequestProcessorFactory[processorsInfo.size()]; + for (int i = 0; i < processorsInfo.size(); i++) { + SolrConfig.PluginInfo info = processorsInfo.get(i); + chain[i] = createInstance(info.className, UpdateRequestProcessorFactory.class, null); + chain[i].init(info.initArgs); } - - // keep a list of the factories... - final ArrayList factories = new ArrayList(links.getLength()); - // Load and initialize the plugin chain - AbstractPluginLoader loader - = new AbstractPluginLoader( "processor chain", false, false ) { - @Override - protected void init(UpdateRequestProcessorFactory plugin, Node node) throws Exception { - plugin.init( (node==null)?null:DOMUtil.childNodesToNamedList(node) ); - } - - @Override - protected UpdateRequestProcessorFactory register(String name, UpdateRequestProcessorFactory plugin) throws Exception { - factories.add( plugin ); - return null; - } - }; - loader.load( solrConfig.getResourceLoader(), links ); - - - UpdateRequestProcessorChain chain = new UpdateRequestProcessorChain( - factories.toArray( new UpdateRequestProcessorFactory[factories.size()] ) ); - if( isDefault || nodes.getLength()==1 ) { - def = chain; - } - if( name != null ) { - map.put(name, chain); + UpdateRequestProcessorChain processorChain = new UpdateRequestProcessorChain(chain); + map.put(e.getKey(), processorChain); + if (e.getKey() == null || processorsInfo == defProcessorChainInfo) { //this is the default one + defaultProcessed = true; + def = processorChain; } } } - - if( def == null ) { + + if (def == null) { // construct the default chain - UpdateRequestProcessorFactory[] factories = new UpdateRequestProcessorFactory[] { - new RunUpdateProcessorFactory(), - new LogUpdateProcessorFactory() + UpdateRequestProcessorFactory[] factories = new UpdateRequestProcessorFactory[]{ + new RunUpdateProcessorFactory(), + new LogUpdateProcessorFactory() }; - def = new UpdateRequestProcessorChain( factories ); + def = new UpdateRequestProcessorChain(factories); } - map.put( null, def ); - map.put( "", def ); + map.put(null, def); + map.put("", def); return map; } - + /** * @return an update processor registered to the given name. Throw an exception if this chain is undefined */ @@ -849,13 +820,13 @@ public final class SolrCore implements SolrInfoMBean { /** * Register the default search components */ - private Map loadSearchComponents( SolrConfig config ) + private Map loadSearchComponents() { Map components = new HashMap(); String xpath = "searchComponent"; NamedListPluginLoader loader = new NamedListPluginLoader( xpath, components ); - loader.load( config.getResourceLoader(), (NodeList)config.evaluate( xpath, XPathConstants.NODESET ) ); + loader.load( solrConfig.getResourceLoader(), (NodeList)solrConfig.evaluate( xpath, XPathConstants.NODESET ) ); final Map> standardcomponents = new HashMap>(); diff --git a/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 5a75c3c1c4f..fa201435151 100644 --- a/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -520,8 +520,8 @@ public class DirectUpdateHandler2 extends UpdateHandler { docsSinceCommit = 0; pending = null; - docsUpperBound = core.getSolrConfig().getInt("updateHandler/autoCommit/maxDocs", -1); - timeUpperBound = core.getSolrConfig().getInt("updateHandler/autoCommit/maxTime", -1); + docsUpperBound = core.getSolrConfig().getUpdateHandlerInfo().autoCommmitMaxDocs; //getInt("updateHandler/autoCommit/maxDocs", -1); + timeUpperBound = core.getSolrConfig().getUpdateHandlerInfo().autoCommmitMaxTime; //getInt("updateHandler/autoCommit/maxTime", -1); SolrCore.log.info("AutoCommit: " + this); }