From 33c0b65e47cb47a33fc5d6befaf76352dd85ff97 Mon Sep 17 00:00:00 2001 From: Erik Hatcher Date: Fri, 16 Feb 2007 16:42:49 +0000 Subject: [PATCH] SOLR-79: Add system property ${[:]} substitution for configuration files loaded, including schema.xml and solrconfig.xml. (Erik Hatcher with inspiration from Andrew Saar) git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@508485 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 4 + build.xml | 3 + src/java/org/apache/solr/core/Config.java | 8 +- .../solr/util/AbstractSolrTestCase.java | 2 +- src/java/org/apache/solr/util/DOMUtil.java | 142 +++++++++++++++++- .../org/apache/solr/core/TestBadConfig.java | 55 +++++++ src/test/org/apache/solr/core/TestConfig.java | 53 +++++++ .../test-files/solr/conf/bad_solrconfig.xml | 29 ++++ src/test/test-files/solr/conf/solrconfig.xml | 3 + 9 files changed, 289 insertions(+), 10 deletions(-) create mode 100644 src/test/org/apache/solr/core/TestBadConfig.java create mode 100644 src/test/org/apache/solr/core/TestConfig.java create mode 100644 src/test/test-files/solr/conf/bad_solrconfig.xml diff --git a/CHANGES.txt b/CHANGES.txt index 674c428244e..2d9be014338 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -82,6 +82,10 @@ New Features 10. SOLR-116: IndexInfoRequestHandler added. (Erik Hatcher) +11. SOLR-79: Add system property ${[:]} substitution for + configuration files loaded, including schema.xml and solrconfig.xml. + (Erik Hatcher with inspiration from Andrew Saar) + Changes in runtime behavior 1. Highlighting using DisMax will only pick up terms from the main user query, not boost or filter queries (klaas). diff --git a/build.xml b/build.xml index 4bf4a9089f6..661cce726e3 100644 --- a/build.xml +++ b/build.xml @@ -282,6 +282,9 @@ + + + diff --git a/src/java/org/apache/solr/core/Config.java b/src/java/org/apache/solr/core/Config.java index 58f05b05682..79105856048 100644 --- a/src/java/org/apache/solr/core/Config.java +++ b/src/java/org/apache/solr/core/Config.java @@ -59,6 +59,8 @@ public class Config { javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); doc = builder.parse(is); + + DOMUtil.substituteSystemProperties(doc); } public Document getDocument() { @@ -124,7 +126,7 @@ public class Config { if (nd==null) return null; String txt = DOMUtil.getText(nd); - + log.fine(name + ' '+path+'='+txt); return txt; @@ -255,7 +257,7 @@ public class Config { /** Singleton classloader loading resources specified in any configs */ private static ClassLoader classLoader = null; - + /** * Returns the singleton classloader to be use when loading resources * specified in any configs. @@ -269,7 +271,7 @@ public class Config { static ClassLoader getClassLoader() { if (null == classLoader) { classLoader = Thread.currentThread().getContextClassLoader(); - + File f = new File(getInstanceDir() + "lib/"); if (f.canRead() && f.isDirectory()) { File[] jarFiles = f.listFiles(); diff --git a/src/java/org/apache/solr/util/AbstractSolrTestCase.java b/src/java/org/apache/solr/util/AbstractSolrTestCase.java index 14c0c3f8202..6f901d11443 100644 --- a/src/java/org/apache/solr/util/AbstractSolrTestCase.java +++ b/src/java/org/apache/solr/util/AbstractSolrTestCase.java @@ -110,7 +110,7 @@ public abstract class AbstractSolrTestCase extends TestCase { * is set. */ public void tearDown() throws Exception { - h.close(); + if (h != null) { h.close(); } String skip = System.getProperty("solr.test.leavedatadir"); if (null != skip && 0 != skip.trim().length()) { System.err.println("NOTE: per solr.test.leavedatadir, dataDir will not be removed: " + dataDir.getAbsolutePath()); diff --git a/src/java/org/apache/solr/util/DOMUtil.java b/src/java/org/apache/solr/util/DOMUtil.java index 12531e3d515..bb06dae11ae 100644 --- a/src/java/org/apache/solr/util/DOMUtil.java +++ b/src/java/org/apache/solr/util/DOMUtil.java @@ -21,11 +21,14 @@ import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.apache.solr.core.SolrException; import java.util.Map; import java.util.HashMap; import java.util.List; import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Iterator; /** * @author yonik @@ -164,12 +167,12 @@ public class DOMUtil { public static String getText(Node nd) { short type = nd.getNodeType(); - + // for most node types, we can defer to the recursive helper method, // but when asked for the text of these types, we must return null // (Not the empty string) switch (type) { - + case Node.DOCUMENT_NODE: /* fall through */ case Node.DOCUMENT_TYPE_NODE: /* fall through */ case Node.NOTATION_NODE: /* fall through */ @@ -183,15 +186,15 @@ public class DOMUtil { /** @see #getText(Node) */ private static void getText(Node nd, StringBuilder buf) { - + short type = nd.getNodeType(); switch (type) { - + case Node.ELEMENT_NODE: /* fall through */ case Node.ENTITY_NODE: /* fall through */ case Node.ENTITY_REFERENCE_NODE: /* fall through */ - case Node.DOCUMENT_FRAGMENT_NODE: + case Node.DOCUMENT_FRAGMENT_NODE: NodeList childs = nd.getChildNodes(); for (int i = 0; i < childs.getLength(); i++) { Node child = childs.item(i); @@ -202,7 +205,7 @@ public class DOMUtil { } } break; - + case Node.ATTRIBUTE_NODE: /* fall through */ /* Putting Attribute nodes in this section does not exactly match the definition of how textContent should behave @@ -232,4 +235,131 @@ public class DOMUtil { } } + + /** + * Replaces ${system.property[:default value]} references in all attributes + * and text nodes of supplied node. If the system property is not defined, an empty string + * is substituted or the default value if provided. + * + * @param node DOM node to walk for substitutions + */ + public static void substituteSystemProperties(Node node) { + // loop through child nodes + Node child; + Node next = node.getFirstChild(); + while ((child = next) != null) { + + // set next before we change anything + next = child.getNextSibling(); + + // handle child by node type + if (child.getNodeType() == Node.TEXT_NODE) { + child.setNodeValue(substituteSystemProperty(child.getNodeValue())); + } else if (child.getNodeType() == Node.ELEMENT_NODE) { + // handle child elements with recursive call + NamedNodeMap attributes = child.getAttributes(); + for (int i = 0; i < attributes.getLength(); i++) { + Node attribute = attributes.item(i); + attribute.setNodeValue(substituteSystemProperty(attribute.getNodeValue())); + } + substituteSystemProperties(child); + } + } + } + + /* + * This method borrowed from Ant's PropertyHelper.replaceProperties: + * http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/ant/PropertyHelper.java + */ + private static String substituteSystemProperty(String value) { + if (value == null || value.indexOf('$') == -1) { + return value; + } + + List fragments = new ArrayList(); + List propertyRefs = new ArrayList(); + parsePropertyString(value, fragments, propertyRefs); + + StringBuffer sb = new StringBuffer(); + Iterator i = fragments.iterator(); + Iterator j = propertyRefs.iterator(); + + while (i.hasNext()) { + String fragment = i.next(); + if (fragment == null) { + String propertyName = j.next(); + String defaultValue = null; + int colon_index = propertyName.indexOf(':'); + if (colon_index > -1) { + defaultValue = propertyName.substring(colon_index + 1); + propertyName = propertyName.substring(0,colon_index); + } + fragment = System.getProperty(propertyName,defaultValue); + if (fragment == null) { + throw new SolrException(500, "No system property or default value specified for " + propertyName); + } + } + sb.append(fragment); + } + return sb.toString(); + } + + /* + * This method borrowed from Ant's PropertyHelper.parsePropertyStringDefault: + * http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/ant/PropertyHelper.java + */ + private static void parsePropertyString(String value, List fragments, List propertyRefs) { + int prev = 0; + int pos; + //search for the next instance of $ from the 'prev' position + while ((pos = value.indexOf("$", prev)) >= 0) { + + //if there was any text before this, add it as a fragment + //TODO, this check could be modified to go if pos>prev; + //seems like this current version could stick empty strings + //into the list + if (pos > 0) { + fragments.add(value.substring(prev, pos)); + } + //if we are at the end of the string, we tack on a $ + //then move past it + if (pos == (value.length() - 1)) { + fragments.add("$"); + prev = pos + 1; + } else if (value.charAt(pos + 1) != '{') { + //peek ahead to see if the next char is a property or not + //not a property: insert the char as a literal + /* + fragments.addElement(value.substring(pos + 1, pos + 2)); + prev = pos + 2; + */ + if (value.charAt(pos + 1) == '$') { + //backwards compatibility two $ map to one mode + fragments.add("$"); + prev = pos + 2; + } else { + //new behaviour: $X maps to $X for all values of X!='$' + fragments.add(value.substring(pos, pos + 2)); + prev = pos + 2; + } + + } else { + //property found, extract its name or bail on a typo + int endName = value.indexOf('}', pos); + if (endName < 0) { + throw new RuntimeException("Syntax error in property: " + value); + } + String propertyName = value.substring(pos + 2, endName); + fragments.add(null); + propertyRefs.add(propertyName); + prev = endName + 1; + } + } + //no more $ signs found + //if there is any tail to the string, append it + if (prev < value.length()) { + fragments.add(value.substring(prev)); + } + } + } diff --git a/src/test/org/apache/solr/core/TestBadConfig.java b/src/test/org/apache/solr/core/TestBadConfig.java new file mode 100644 index 00000000000..3b7c5148dec --- /dev/null +++ b/src/test/org/apache/solr/core/TestBadConfig.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import org.apache.solr.util.AbstractSolrTestCase; +import org.apache.solr.util.TestHarness; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPathConstants; +import java.io.File; + +public class TestBadConfig extends AbstractSolrTestCase { + + public String getSchemaFile() { return "schema.xml"; } + public String getSolrConfigFile() { return "bad_solrconfig.xml"; } + + public void setUp() throws Exception { + + dataDir = new File(System.getProperty("java.io.tmpdir") + + System.getProperty("file.separator") + + getClass().getName() + "-" + getName() + "-" + + System.currentTimeMillis()); + dataDir.mkdirs(); + try { + h = new TestHarness(dataDir.getAbsolutePath(), + getSolrConfigFile(), + getSchemaFile()); + fail("Exception should have been thrown"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("unset.sys.property")); + } + } + + + public void testNothing() { + // Empty test case as the real test is that the initialization of the TestHarness fails + assertTrue(true); + } +} \ No newline at end of file diff --git a/src/test/org/apache/solr/core/TestConfig.java b/src/test/org/apache/solr/core/TestConfig.java new file mode 100644 index 00000000000..ea454653307 --- /dev/null +++ b/src/test/org/apache/solr/core/TestConfig.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import org.apache.solr.util.AbstractSolrTestCase; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPathConstants; + +public class TestConfig extends AbstractSolrTestCase { + + public String getSchemaFile() { return "schema.xml"; } + public String getSolrConfigFile() { return "solrconfig.xml"; } + + public void testJavaProperty() { + // property values defined in build.xml + + String s = SolrConfig.config.get("propTest"); + assertEquals("prefix-proptwo-suffix", s); + + s = SolrConfig.config.get("propTest/@attr1", "default"); + assertEquals("propone-${literal}", s); + + s = SolrConfig.config.get("propTest/@attr2", "default"); + assertEquals("default-from-config", s); + + s = SolrConfig.config.get("propTest[@attr2='default-from-config']", "default"); + assertEquals("prefix-proptwo-suffix", s); + + NodeList nl = (NodeList) SolrConfig.config.evaluate("propTest", XPathConstants.NODESET); + assertEquals(1, nl.getLength()); + assertEquals("prefix-proptwo-suffix", nl.item(0).getTextContent()); + + Node node = SolrConfig.config.getNode("propTest", true); + assertEquals("prefix-proptwo-suffix", node.getTextContent()); + } +} \ No newline at end of file diff --git a/src/test/test-files/solr/conf/bad_solrconfig.xml b/src/test/test-files/solr/conf/bad_solrconfig.xml new file mode 100644 index 00000000000..6f6213ee5d5 --- /dev/null +++ b/src/test/test-files/solr/conf/bad_solrconfig.xml @@ -0,0 +1,29 @@ + + + + + + + + + ${unset.sys.property} + + diff --git a/src/test/test-files/solr/conf/solrconfig.xml b/src/test/test-files/solr/conf/solrconfig.xml index 03da74dafe1..1030b26fd7a 100644 --- a/src/test/test-files/solr/conf/solrconfig.xml +++ b/src/test/test-files/solr/conf/solrconfig.xml @@ -261,6 +261,9 @@ solrconfig.xml scheam.xml admin-extra.html + + prefix-${solr.test.sys.prop2}-suffix