SOLR-79: Add system property ${<sys.prop>[:<default>]} substitution for

configuration files loaded, including schema.xml and solrconfig.xml.
    (Erik Hatcher with inspiration from Andrew Saar)



git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@508485 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -82,6 +82,10 @@ New Features
 
 10. SOLR-116: IndexInfoRequestHandler added. (Erik Hatcher)
 
+11. SOLR-79: Add system property ${<sys.prop>[:<default>]} substitution for
+    configuration files loaded, including schema.xml and solrconfig.xml.
+    (Erik Hatcher with inspiration from Andrew Saar)
+
 Changes in runtime behavior
  1. Highlighting using DisMax will only pick up terms from the main 
     user query, not boost or filter queries (klaas).

build.xml

@@ -282,6 +282,9 @@
       <syspropertyset>
         <propertyref prefix="solr" />
       </syspropertyset>
+      <!-- solr.test.sys.prop1/2 used by TestConfig -->
+      <sysproperty key="solr.test.sys.prop1" value="propone"/>
+      <sysproperty key="solr.test.sys.prop2" value="proptwo"/>
       <classpath refid="test.run.classpath"/>
       <formatter type="xml"/>
       <batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">

src/java/org/apache/solr/core/Config.java

@@ -59,6 +59,8 @@ public class Config {
 
     javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
     doc = builder.parse(is);
+
+    DOMUtil.substituteSystemProperties(doc);
   }
 
   public Document getDocument() {

src/java/org/apache/solr/util/AbstractSolrTestCase.java

@@ -110,7 +110,7 @@ public abstract class AbstractSolrTestCase extends TestCase {
    * is set.
    */
   public void tearDown() throws Exception {
-    h.close();
+    if (h != null) { h.close(); }
     String skip = System.getProperty("solr.test.leavedatadir");
     if (null != skip && 0 != skip.trim().length()) {
       System.err.println("NOTE: per solr.test.leavedatadir, dataDir will not be removed: " + dataDir.getAbsolutePath());

src/java/org/apache/solr/util/DOMUtil.java

@@ -21,11 +21,14 @@ import org.w3c.dom.Element;
 import org.w3c.dom.NamedNodeMap;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
+import org.apache.solr.core.SolrException;
 
 import java.util.Map;
 import java.util.HashMap;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.Iterator;
 
 /**
  * @author yonik
@@ -232,4 +235,131 @@ public class DOMUtil {
 
     }
   }
+
+  /**
+   * Replaces ${system.property[:default value]} references in all attributes
+   * and text nodes of supplied node.  If the system property is not defined, an empty string
+   * is substituted or the default value if provided.
+   *
+   * @param node DOM node to walk for substitutions
+   */
+  public static void substituteSystemProperties(Node node) {
+    // loop through child nodes
+    Node child;
+    Node next = node.getFirstChild();
+    while ((child = next) != null) {
+
+      // set next before we change anything
+      next = child.getNextSibling();
+
+      // handle child by node type
+      if (child.getNodeType() == Node.TEXT_NODE) {
+        child.setNodeValue(substituteSystemProperty(child.getNodeValue()));
+      } else if (child.getNodeType() == Node.ELEMENT_NODE) {
+        // handle child elements with recursive call
+        NamedNodeMap attributes = child.getAttributes();
+        for (int i = 0; i < attributes.getLength(); i++) {
+          Node attribute = attributes.item(i);
+          attribute.setNodeValue(substituteSystemProperty(attribute.getNodeValue()));
+        }
+        substituteSystemProperties(child);
+      }
+    }
+  }
+
+  /*
+   * This method borrowed from Ant's PropertyHelper.replaceProperties:
+   *   http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/ant/PropertyHelper.java
+   */
+  private static String substituteSystemProperty(String value) {
+    if (value == null || value.indexOf('$') == -1) {
+      return value;
+    }
+
+    List<String> fragments = new ArrayList<String>();
+    List<String> propertyRefs = new ArrayList<String>();
+    parsePropertyString(value, fragments, propertyRefs);
+
+    StringBuffer sb = new StringBuffer();
+    Iterator<String> i = fragments.iterator();
+    Iterator<String> j = propertyRefs.iterator();
+
+    while (i.hasNext()) {
+      String fragment = i.next();
+      if (fragment == null) {
+        String propertyName = j.next();
+        String defaultValue = null;
+        int colon_index = propertyName.indexOf(':');
+        if (colon_index > -1) {
+          defaultValue = propertyName.substring(colon_index + 1);
+          propertyName = propertyName.substring(0,colon_index);
+        }
+        fragment = System.getProperty(propertyName,defaultValue);
+        if (fragment == null) {
+          throw new SolrException(500, "No system property or default value specified for " + propertyName);
+        }
+      }
+      sb.append(fragment);
+    }
+    return sb.toString();
+  }
+
+  /*
+   * This method borrowed from Ant's PropertyHelper.parsePropertyStringDefault:
+   *   http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/ant/PropertyHelper.java
+   */
+  private static void parsePropertyString(String value, List<String> fragments, List<String> propertyRefs) {
+      int prev = 0;
+      int pos;
+      //search for the next instance of $ from the 'prev' position
+      while ((pos = value.indexOf("$", prev)) >= 0) {
+
+          //if there was any text before this, add it as a fragment
+          //TODO, this check could be modified to go if pos>prev;
+          //seems like this current version could stick empty strings
+          //into the list
+          if (pos > 0) {
+              fragments.add(value.substring(prev, pos));
+          }
+          //if we are at the end of the string, we tack on a $
+          //then move past it
+          if (pos == (value.length() - 1)) {
+              fragments.add("$");
+              prev = pos + 1;
+          } else if (value.charAt(pos + 1) != '{') {
+              //peek ahead to see if the next char is a property or not
+              //not a property: insert the char as a literal
+              /*
+              fragments.addElement(value.substring(pos + 1, pos + 2));
+              prev = pos + 2;
+              */
+              if (value.charAt(pos + 1) == '$') {
+                  //backwards compatibility two $ map to one mode
+                  fragments.add("$");
+                  prev = pos + 2;
+              } else {
+                  //new behaviour: $X maps to $X for all values of X!='$'
+                  fragments.add(value.substring(pos, pos + 2));
+                  prev = pos + 2;
+              }
+
+          } else {
+              //property found, extract its name or bail on a typo
+              int endName = value.indexOf('}', pos);
+              if (endName < 0) {
+                throw new RuntimeException("Syntax error in property: " + value);
+              }
+              String propertyName = value.substring(pos + 2, endName);
+              fragments.add(null);
+              propertyRefs.add(propertyName);
+              prev = endName + 1;
+          }
+      }
+      //no more $ signs found
+      //if there is any tail to the string, append it
+      if (prev < value.length()) {
+          fragments.add(value.substring(prev));
+      }
+  }
+
 }

src/test/org/apache/solr/core/TestBadConfig.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.TestHarness;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import javax.xml.xpath.XPathConstants;
+import java.io.File;
+
+public class TestBadConfig extends AbstractSolrTestCase {
+
+  public String getSchemaFile() { return "schema.xml"; }
+  public String getSolrConfigFile() { return "bad_solrconfig.xml"; }
+
+  public void setUp() throws Exception {
+
+    dataDir = new File(System.getProperty("java.io.tmpdir")
+                       + System.getProperty("file.separator")
+                       + getClass().getName() + "-" + getName() + "-"
+                       + System.currentTimeMillis());
+    dataDir.mkdirs();
+    try {
+      h = new TestHarness(dataDir.getAbsolutePath(),
+                          getSolrConfigFile(),
+                          getSchemaFile());
+      fail("Exception should have been thrown");
+    } catch (Exception e) {
+      assertTrue(e.getMessage().contains("unset.sys.property"));
+    }
+  }
+    
+
+  public void testNothing() {
+    // Empty test case as the real test is that the initialization of the TestHarness fails
+    assertTrue(true);
+  }
+}

src/test/org/apache/solr/core/TestConfig.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import javax.xml.xpath.XPathConstants;
+
+public class TestConfig extends AbstractSolrTestCase {
+
+  public String getSchemaFile() { return "schema.xml"; }
+  public String getSolrConfigFile() { return "solrconfig.xml"; }
+
+  public void testJavaProperty() {
+    // property values defined in build.xml
+
+    String s = SolrConfig.config.get("propTest");
+    assertEquals("prefix-proptwo-suffix", s);
+
+    s = SolrConfig.config.get("propTest/@attr1", "default");
+    assertEquals("propone-${literal}", s);
+
+    s = SolrConfig.config.get("propTest/@attr2", "default");
+    assertEquals("default-from-config", s);
+
+    s = SolrConfig.config.get("propTest[@attr2='default-from-config']", "default");
+    assertEquals("prefix-proptwo-suffix", s);
+
+    NodeList nl = (NodeList) SolrConfig.config.evaluate("propTest", XPathConstants.NODESET);
+    assertEquals(1, nl.getLength());
+    assertEquals("prefix-proptwo-suffix", nl.item(0).getTextContent());
+
+    Node node = SolrConfig.config.getNode("propTest", true);
+    assertEquals("prefix-proptwo-suffix", node.getTextContent());
+  }
+}

src/test/test-files/solr/conf/bad_solrconfig.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
+     $Source$
+     $Name$
+  -->
+
+<config>
+  <indexDefaults>
+    <useCompoundFile>${unset.sys.property}</useCompoundFile>
+  </indexDefaults>
+</config>

src/test/test-files/solr/conf/solrconfig.xml

@@ -261,6 +261,9 @@
     <gettableFiles>solrconfig.xml scheam.xml admin-extra.html</gettableFiles>
   </admin>
 
+  <!-- test getting system property -->
+  <propTest attr1="${solr.test.sys.prop1}-$${literal}"
+            attr2="${non.existent.sys.prop:default-from-config}">prefix-${solr.test.sys.prop2}-suffix</propTest>
 
 
 </config>