SOLR-7582: Allow auto-commit to be set with system properties in data_driven_schema_configs and enable auto soft-commits for the bin/solr -e cloud example using the Config API.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1681177 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -443,6 +443,10 @@ Other Changes
 * SOLR-7463: Stop forcing MergePolicy's "NoCFSRatio" based on the IWC "useCompoundFile" configuration
   (Tomás Fernández Löbbe)
 
+* SOLR-7582: Allow auto-commit to be set with system properties in data_driven_schema_configs and
+  enable auto soft-commits for the bin/solr -e cloud example using the Config API.
+  (Timothy Potter)
+
 ==================  5.1.0 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release

solr/bin/solr

@@ -1573,6 +1573,11 @@ else
     -confname "$CLOUD_COLLECTION" -confdir "$CLOUD_CONFIG" \
     -configsetsDir "$SOLR_TIP/server/solr/configsets" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr"
 
+  # enable soft-autocommits for the gettingstarted collection
+  echo -e "\nEnabling auto soft-commits with maxTime 3 secs using the Config API"
+  run_tool config -collection "$CLOUD_COLLECTION" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr" \
+    -property updateHandler.autoSoftCommit.maxTime -value 3000
+
   echo -e "\n\nSolrCloud example running, please visit $SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr \n\n"
 fi
 

solr/bin/solr.cmd

@@ -1099,6 +1099,12 @@ goto create_collection
   org.apache.solr.util.SolrCLI create_collection -name !CLOUD_COLLECTION! -shards !CLOUD_NUM_SHARDS! -replicationFactor !CLOUD_REPFACT! ^
   -confdir !CLOUD_CONFIG! -configsetsDir "%SOLR_SERVER_DIR%\solr\configsets" -zkHost %zk_host%
 
+@echo.
+echo Enabling auto soft-commits with maxTime 3 secs using the Config API
+"%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^
+  -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^
+  org.apache.solr.util.SolrCLI config -collection !CLOUD_COLLECTION! -property updateHandler.autoSoftCommit.maxTime -value 3000 -zkHost %zk_host%
+
 echo.
 echo SolrCloud example is running, please visit !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%NODE1_PORT%/solr"
 echo.

solr/core/src/java/org/apache/solr/util/SolrCLI.java

@@ -27,6 +27,7 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -60,12 +61,14 @@ import org.apache.http.util.EntityUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientConfigurer;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
@@ -75,6 +78,8 @@ import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
 import org.noggit.CharArr;
 import org.noggit.JSONParser;
 import org.noggit.JSONWriter;
@@ -253,6 +258,8 @@ public class SolrCLI {
       return new CreateTool();
     else if ("delete".equals(toolType))
       return new DeleteTool();
+    else if ("config".equals(toolType))
+      return new ConfigTool();
 
     // If you add a built-in tool to this class, add it here to avoid
     // classpath scanning
@@ -275,6 +282,7 @@ public class SolrCLI {
     formatter.printHelp("create_core", getToolOptions(new CreateCoreTool()));
     formatter.printHelp("create", getToolOptions(new CreateTool()));
     formatter.printHelp("delete", getToolOptions(new DeleteTool()));
+    formatter.printHelp("config", getToolOptions(new ConfigTool()));
 
     List<Class<Tool>> toolClasses = findToolClassesInPackage("org.apache.solr.util");
     for (Class<Tool> next : toolClasses) {
@@ -1098,6 +1106,35 @@ public class SolrCLI {
             .create("configsetsDir")
   };
 
+  /**
+   * Get the base URL of a live Solr instance from either the solrUrl command-line option from ZooKeeper.
+   */
+  public static String resolveSolrUrl(CommandLine cli) throws Exception {
+    String solrUrl = cli.getOptionValue("solrUrl");
+    if (solrUrl == null) {
+      String zkHost = cli.getOptionValue("zkHost");
+      if (zkHost == null)
+        throw new IllegalStateException("Must provide either the '-solrUrl' or '-zkHost' parameters!");
+
+      LogManager.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+      LogManager.getLogger("org.apache.solr.common.cloud").setLevel(Level.WARN);
+      try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) {
+        cloudSolrClient.connect();
+        Set<String> liveNodes = cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes();
+        if (liveNodes.isEmpty())
+          throw new IllegalStateException("No live nodes found! Cannot determine 'solrUrl' from ZooKeeper: "+zkHost);
+
+        String firstLiveNode = liveNodes.iterator().next();
+        solrUrl = cloudSolrClient.getZkStateReader().getBaseUrlForNodeName(firstLiveNode);
+      }
+    }
+    return solrUrl;
+  }
+
+  /**
+   * Get the ZooKeeper connection string from either the zkHost command-line option or by looking it
+   * up from a running Solr instance based on the solrUrl option.
+   */
   public static String getZkHost(CommandLine cli) throws Exception {
     String zkHost = cli.getOptionValue("zkHost");
     if (zkHost != null)
@@ -1136,6 +1173,18 @@ public class SolrCLI {
     return zkHost;
   }
 
+  public static boolean safeCheckCollectionExists(String url, String collection) {
+    boolean exists = false;
+    try {
+      Map<String,Object> existsCheckResult = getJson(url);
+      List<String> collections = (List<String>) existsCheckResult.get("collections");
+      exists = collections != null && collections.contains(collection);
+    } catch (Exception exc) {
+      // just ignore it since we're only interested in a positive result here
+    }
+    return exists;
+  }
+
   /**
    * Supports create_collection command in the bin/solr script.
    */
@@ -1301,18 +1350,6 @@ public class SolrCLI {
       return 0;
     }
 
-    protected boolean safeCheckCollectionExists(String url, String collection) {
-      boolean exists = false;
-      try {
-        Map<String,Object> existsCheckResult = getJson(url);
-        List<String> collections = (List<String>) existsCheckResult.get("collections");
-        exists = collections != null && collections.contains(collection);
-      } catch (Exception exc) {
-        // just ignore it since we're only interested in a positive result here
-      }
-      return exists;
-    }
-
     protected int optionAsInt(CommandLine cli, String option, int defaultVal) {
       return Integer.parseInt(cli.getOptionValue(option, String.valueOf(defaultVal)));
     }
@@ -1733,4 +1770,115 @@ public class SolrCLI {
     }
 
   } // end DeleteTool class
+
+  /**
+   * Sends a POST to the Config API to perform a specified action.
+   */
+  public static class ConfigTool implements Tool {
+
+    @Override
+    public String getName() {
+      return "config";
+    }
+
+    @SuppressWarnings("static-access")
+    @Override
+    public Option[] getOptions() {
+      return new Option[] {
+          OptionBuilder
+              .withArgName("ACTION")
+              .hasArg()
+              .isRequired(false)
+              .withDescription("Config API action, one of: set-property, unset-property; default is set-property")
+              .create("action"),
+          OptionBuilder
+              .withArgName("PROP")
+              .hasArg()
+              .isRequired(true)
+              .withDescription("Name of the Config API property to apply the action to, such as: updateHandler.autoSoftCommit.maxTime")
+              .create("property"),
+          OptionBuilder
+              .withArgName("VALUE")
+              .hasArg()
+              .isRequired(false)
+              .withDescription("Set the property to this value; accepts JSON objects and strings")
+              .create("value"),
+          OptionBuilder
+              .withArgName("COLL")
+              .hasArg()
+              .isRequired(false)
+              .withDescription("Collection; defaults to gettingstarted")
+              .create("collection"),
+          OptionBuilder
+              .withArgName("HOST")
+              .hasArg()
+              .isRequired(false)
+              .withDescription("Address of the Zookeeper ensemble")
+              .create("zkHost"),
+          OptionBuilder
+              .withArgName("HOST")
+              .hasArg()
+              .isRequired(false)
+              .withDescription("Base Solr URL, which can be used to determine the zkHost if that's not known")
+              .create("solrUrl")
+      };
+    }
+
+    @Override
+    public int runTool(CommandLine cli) throws Exception {
+      String solrUrl = resolveSolrUrl(cli);
+      String action = cli.getOptionValue("action", "set-property");
+      String collection = cli.getOptionValue("collection", "gettingstarted");
+      String property = cli.getOptionValue("property");
+      String value = cli.getOptionValue("value");
+
+      Map<String,Object> jsonObj = new HashMap<>();
+      if (value != null) {
+        Map<String,String> setMap = new HashMap<>();
+        setMap.put(property, value);
+        jsonObj.put(action, setMap);
+      } else {
+        jsonObj.put(action, property);
+      }
+
+      CharArr arr = new CharArr();
+      (new JSONWriter(arr, 0)).write(jsonObj);
+      String jsonBody = arr.toString();
+
+      String updatePath = "/"+collection+"/config";
+
+      System.out.println("\nPOSTing request to Config API: "+solrUrl+updatePath);
+      System.out.println(jsonBody);
+      System.out.println();
+
+      int exitStatus = 0;
+      try (SolrClient solrClient = new HttpSolrClient(solrUrl)) {
+        NamedList<Object> result = postJsonToSolr(solrClient, updatePath, jsonBody);
+        Integer statusCode = (Integer)((NamedList)result.get("responseHeader")).get("status");
+        if (statusCode == 0) {
+          if (value != null) {
+            System.out.println("Successfully "+action+" "+property+" to "+value);
+          } else {
+            System.out.println("Successfully "+action+" "+property);
+          }
+        } else {
+          String errMsg = "Failed to "+action+" property due to:\n"+result;
+          System.err.println("\nERROR: "+errMsg+"\n");
+          exitStatus = 1;
+        }
+      }
+      return exitStatus;
+    }
+
+  } // end ConfigTool class
+
+  public static final String JSON_CONTENT_TYPE = "application/json";
+
+  public static NamedList<Object> postJsonToSolr(SolrClient solrClient, String updatePath, String jsonBody) throws Exception {
+    ContentStreamBase.StringStream contentStream = new ContentStreamBase.StringStream(jsonBody);
+    contentStream.setContentType(JSON_CONTENT_TYPE);
+    ContentStreamUpdateRequest req = new ContentStreamUpdateRequest(updatePath);
+    req.addContentStream(contentStream);
+    return solrClient.request(req);
+  }
 }

solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java

@@ -23,6 +23,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 
@@ -136,8 +137,75 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {
     QueryResponse qr = cloudClient.query(new SolrQuery("*:*"));
     int numFound = (int)qr.getResults().getNumFound();
     assertEquals("*:* found unexpected number of documents", expectedXmlDocCount, numFound);
-                 
+
+    log.info("Updating Config for " + testCollectionName);
+    doTestConfigUpdate(testCollectionName, solrUrl);
+
+    log.info("Running healthcheck for " + testCollectionName);
+    doTestHealthcheck(testCollectionName, cloudClient.getZkHost());
+
+    // verify the delete action works too
+    log.info("Running delete for "+testCollectionName);
+    doTestDeleteAction(testCollectionName, solrUrl);
 
     log.info("testLoadDocsIntoGettingStartedCollection succeeded ... shutting down now!");
   }
+
+  protected void doTestHealthcheck(String testCollectionName, String zkHost) throws Exception {
+    String[] args = new String[]{
+        "healthcheck",
+        "-collection", testCollectionName,
+        "-zkHost", zkHost
+    };
+    SolrCLI.HealthcheckTool tool = new SolrCLI.HealthcheckTool();
+    CommandLine cli =
+        SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
+    assertTrue("Healthcheck action failed!", tool.runTool(cli) == 0);
+  }
+
+  protected void doTestDeleteAction(String testCollectionName, String solrUrl) throws Exception {
+    String[] args = new String[] {
+        "delete",
+        "-name", testCollectionName,
+        "-solrUrl", solrUrl
+    };
+    SolrCLI.DeleteTool tool = new SolrCLI.DeleteTool();
+    CommandLine cli =
+        SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
+    assertTrue("Delete action failed!", tool.runTool(cli) == 0);
+    assertTrue(!SolrCLI.safeCheckCollectionExists(solrUrl, testCollectionName)); // it should not exist anymore
+  }
+
+  /**
+   * Uses the SolrCLI config action to activate soft auto-commits for the getting started collection.
+   */
+  protected void doTestConfigUpdate(String testCollectionName, String solrUrl) throws Exception {
+    if (!solrUrl.endsWith("/"))
+      solrUrl += "/";
+    String configUrl = solrUrl + testCollectionName + "/config";
+
+    Map<String, Object> configJson = SolrCLI.getJson(configUrl);
+    Object maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson);
+    assertNotNull(maxTimeFromConfig);
+    assertEquals(new Long(-1L), maxTimeFromConfig);
+
+    String prop = "updateHandler.autoSoftCommit.maxTime";
+    Long maxTime = new Long(3000L);
+    String[] args = new String[]{
+        "config",
+        "-collection", testCollectionName,
+        "-property", prop,
+        "-value", maxTime.toString(),
+        "-solrUrl", solrUrl
+    };
+    SolrCLI.ConfigTool tool = new SolrCLI.ConfigTool();
+    CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args);
+    log.info("Sending set-property '" + prop + "'=" + maxTime + " to SolrCLI.ConfigTool.");
+    assertTrue("Set config property failed!", tool.runTool(cli) == 0);
+
+    configJson = SolrCLI.getJson(configUrl);
+    maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson);
+    assertNotNull(maxTimeFromConfig);
+    assertEquals(maxTime, maxTimeFromConfig);
+  }
 }

solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml

@@ -360,7 +360,7 @@
          have some sort of hard autoCommit to limit the log size.
       -->
     <autoCommit>
-      <maxTime>15000</maxTime>
+      <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
       <openSearcher>false</openSearcher>
     </autoCommit>
 
@@ -369,11 +369,10 @@
          but does not ensure that data is synced to disk.  This is
          faster and more near-realtime friendly than a hard commit.
       -->
-    <!--
+
-      <autoSoftCommit> 
+    <autoSoftCommit>
-        <maxTime>1000</maxTime> 
+      <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
-      </autoSoftCommit>
+    </autoSoftCommit>
-     -->
 
     <!-- Update Related Event Listeners