From 6b8eafe6256074846a04b719adef5ebc7a654960 Mon Sep 17 00:00:00 2001 From: Timothy Potter Date: Fri, 22 May 2015 17:58:47 +0000 Subject: [PATCH] SOLR-7583: Allow auto-commit to be set with system properties in data_driven_schema_configs and enable auto soft-commits for the bin/solr -e cloud example using the Config API. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1681173 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 4 + solr/bin/solr | 5 + solr/bin/solr.cmd | 6 + .../java/org/apache/solr/util/SolrCLI.java | 172 ++++++++++++++++-- .../solr/cloud/SolrCloudExampleTest.java | 70 ++++++- .../conf/solrconfig.xml | 11 +- 6 files changed, 249 insertions(+), 19 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5980cc2cb0a..495acf64f9f 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -443,6 +443,10 @@ Other Changes * SOLR-7463: Stop forcing MergePolicy's "NoCFSRatio" based on the IWC "useCompoundFile" configuration (Tomás Fernández Löbbe) +* SOLR-7583: Allow auto-commit to be set with system properties in data_driven_schema_configs and + enable auto soft-commits for the bin/solr -e cloud example using the Config API. + (Timothy Potter) + ================== 5.1.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/bin/solr b/solr/bin/solr index 10793a79ddd..b4d9e858e45 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -1573,6 +1573,11 @@ else -confname "$CLOUD_COLLECTION" -confdir "$CLOUD_CONFIG" \ -configsetsDir "$SOLR_TIP/server/solr/configsets" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr" + # enable soft-autocommits for the gettingstarted collection + echo -e "\nEnabling auto soft-commits with maxTime 3 secs using the Config API" + run_tool config -collection "$CLOUD_COLLECTION" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr" \ + -property updateHandler.autoSoftCommit.maxTime -value 3000 + echo -e "\n\nSolrCloud example running, please visit $SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr \n\n" fi diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index fb1e34b67bd..9ed0bd31855 100644 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -1099,6 +1099,12 @@ goto create_collection org.apache.solr.util.SolrCLI create_collection -name !CLOUD_COLLECTION! -shards !CLOUD_NUM_SHARDS! -replicationFactor !CLOUD_REPFACT! ^ -confdir !CLOUD_CONFIG! -configsetsDir "%SOLR_SERVER_DIR%\solr\configsets" -zkHost %zk_host% +@echo. +echo Enabling auto soft-commits with maxTime 3 secs using the Config API +"%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.util.SolrCLI config -collection !CLOUD_COLLECTION! -property updateHandler.autoSoftCommit.maxTime -value 3000 -zkHost %zk_host% + echo. echo SolrCloud example is running, please visit !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%NODE1_PORT%/solr" echo. diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index c5894a6954a..a5da8a0df18 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -27,6 +27,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Enumeration; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -60,12 +61,14 @@ import org.apache.http.util.EntityUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientConfigurer; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -75,6 +78,8 @@ import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.common.util.NamedList; import org.noggit.CharArr; import org.noggit.JSONParser; import org.noggit.JSONWriter; @@ -253,6 +258,8 @@ public class SolrCLI { return new CreateTool(); else if ("delete".equals(toolType)) return new DeleteTool(); + else if ("config".equals(toolType)) + return new ConfigTool(); // If you add a built-in tool to this class, add it here to avoid // classpath scanning @@ -275,6 +282,7 @@ public class SolrCLI { formatter.printHelp("create_core", getToolOptions(new CreateCoreTool())); formatter.printHelp("create", getToolOptions(new CreateTool())); formatter.printHelp("delete", getToolOptions(new DeleteTool())); + formatter.printHelp("config", getToolOptions(new ConfigTool())); List> toolClasses = findToolClassesInPackage("org.apache.solr.util"); for (Class next : toolClasses) { @@ -1098,6 +1106,35 @@ public class SolrCLI { .create("configsetsDir") }; + /** + * Get the base URL of a live Solr instance from either the solrUrl command-line option from ZooKeeper. + */ + public static String resolveSolrUrl(CommandLine cli) throws Exception { + String solrUrl = cli.getOptionValue("solrUrl"); + if (solrUrl == null) { + String zkHost = cli.getOptionValue("zkHost"); + if (zkHost == null) + throw new IllegalStateException("Must provide either the '-solrUrl' or '-zkHost' parameters!"); + + LogManager.getLogger("org.apache.zookeeper").setLevel(Level.ERROR); + LogManager.getLogger("org.apache.solr.common.cloud").setLevel(Level.WARN); + try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + cloudSolrClient.connect(); + Set liveNodes = cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes(); + if (liveNodes.isEmpty()) + throw new IllegalStateException("No live nodes found! Cannot determine 'solrUrl' from ZooKeeper: "+zkHost); + + String firstLiveNode = liveNodes.iterator().next(); + solrUrl = cloudSolrClient.getZkStateReader().getBaseUrlForNodeName(firstLiveNode); + } + } + return solrUrl; + } + + /** + * Get the ZooKeeper connection string from either the zkHost command-line option or by looking it + * up from a running Solr instance based on the solrUrl option. + */ public static String getZkHost(CommandLine cli) throws Exception { String zkHost = cli.getOptionValue("zkHost"); if (zkHost != null) @@ -1136,6 +1173,18 @@ public class SolrCLI { return zkHost; } + public static boolean safeCheckCollectionExists(String url, String collection) { + boolean exists = false; + try { + Map existsCheckResult = getJson(url); + List collections = (List) existsCheckResult.get("collections"); + exists = collections != null && collections.contains(collection); + } catch (Exception exc) { + // just ignore it since we're only interested in a positive result here + } + return exists; + } + /** * Supports create_collection command in the bin/solr script. */ @@ -1301,18 +1350,6 @@ public class SolrCLI { return 0; } - protected boolean safeCheckCollectionExists(String url, String collection) { - boolean exists = false; - try { - Map existsCheckResult = getJson(url); - List collections = (List) existsCheckResult.get("collections"); - exists = collections != null && collections.contains(collection); - } catch (Exception exc) { - // just ignore it since we're only interested in a positive result here - } - return exists; - } - protected int optionAsInt(CommandLine cli, String option, int defaultVal) { return Integer.parseInt(cli.getOptionValue(option, String.valueOf(defaultVal))); } @@ -1733,4 +1770,115 @@ public class SolrCLI { } } // end DeleteTool class + + /** + * Sends a POST to the Config API to perform a specified action. + */ + public static class ConfigTool implements Tool { + + @Override + public String getName() { + return "config"; + } + + @SuppressWarnings("static-access") + @Override + public Option[] getOptions() { + return new Option[] { + OptionBuilder + .withArgName("ACTION") + .hasArg() + .isRequired(false) + .withDescription("Config API action, one of: set-property, unset-property; default is set-property") + .create("action"), + OptionBuilder + .withArgName("PROP") + .hasArg() + .isRequired(true) + .withDescription("Name of the Config API property to apply the action to, such as: updateHandler.autoSoftCommit.maxTime") + .create("property"), + OptionBuilder + .withArgName("VALUE") + .hasArg() + .isRequired(false) + .withDescription("Set the property to this value; accepts JSON objects and strings") + .create("value"), + OptionBuilder + .withArgName("COLL") + .hasArg() + .isRequired(false) + .withDescription("Collection; defaults to gettingstarted") + .create("collection"), + OptionBuilder + .withArgName("HOST") + .hasArg() + .isRequired(false) + .withDescription("Address of the Zookeeper ensemble") + .create("zkHost"), + OptionBuilder + .withArgName("HOST") + .hasArg() + .isRequired(false) + .withDescription("Base Solr URL, which can be used to determine the zkHost if that's not known") + .create("solrUrl") + }; + } + + @Override + public int runTool(CommandLine cli) throws Exception { + String solrUrl = resolveSolrUrl(cli); + String action = cli.getOptionValue("action", "set-property"); + String collection = cli.getOptionValue("collection", "gettingstarted"); + String property = cli.getOptionValue("property"); + String value = cli.getOptionValue("value"); + + Map jsonObj = new HashMap<>(); + if (value != null) { + Map setMap = new HashMap<>(); + setMap.put(property, value); + jsonObj.put(action, setMap); + } else { + jsonObj.put(action, property); + } + + CharArr arr = new CharArr(); + (new JSONWriter(arr, 0)).write(jsonObj); + String jsonBody = arr.toString(); + + String updatePath = "/"+collection+"/config"; + + System.out.println("\nPOSTing request to Config API: "+solrUrl+updatePath); + System.out.println(jsonBody); + System.out.println(); + + int exitStatus = 0; + try (SolrClient solrClient = new HttpSolrClient(solrUrl)) { + NamedList result = postJsonToSolr(solrClient, updatePath, jsonBody); + Integer statusCode = (Integer)((NamedList)result.get("responseHeader")).get("status"); + if (statusCode == 0) { + if (value != null) { + System.out.println("Successfully "+action+" "+property+" to "+value); + } else { + System.out.println("Successfully "+action+" "+property); + } + } else { + String errMsg = "Failed to "+action+" property due to:\n"+result; + System.err.println("\nERROR: "+errMsg+"\n"); + exitStatus = 1; + } + } + return exitStatus; + } + + } // end ConfigTool class + + public static final String JSON_CONTENT_TYPE = "application/json"; + + public static NamedList postJsonToSolr(SolrClient solrClient, String updatePath, String jsonBody) throws Exception { + ContentStreamBase.StringStream contentStream = new ContentStreamBase.StringStream(jsonBody); + contentStream.setContentType(JSON_CONTENT_TYPE); + ContentStreamUpdateRequest req = new ContentStreamUpdateRequest(updatePath); + req.addContentStream(contentStream); + return solrClient.request(req); + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java index ecca28c8a77..a7155cf18ce 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Set; @@ -136,8 +137,75 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { QueryResponse qr = cloudClient.query(new SolrQuery("*:*")); int numFound = (int)qr.getResults().getNumFound(); assertEquals("*:* found unexpected number of documents", expectedXmlDocCount, numFound); - + + log.info("Updating Config for " + testCollectionName); + doTestConfigUpdate(testCollectionName, solrUrl); + + log.info("Running healthcheck for " + testCollectionName); + doTestHealthcheck(testCollectionName, cloudClient.getZkHost()); + + // verify the delete action works too + log.info("Running delete for "+testCollectionName); + doTestDeleteAction(testCollectionName, solrUrl); log.info("testLoadDocsIntoGettingStartedCollection succeeded ... shutting down now!"); } + + protected void doTestHealthcheck(String testCollectionName, String zkHost) throws Exception { + String[] args = new String[]{ + "healthcheck", + "-collection", testCollectionName, + "-zkHost", zkHost + }; + SolrCLI.HealthcheckTool tool = new SolrCLI.HealthcheckTool(); + CommandLine cli = + SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + assertTrue("Healthcheck action failed!", tool.runTool(cli) == 0); + } + + protected void doTestDeleteAction(String testCollectionName, String solrUrl) throws Exception { + String[] args = new String[] { + "delete", + "-name", testCollectionName, + "-solrUrl", solrUrl + }; + SolrCLI.DeleteTool tool = new SolrCLI.DeleteTool(); + CommandLine cli = + SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + assertTrue("Delete action failed!", tool.runTool(cli) == 0); + assertTrue(!SolrCLI.safeCheckCollectionExists(solrUrl, testCollectionName)); // it should not exist anymore + } + + /** + * Uses the SolrCLI config action to activate soft auto-commits for the getting started collection. + */ + protected void doTestConfigUpdate(String testCollectionName, String solrUrl) throws Exception { + if (!solrUrl.endsWith("/")) + solrUrl += "/"; + String configUrl = solrUrl + testCollectionName + "/config"; + + Map configJson = SolrCLI.getJson(configUrl); + Object maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson); + assertNotNull(maxTimeFromConfig); + assertEquals(new Long(-1L), maxTimeFromConfig); + + String prop = "updateHandler.autoSoftCommit.maxTime"; + Long maxTime = new Long(3000L); + String[] args = new String[]{ + "config", + "-collection", testCollectionName, + "-property", prop, + "-value", maxTime.toString(), + "-solrUrl", solrUrl + }; + SolrCLI.ConfigTool tool = new SolrCLI.ConfigTool(); + CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + log.info("Sending set-property '" + prop + "'=" + maxTime + " to SolrCLI.ConfigTool."); + assertTrue("Set config property failed!", tool.runTool(cli) == 0); + + configJson = SolrCLI.getJson(configUrl); + maxTimeFromConfig = SolrCLI.atPath("/config/updateHandler/autoSoftCommit/maxTime", configJson); + assertNotNull(maxTimeFromConfig); + assertEquals(maxTime, maxTimeFromConfig); + } } diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml index 7bf5df4d5d4..21392b60462 100644 --- a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml @@ -360,7 +360,7 @@ have some sort of hard autoCommit to limit the log size. --> - 15000 + ${solr.autoCommit.maxTime:15000} false @@ -369,11 +369,10 @@ but does not ensure that data is synced to disk. This is faster and more near-realtime friendly than a hard commit. --> - + + + ${solr.autoSoftCommit.maxTime:-1} +