From 6edbda74291fa9fabb5e6cdc1141e799b738f5ef Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Thu, 14 Nov 2019 18:21:35 +0530 Subject: [PATCH] SOLR-13662: Package manager (CLI) --- lucene/ivy-versions.properties | 2 + solr/CHANGES.txt | 14 +- solr/bin/solr | 55 +++ solr/bin/solr.cmd | 10 + solr/core/ivy.xml | 3 + .../DefaultPackageRepository.java | 116 +++++ .../solr/packagemanager/PackageManager.java | 416 ++++++++++++++++++ .../packagemanager/PackageRepository.java | 53 +++ .../solr/packagemanager/PackageUtils.java | 251 +++++++++++ .../packagemanager/RepositoryManager.java | 317 +++++++++++++ .../solr/packagemanager/SolrPackage.java | 140 ++++++ .../packagemanager/SolrPackageInstance.java | 66 +++ .../solr/packagemanager/package-info.java | 21 + .../java/org/apache/solr/pkg/PackageAPI.java | 8 + .../org/apache/solr/util/PackageTool.java | 293 ++++++++++++ .../java/org/apache/solr/util/SolrCLI.java | 4 + ...question-answer-repository-private-key.pem | 9 + .../question-answer-repository/publickey.der | Bin 0 -> 94 bytes .../question-answer-request-handler-1.0.jar | Bin 0 -> 5579 bytes .../question-answer-request-handler-1.1.jar | Bin 0 -> 6152 bytes .../repository.json | 56 +++ .../solr/cloud/PackageManagerCLITest.java | 201 +++++++++ solr/licenses/java-semver-0.9.0.jar.sha1 | 1 + solr/licenses/java-semver-LICENSE-MIT.txt | 21 + .../client/solrj/request/beans/Package.java | 4 + 25 files changed, 2057 insertions(+), 4 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/package-info.java create mode 100644 solr/core/src/java/org/apache/solr/util/PackageTool.java create mode 100644 solr/core/src/test-files/solr/question-answer-repository-private-key.pem create mode 100644 solr/core/src/test-files/solr/question-answer-repository/publickey.der create mode 100644 solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar create mode 100644 solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar create mode 100644 solr/core/src/test-files/solr/question-answer-repository/repository.json create mode 100644 solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java create mode 100644 solr/licenses/java-semver-0.9.0.jar.sha1 create mode 100644 solr/licenses/java-semver-LICENSE-MIT.txt diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 4d3b6c0caac..a1f6db3689d 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -26,6 +26,8 @@ com.fasterxml.jackson.core.version = 2.9.9 /com.github.ben-manes.caffeine/caffeine = 2.8.0 /com.github.virtuald/curvesapi = 1.04 +/com.github.zafarkhaja/java-semver = 0.9.0 + /com.google.guava/guava = 25.1-jre /com.google.protobuf/protobuf-java = 3.6.1 /com.google.re2j/re2j = 1.2 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ce03d36326b..647eb809ed7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -39,10 +39,16 @@ New Features * SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya) -* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store - the configuration, APIs to read/edit them and isolated classloaders to load the classes from - hose packages if the 'class' attribute is prefixed with `:` (noble, Ishan Chattopadhyaya) - +* SOLR-13822: A Package management system with the following features: + (a) A packages.json in ZK to store the configuration, + (b) APIs to read/edit them, and + (c) Isolated classloaders to load the classes from those packages when the 'class' + attribute is prefixed with ':' + (noble, Ishan Chattopadhyaya) + +* SOLR-13662: A CLI based Package Manager ("bin/solr package help" for more details). + (Ishan Chattopadhyaya, noble, David Smiley, Jan Hoydahl) + * SOLR-10786: Add DBSCAN clustering Streaming Evaluator (Joel Bernstein) Improvements diff --git a/solr/bin/solr b/solr/bin/solr index 596242fac4b..4b6236537bf 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -764,6 +764,56 @@ function get_info() { return $CODE } # end get_info +function run_package() { + runningSolrUrl="" + + numSolrs=`find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | wc -l | tr -d ' '` + if [ "$numSolrs" != "0" ]; then + #echo -e "\nFound $numSolrs Solr nodes: " + while read PIDF + do + ID=`cat "$PIDF"` + port=`jetty_port "$ID"` + if [ "$port" != "" ]; then + #echo -e "\nSolr process $ID running on port $port" + runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr" + break + CODE=$? + echo "" + else + echo -e "\nSolr process $ID from $PIDF not found." + CODE=1 + fi + done < <(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f) + else + # no pid files but check using ps just to be sure + numSolrs=`ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//'` + if [ "$numSolrs" != "0" ]; then + echo -e "\nFound $numSolrs Solr nodes: " + PROCESSES=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | awk '{print $2}' | sort -r) + for ID in $PROCESSES + do + port=`jetty_port "$ID"` + if [ "$port" != "" ]; then + echo "" + echo "Solr process $ID running on port $port" + runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr" + break + CODE=$? + echo "" + fi + done + else + echo -e "\nNo Solr nodes are running.\n" + exit 1 + CODE=3 + fi + fi + + run_tool package -solrUrl "$runningSolrUrl" $@ + #exit $? +} + # tries to gracefully stop Solr using the Jetty # stop command and if that fails, then uses kill -9 function stop_solr() { @@ -1359,6 +1409,11 @@ if [[ "$SCRIPT_CMD" == "export" ]]; then exit $? fi +if [[ "$SCRIPT_CMD" == "package" ]]; then + run_package $@ + exit $? +fi + if [[ "$SCRIPT_CMD" == "auth" ]]; then VERBOSE="" diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 221b514f1e7..792e621a05e 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -215,6 +215,7 @@ IF "%1"=="-version" goto get_version IF "%1"=="assert" goto run_assert IF "%1"=="autoscaling" goto run_autoscaling IF "%1"=="export" goto run_export +IF "%1"=="package" goto run_package REM Only allow the command to be the first argument, assume start if not supplied IF "%1"=="start" goto set_script_cmd @@ -1422,6 +1423,15 @@ goto done: org.apache.solr.util.SolrCLI %* goto done: +:run_package +REM TODO: Compute the running Solr URL and populate it as a parameter (as has been done for the shell script) +REM Without that, users will have to supply -solrUrl parameter in every request. Life can be so hard for Windows users! +"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.util.SolrCLI %* +goto done: + :parse_config_args IF [%1]==[] goto run_config IF "%1"=="-z" goto set_config_zk diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 9fba66347ac..f7843a1eff8 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -138,6 +138,9 @@ + + + diff --git a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java new file mode 100644 index 00000000000..dee55a5047c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +/** + * This is a serializable bean (for the JSON that is stored in /repository.json) representing a repository of Solr packages. + * Supports standard repositories based on a webservice. + */ +public class DefaultPackageRepository extends PackageRepository { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public DefaultPackageRepository() { // this is needed for deserialization from JSON + + } + + public DefaultPackageRepository(String repositoryName, String repositoryURL) { + this.name = repositoryName; + this.repositoryURL = repositoryURL; + } + + @Override + public void refresh() { + packages = null; + } + + @JsonIgnore + private Map packages; + + @Override + public Map getPackages() { + if (packages == null) { + initPackages(); + } + + return packages; + } + + @Override + public SolrPackage getPackage(String packageName) { + return getPackages().get(packageName); + } + + @Override + public boolean hasPackage(String packageName) { + return getPackages().containsKey(packageName); + } + + @Override + public Path download(String artifactName) throws SolrException, IOException { + Path tmpDirectory = Files.createTempDirectory("solr-packages"); + tmpDirectory.toFile().deleteOnExit(); + URL url = new URL(new URL(repositoryURL), artifactName); + String fileName = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); + Path destination = tmpDirectory.resolve(fileName); + + switch (url.getProtocol()) { + case "http": + case "https": + case "ftp": + FileUtils.copyURLToFile(url, destination.toFile()); + break; + default: + throw new SolrException(ErrorCode.BAD_REQUEST, "URL protocol " + url.getProtocol() + " not supported"); + } + + return destination; + } + + private void initPackages() { + try (CloseableHttpClient client = HttpClientBuilder.create().build()) { + SolrPackage[] items = PackageUtils.getJson(client, repositoryURL + "/repository.json", SolrPackage[].class); + + packages = new HashMap<>(items.length); + for (SolrPackage pkg : items) { + pkg.setRepository(name); + packages.put(pkg.name, pkg); + } + } catch (IOException ex) { + throw new SolrException(ErrorCode.INVALID_STATE, ex); + } + log.debug("Found {} packages in repository '{}'", packages.size(), name); + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java new file mode 100644 index 00000000000..31471e442e7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import static org.apache.solr.packagemanager.PackageUtils.getMapper; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.common.NavigableObject; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.Utils; +import org.apache.solr.packagemanager.SolrPackage.Command; +import org.apache.solr.packagemanager.SolrPackage.Manifest; +import org.apache.solr.packagemanager.SolrPackage.Plugin; +import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.util.SolrCLI; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Strings; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; + +/** + * Handles most of the management of packages that are already installed in Solr. + */ +public class PackageManager implements Closeable { + + final String solrBaseUrl; + final HttpSolrClient solrClient; + final SolrZkClient zkClient; + + private Map> packages = null; + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + + public PackageManager(HttpSolrClient solrClient, String solrBaseUrl, String zkHost) { + this.solrBaseUrl = solrBaseUrl; + this.solrClient = solrClient; + this.zkClient = new SolrZkClient(zkHost, 30000); + log.info("Done initializing a zkClient instance..."); + } + + @Override + public void close() throws IOException { + if (zkClient != null) { + zkClient.close(); + } + } + + public List fetchInstalledPackageInstances() throws SolrException { + log.info("Getting packages from packages.json..."); + List ret = new ArrayList(); + packages = new HashMap>(); + try { + Map packagesZnodeMap = null; + + if (zkClient.exists("/packages.json", true) == true) { + packagesZnodeMap = (Map)getMapper().readValue( + new String(zkClient.getData("/packages.json", null, null, true), "UTF-8"), Map.class).get("packages"); + for (Object packageName: packagesZnodeMap.keySet()) { + List pkg = (List)packagesZnodeMap.get(packageName); + for (Map pkgVersion: (List)pkg) { + Manifest manifest = PackageUtils.fetchManifest(solrClient, solrBaseUrl, pkgVersion.get("manifest").toString(), pkgVersion.get("manifestSHA512").toString()); + List solrplugins = manifest.plugins; + SolrPackageInstance pkgInstance = new SolrPackageInstance(packageName.toString(), null, + pkgVersion.get("version").toString(), manifest, solrplugins, manifest.parameterDefaults); + List list = packages.containsKey(packageName)? packages.get(packageName): new ArrayList(); + list.add(pkgInstance); + packages.put(packageName.toString(), list); + ret.add(pkgInstance); + } + } + } + } catch (Exception e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + log.info("Got packages: "+ret); + return ret; + } + + public Map getPackagesDeployed(String collection) { + Map packages = null; + try { + NavigableObject result = (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), + solrBaseUrl+"/api/collections/"+collection+"/config/params/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER); + packages = (Map) result._get("/response/params/PKG_VERSIONS", Collections.emptyMap()); + } catch (PathNotFoundException ex) { + // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager. + } + if (packages == null) return Collections.emptyMap(); + Map ret = new HashMap(); + for (String packageName: packages.keySet()) { + if (Strings.isNullOrEmpty(packageName) == false) { // There can be an empty key, storing the version here + ret.put(packageName, getPackageInstance(packageName, packages.get(packageName))); + } + } + return ret; + } + + private boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, boolean noprompt, + List collections, String overrides[]) { + for (String collection: collections) { + + SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageInstance.name); + if (packageInstance.equals(deployedPackage)) { + if (!pegToLatest) { + PackageUtils.printRed("Package " + packageInstance + " already deployed on "+collection); + continue; + } + } else { + if (deployedPackage != null && !isUpdate) { + PackageUtils.printRed("Package " + deployedPackage + " already deployed on "+collection+". To update to "+packageInstance+", pass --update parameter."); + continue; + } + } + + Map collectionParameterOverrides = getCollectionParameterOverrides(packageInstance, isUpdate, overrides, collection); + + // Get package params + try { + boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/abc/config/params/packages", Map.class) + .getOrDefault("response", Collections.emptyMap())).containsKey("params"); + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + getMapper().writeValueAsString(Collections.singletonMap(packageParamsExist? "update": "set", + Collections.singletonMap("packages", Collections.singletonMap(packageInstance.name, collectionParameterOverrides))))); + } catch (Exception e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + + // Set the package version in the collection's parameters + try { + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + + // If updating, refresh the package version for this to take effect + if (isUpdate || pegToLatest) { + try { + SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageInstance.name + "\"}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } + + // If it is a fresh deploy on a collection, run setup commands all the plugins in the package + if (!isUpdate) { + Map systemParams = PackageUtils.map("collection", collection, "package-name", packageInstance.name, "package-version", packageInstance.version); + + for (Plugin plugin: packageInstance.plugins) { + Command cmd = plugin.setupCommand; + if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) { + if ("POST".equalsIgnoreCase(cmd.method)) { + try { + String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), packageInstance.parameterDefaults, collectionParameterOverrides, systemParams); + String path = PackageUtils.resolve(cmd.path, packageInstance.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Executing " + payload + " for path:" + path); + boolean shouldExecute = true; + if (!noprompt) { // show a prompt asking user to execute the setup command for the plugin + PackageUtils.print(PackageUtils.YELLOW, "Execute this command (y/n): "); + String userInput = new Scanner(System.in, "UTF-8").next(); + if (!"yes".equalsIgnoreCase(userInput) && !"y".equalsIgnoreCase(userInput)) { + shouldExecute = false; + PackageUtils.printRed("Skipping setup command for deploying (deployment verification may fail)." + + " Please run this step manually or refer to package documentation."); + } + } + if (shouldExecute) { + SolrCLI.postJsonToSolr(solrClient, path, payload); + } + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for setup commands"); + } + } else { + PackageUtils.printRed("There is no setup command to execute for plugin: " + plugin.name); + } + } + } + + // Set the package version in the collection's parameters + try { + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } + + // Verify that package was successfully deployed + boolean success = verify(packageInstance, collections); + if (success) { + PackageUtils.printGreen("Deployed and verified package: " + packageInstance.name + ", version: " + packageInstance.version); + } + return success; + } + + private Map getCollectionParameterOverrides(SolrPackageInstance packageInstance, boolean isUpdate, + String[] overrides, String collection) { + Map collectionParameterOverrides = isUpdate? getPackageParams(packageInstance.name, collection): new HashMap(); + if (overrides != null) { + for (String override: overrides) { + collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]); + } + } + return collectionParameterOverrides; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + Map getPackageParams(String packageName, String collection) { + try { + return (Map)((Map)((Map)((Map) + PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/packages", Map.class) + .get("response")) + .get("params")) + .get("packages")).get(packageName); + } catch (Exception ex) { + // This should be because there are no parameters. Be tolerant here. + return Collections.emptyMap(); + } + } + + /** + * Given a package and list of collections, verify if the package is installed + * in those collections. It uses the verify command of every plugin in the package (if defined). + */ + public boolean verify(SolrPackageInstance pkg, List collections) { + boolean success = true; + for (Plugin plugin: pkg.plugins) { + PackageUtils.printGreen(plugin.verifyCommand); + for (String collection: collections) { + Map collectionParameterOverrides = getPackageParams(pkg.name, collection); + Command cmd = plugin.verifyCommand; + + Map systemParams = PackageUtils.map("collection", collection, "package-name", pkg.name, "package-version", pkg.version); + String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Executing " + url + " for collection:" + collection); + + if ("GET".equalsIgnoreCase(cmd.method)) { + String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url); + PackageUtils.printGreen(response); + String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration()) + .read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams)); + String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue); + if (!expectedValue.equals(actualValue)) { + PackageUtils.printRed("Failed to deploy plugin: " + plugin.name); + success = false; + } + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for setup commands"); + } + } + } + return success; + } + + /** + * Get the installed instance of a specific version of a package. If version is null, PackageUtils.LATEST or PackagePluginHolder.LATEST, + * then it returns the highest version available in the system for the package. + */ + public SolrPackageInstance getPackageInstance(String packageName, String version) { + fetchInstalledPackageInstances(); + List versions = packages.get(packageName); + SolrPackageInstance latest = null; + if (versions != null && !versions.isEmpty()) { + latest = versions.get(0); + for (int i=0; i collectionParameterOverrides = getPackageParams(packageName, collection); + + // Run the uninstall command for all plugins + Map systemParams = PackageUtils.map("collection", collection, "package-name", deployedPackage.name, "package-version", deployedPackage.version); + + for (Plugin plugin: deployedPackage.plugins) { + Command cmd = plugin.uninstallCommand; + if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) { + if ("POST".equalsIgnoreCase(cmd.method)) { + try { + String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams); + String path = PackageUtils.resolve(cmd.path, deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Executing " + payload + " for path:" + path); + SolrCLI.postJsonToSolr(solrClient, path, payload); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for uninstall commands"); + } + } else { + PackageUtils.printRed("There is no uninstall command to execute for plugin: " + plugin.name); + } + } + + // Set the package version in the collection's parameters + try { + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", "{set: {PKG_VERSIONS: {"+packageName+": null}}}"); + SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageName + "\"}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + + // TODO: Also better to remove the package parameters + } + } + + /** + * Given a package, return a map of collections where this package is + * installed to the installed version (which can be {@link PackagePluginHolder#LATEST}) + */ + public Map getDeployedCollections(String packageName) { + List allCollections; + try { + allCollections = zkClient.getChildren("/collections", null, true); + } catch (KeeperException | InterruptedException e) { + throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e); + } + Map deployed = new HashMap(); + for (String collection: allCollections) { + // Check package version installed + String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/PKG_VERSIONS?omitHeader=true"); + String version = null; + try { + version = JsonPath.parse(paramsJson, PackageUtils.jsonPathConfiguration()) + .read("$['response'].['params'].['PKG_VERSIONS'].['"+packageName+"'])"); + } catch (PathNotFoundException ex) { + // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager. + } + if (version != null) { + deployed.put(collection, version); + } + } + return deployed; + } + +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java new file mode 100644 index 00000000000..43b4ada23a1 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.packagemanager; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; + +/** + * Abstract class for a repository, holding {@link SolrPackage} items. + */ +public abstract class PackageRepository { + + @JsonProperty("name") + public String name = null; + + @JsonProperty("url") + public String repositoryURL = null; + + public abstract void refresh(); + + /** + * Returns a map of package name to {@link SolrPackage}s. + */ + public abstract Map getPackages(); + + public abstract SolrPackage getPackage(String packageName); + + public abstract boolean hasPackage(String packageName); + + /** + * Provides a method to download an artifact from this repository. + */ + public abstract Path download(String artifactName) throws SolrException, IOException; + +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java new file mode 100644 index 00000000000..0336c473299 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.packagemanager; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.response.V2Response; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.BlobRepository; +import org.apache.solr.packagemanager.SolrPackage.Manifest; +import org.apache.solr.util.SolrJacksonAnnotationInspector; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.zafarkhaja.semver.Version; +import com.google.common.base.Strings; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import com.jayway.jsonpath.spi.mapper.MappingProvider; + +public class PackageUtils { + + /** + * Represents a version which denotes the latest version available at the moment. + */ + public static String LATEST = "latest"; + + public static Configuration jsonPathConfiguration() { + MappingProvider provider = new JacksonMappingProvider(); + JsonProvider jsonProvider = new JacksonJsonProvider(); + Configuration c = Configuration.builder().jsonProvider(jsonProvider).mappingProvider(provider).options(com.jayway.jsonpath.Option.REQUIRE_PROPERTIES).build(); + return c; + } + + public static ObjectMapper getMapper() { + return new ObjectMapper().setAnnotationIntrospector(new SolrJacksonAnnotationInspector()); + } + + /** + * Uploads a file to the package store / file store of Solr. + * + * @param client A Solr client + * @param buffer File contents + * @param name Name of the file as it will appear in the file store (can be hierarchical) + * @param sig Signature digest (public key should be separately uploaded to ZK) + */ + public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig) + throws SolrServerException, IOException { + String resource = "/api/cluster/files" + name; + ModifiableSolrParams params = new ModifiableSolrParams(); + if (sig != null) { + params.add("sig", sig); + } + V2Response rsp = new V2Request.Builder(resource) + .withMethod(SolrRequest.METHOD.PUT) + .withPayload(buffer) + .forceV2(true) + .withMimeType("application/octet-stream") + .withParams(params) + .build() + .process(client); + if (!name.equals(rsp.getResponse().get(CommonParams.FILE))) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Mismatch in file uploaded. Uploaded: " + + rsp.getResponse().get(CommonParams.FILE)+", Original: "+name); + } + } + + /** + * Download JSON from the url and deserialize into klass. + */ + public static T getJson(HttpClient client, String url, Class klass) { + try { + return getMapper().readValue(getJsonStringFromUrl(client, url), klass); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Search through the list of jar files for a given file. Returns string of + * the file contents or null if file wasn't found. This is suitable for looking + * for manifest or property files within pre-downloaded jar files. + * Please note that the first instance of the file found is returned. + */ + public static String getFileFromJarsAsString(List jars, String filename) { + for (Path jarfile: jars) { + try (ZipFile zipFile = new ZipFile(jarfile.toFile())) { + ZipEntry entry = zipFile.getEntry(filename); + if (entry == null) continue; + return IOUtils.toString(zipFile.getInputStream(entry), "UTF-8"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.BAD_REQUEST, ex); + } + } + return null; + } + + /** + * Returns JSON string from a given URL + */ + public static String getJsonStringFromUrl(HttpClient client, String url) { + try { + return IOUtils.toString(client.execute(new HttpGet(url)).getEntity().getContent(), "UTF-8"); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Checks whether a given version satisfies the constraint (defined by a semver expression) + */ + public static boolean checkVersionConstraint(String ver, String constraint) { + return Strings.isNullOrEmpty(constraint) || Version.valueOf(ver).satisfies(constraint); + } + + /** + * Fetches a manifest file from the File Store / Package Store. A SHA512 check is enforced after fetching. + */ + public static Manifest fetchManifest(HttpSolrClient solrClient, String solrBaseUrl, String manifestFilePath, String expectedSHA512) throws MalformedURLException, IOException { + String manifestJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/node/files" + manifestFilePath); + String calculatedSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8"))); + if (expectedSHA512.equals(calculatedSHA512) == false) { + throw new SolrException(ErrorCode.UNAUTHORIZED, "The manifest SHA512 doesn't match expected SHA512. Possible unauthorized manipulation. " + + "Expected: " + expectedSHA512 + ", calculated: " + calculatedSHA512 + ", manifest location: " + manifestFilePath); + } + Manifest manifest = getMapper().readValue(manifestJson, Manifest.class); + return manifest; + } + + /** + * Replace a templatized string with parameter substituted string. First applies the overrides, then defaults and then systemParams. + */ + public static String resolve(String str, Map defaults, Map overrides, Map systemParams) { + // TODO: Should perhaps use Matchers etc. instead of this clumsy replaceAll(). + + if (str == null) return null; + for (String param: defaults.keySet()) { + str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param)); + } + for (String param: overrides.keySet()) { + str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param)); + } + for (String param: systemParams.keySet()) { + str = str.replaceAll("\\$\\{"+param+"\\}", systemParams.get(param)); + } + return str; + } + + /** + * Compares two versions v1 and v2. Returns negative if v1 isLessThan v2, positive if v1 isGreaterThan v2 and 0 if equal. + */ + public static int compareVersions(String v1, String v2) { + return Version.valueOf(v1).compareTo(Version.valueOf(v2)); + } + + public static String BLACK = "\u001B[30m"; + public static String RED = "\u001B[31m"; + public static String GREEN = "\u001B[32m"; + public static String YELLOW = "\u001B[33m"; + public static String BLUE = "\u001B[34m"; + public static String PURPLE = "\u001B[35m"; + public static String CYAN = "\u001B[36m"; + public static String WHITE = "\u001B[37m"; + + /** + * Console print using green color + */ + public static void printGreen(Object message) { + PackageUtils.print(PackageUtils.GREEN, message); + } + + /** + * Console print using red color + */ + public static void printRed(Object message) { + PackageUtils.print(PackageUtils.RED, message); + } + + public static void print(Object message) { + print(null, message); + } + + @SuppressForbidden(reason = "Need to use System.out.println() instead of log4j/slf4j for cleaner output") + public static void print(String color, Object message) { + String RESET = "\u001B[0m"; + + if (color != null) { + System.out.println(color + String.valueOf(message) + RESET); + } else { + System.out.println(message); + } + } + + public static String[] validateCollections(String collections[]) { + String collectionNameRegex = "^[a-zA-Z0-9_-]*$"; + for (String c: collections) { + if (c.matches(collectionNameRegex) == false) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid collection name: " + c + + ". Didn't match the pattern: '"+collectionNameRegex+"'"); + } + } + return collections; + } + + /** + * Replacement for Java 11's Map.of(). Borrowed from SolrTestCaseJ4's map(). + */ + public static Map map(Object... params) { + LinkedHashMap ret = new LinkedHashMap(); + for (int i=0; i getPackages() { + List list = new ArrayList<>(getPackagesMap().values()); + Collections.sort(list); + return list; + } + + /** + * Get a map of package name to {@link SolrPackage} objects + */ + public Map getPackagesMap() { + Map packagesMap = new HashMap<>(); + for (PackageRepository repository: getRepositories()) { + packagesMap.putAll(repository.getPackages()); + } + + return packagesMap; + } + + /** + * List of added repositories + */ + public List getRepositories() { + // TODO: Instead of fetching again and again, we should look for caching this + PackageRepository items[]; + try { + items = getMapper().readValue(getRepositoriesJson(packageManager.zkClient), DefaultPackageRepository[].class); + } catch (IOException | KeeperException | InterruptedException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + List repositories = Arrays.asList(items); + + for (PackageRepository updateRepository: repositories) { + updateRepository.refresh(); + } + + return repositories; + } + + /** + * Add a repository to Solr + */ + public void addRepository(String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException { + String existingRepositoriesJson = getRepositoriesJson(packageManager.zkClient); + log.info(existingRepositoriesJson); + + List repos = getMapper().readValue(existingRepositoriesJson, List.class); + repos.add(new DefaultPackageRepository(name, uri)); + if (packageManager.zkClient.exists("/repositories.json", true) == false) { + packageManager.zkClient.create("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true); + } else { + packageManager.zkClient.setData("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), true); + } + + if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe/"+name+".der", true)==false) { + packageManager.zkClient.create("/keys/exe/"+name+".der", new byte[0], CreateMode.PERSISTENT, true); + } + packageManager.zkClient.setData("/keys/exe/"+name+".der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true); + } + + private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException { + if (zkClient.exists("/repositories.json", true)) { + return new String(zkClient.getData("/repositories.json", null, null, true), "UTF-8"); + } + return "[]"; + } + + /** + * Install a given package and version from the available repositories to Solr. + * The various steps for doing so are, briefly, a) find upload a manifest to package store, + * b) download the artifacts and upload to package store, c) call {@link PackageAPI} to register + * the package. + */ + private boolean installPackage(String packageName, String version) throws SolrException { + SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); + if (existingPlugin != null && existingPlugin.version.equals(version)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Plugin already installed."); + } + + SolrPackageRelease release = getPackageRelease(packageName, version); + List downloaded = downloadPackageArtifacts(packageName, version); + // TODO: Should we introduce a checksum to validate the downloading? + // Currently, not a big problem since signature based checking happens anyway + + try { + // post the manifest + PackageUtils.printGreen("Posting manifest..."); + + if (release.manifest == null) { + String manifestJson = PackageUtils.getFileFromJarsAsString(downloaded, "manifest.json"); + if (manifestJson == null) { + throw new SolrException(ErrorCode.NOT_FOUND, "No manifest found for package: " + packageName + ", version: " + version); + } + release.manifest = getMapper().readValue(manifestJson, SolrPackage.Manifest.class); + } + String manifestJson = getMapper().writeValueAsString(release.manifest); + String manifestSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8"))); + PackageUtils.postFile(solrClient, ByteBuffer.wrap(manifestJson.getBytes("UTF-8")), + String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json"), null); + + // post the artifacts + PackageUtils.printGreen("Posting artifacts..."); + for (int i=0; i String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, file.getFileName().toString())).collect(Collectors.toList()); + add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; + add.manifestSHA512 = manifestSHA512; + + V2Request req = new V2Request.Builder("/api/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + + try { + V2Response resp = req.process(solrClient); + PackageUtils.printGreen("Response: "+resp.jsonStr()); + } catch (SolrServerException | IOException e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + + } catch (SolrServerException | IOException e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + return false; + } + + private List downloadPackageArtifacts(String packageName, String version) throws SolrException { + try { + SolrPackageRelease release = getPackageRelease(packageName, version); + List downloadedPaths = new ArrayList(release.artifacts.size()); + + for (PackageRepository repo: getRepositories()) { + if (repo.hasPackage(packageName)) { + for (Artifact art: release.artifacts) { + downloadedPaths.add(repo.download(art.url)); + } + return downloadedPaths; + } + } + } catch (IOException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Error during download of package " + packageName, e); + } + throw new SolrException(ErrorCode.NOT_FOUND, "Package not found in any repository."); + } + + /** + * Given a package name and version, find the release/version object as found in the repository + */ + private SolrPackageRelease getPackageRelease(String packageName, String version) throws SolrException { + SolrPackage pkg = getPackagesMap().get(packageName); + if (pkg == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Package "+packageName+" not found in any repository"); + } + if (version == null || PackageUtils.LATEST.equals(version)) { + return getLastPackageRelease(pkg); + } + for (SolrPackageRelease release : pkg.versions) { + if (PackageUtils.compareVersions(version, release.version) == 0) { + return release; + } + } + throw new SolrException(ErrorCode.BAD_REQUEST, "Package " + packageName + ":" + version + " does not exist in any repository."); + } + + public SolrPackageRelease getLastPackageRelease(String packageName) { + SolrPackage pkg = getPackagesMap().get(packageName); + if (pkg == null) { + return null; + } + return getLastPackageRelease(pkg); + } + + private SolrPackageRelease getLastPackageRelease(SolrPackage pkg) { + SolrPackageRelease latest = null; + for (SolrPackageRelease release: pkg.versions) { + if (latest == null) { + latest = release; + } else { + if (PackageUtils.compareVersions(latest.version, release.version) < 0) { + latest = release; + } + } + } + return latest; + } + + /** + * Is there a version of the package available in the repositories that is more + * latest than our latest installed version of the package? + */ + public boolean hasPackageUpdate(String packageName) { + SolrPackage pkg = getPackagesMap().get(packageName); + if (pkg == null) { + return false; + } + String installedVersion = packageManager.getPackageInstance(packageName, null).version; + SolrPackageRelease last = getLastPackageRelease(packageName); + return last != null && PackageUtils.compareVersions(last.version, installedVersion) > 0; + } + + /** + * Install a version of the package. Also, run verify commands in case some + * collection was using {@link PackagePluginHolder#LATEST} version of this package and got auto-updated. + */ + public void install(String packageName, String version) throws SolrException { + String latestVersion = getLastPackageRelease(packageName).version; + + Map collectionsDeployedIn = packageManager.getDeployedCollections(packageName); + List peggedToLatest = collectionsDeployedIn.keySet().stream(). + filter(collection -> collectionsDeployedIn.get(collection).equals(PackagePluginHolder.LATEST)).collect(Collectors.toList()); + if (!peggedToLatest.isEmpty()) { + PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+peggedToLatest); + } + + if (version == null || version.equals(PackageUtils.LATEST)) { + installPackage(packageName, latestVersion); + } else { + installPackage(packageName, version); + } + + SolrPackageInstance updatedPackage = packageManager.getPackageInstance(packageName, PackageUtils.LATEST); + boolean res = packageManager.verify(updatedPackage, peggedToLatest); + PackageUtils.printGreen("Verifying version " + updatedPackage.version + + " on " + peggedToLatest + ", result: " + res); + if (!res) throw new SolrException(ErrorCode.BAD_REQUEST, "Failed verification after deployment"); + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java new file mode 100644 index 00000000000..eaa4334107f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.util.ReflectMapWriter; + +/** + * Describes a package (along with all released versions) as it appears in a repository. + */ +public class SolrPackage implements Comparable, ReflectMapWriter { + + @JsonProperty("name") + public String name; + + @JsonProperty("description") + public String description; + + @JsonProperty("versions") + public List versions; + + @JsonProperty("repository") + private String repository; + + @Override + public String toString() { + return jsonStr(); + } + + public static class SolrPackageRelease implements ReflectMapWriter { + @JsonProperty("version") + public String version; + + @JsonProperty("date") + public Date date; + + @JsonProperty("artifacts") + public List artifacts; + + @JsonProperty("manifest") + public Manifest manifest; + + @Override + public String toString() { + return jsonStr(); + } + } + + public static class Artifact implements ReflectMapWriter { + @JsonProperty("url") + public String url; + + @JsonProperty("sig") + public String sig; + } + + public static class Manifest implements ReflectMapWriter { + @JsonProperty("version-constraint") + public String versionConstraint; + + @JsonProperty("plugins") + public List plugins; + + @JsonProperty("parameter-defaults") + public Map parameterDefaults; + } + + public static class Plugin implements ReflectMapWriter { + public String name; + @JsonProperty("setup-command") + public Command setupCommand; + + @JsonProperty("uninstall-command") + public Command uninstallCommand; + + @JsonProperty("verify-command") + public Command verifyCommand; + + @Override + public String toString() { + return jsonStr(); + } + } + + @Override + public int compareTo(SolrPackage o) { + return name.compareTo(o.name); + } + + public String getRepository() { + return repository; + } + + public void setRepository(String repository) { + this.repository = repository; + } + + public static class Command implements ReflectMapWriter { + @JsonProperty("path") + public String path; + + @JsonProperty("method") + public String method; + + @JsonProperty("payload") + public Map payload; + + @JsonProperty("condition") + public String condition; + + @JsonProperty("expected") + public String expected; + + @Override + public String toString() { + return jsonStr(); + } + } +} + diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java new file mode 100644 index 00000000000..f913bfb22c3 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import java.util.List; +import java.util.Map; + +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.packagemanager.SolrPackage.Manifest; +import org.apache.solr.packagemanager.SolrPackage.Plugin; + +/** + * Describes one instance of a package as it exists in Solr when installed. + */ +public class SolrPackageInstance implements ReflectMapWriter { + @JsonProperty("name") + final public String name; + + final public String description; + + @JsonProperty("version") + final public String version; + + final public Manifest manifest; + + final public List plugins; + + final public Map parameterDefaults; + + public SolrPackageInstance(String id, String description, String version, Manifest manifest, + List plugins, Map params) { + this.name = id; + this.description = description; + this.version = version; + this.manifest = manifest; + this.plugins = plugins; + this.parameterDefaults = params; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) return false; + return name.equals(((SolrPackageInstance)obj).name) && version.equals(((SolrPackageInstance)obj).version); + } + + @Override + public String toString() { + return jsonStr(); + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/package-info.java b/solr/core/src/java/org/apache/solr/packagemanager/package-info.java new file mode 100644 index 00000000000..66f135a2715 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains Package Manager (CLI) implementation + */ +package org.apache.solr.packagemanager; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 8327d3662f9..3b005445dac 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -180,12 +180,20 @@ public class PackageAPI { @JsonProperty public List files; + @JsonProperty + public String manifest; + + @JsonProperty + public String manifestSHA512; + public PkgVersion() { } public PkgVersion(Package.AddVersion addVersion) { this.version = addVersion.version; this.files = addVersion.files; + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; } @Override diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java new file mode 100644 index 00000000000..8464ec14597 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util; + +import static org.apache.solr.packagemanager.PackageUtils.printGreen; +import static org.apache.solr.packagemanager.PackageUtils.print; + +import java.lang.invoke.MethodHandles; +import java.util.Map; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.Pair; +import org.apache.solr.packagemanager.PackageManager; +import org.apache.solr.packagemanager.PackageUtils; +import org.apache.solr.packagemanager.RepositoryManager; +import org.apache.solr.packagemanager.SolrPackage; +import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; +import org.apache.solr.packagemanager.SolrPackageInstance; +import org.apache.solr.util.SolrCLI.StatusTool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class PackageTool extends SolrCLI.ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.") + public PackageTool() { + // Need a logging free, clean output going through to the user. + Configurator.setRootLevel(Level.OFF); + } + + @Override + public String getName() { + return "package"; + } + + public static String solrUrl = null; + public static String solrBaseUrl = null; + public PackageManager packageManager; + public RepositoryManager repositoryManager; + + @Override + @SuppressForbidden(reason = "We really need to print the stacktrace here, otherwise " + + "there shall be little else information to debug problems. Other SolrCLI tools " + + "don't print stack traces, hence special treatment is needed here.") + protected void runImpl(CommandLine cli) throws Exception { + try { + solrUrl = cli.getOptionValues("solrUrl")[cli.getOptionValues("solrUrl").length-1]; + solrBaseUrl = solrUrl.replaceAll("\\/solr$", ""); // strip out ending "/solr" + log.info("Solr url: "+solrUrl+", solr base url: "+solrBaseUrl); + String zkHost = getZkHost(cli); + + log.info("ZK: "+zkHost); + String cmd = cli.getArgList().size() == 0? "help": cli.getArgs()[0]; + + try (HttpSolrClient solrClient = new HttpSolrClient.Builder(solrBaseUrl).build()) { + if (cmd != null) { + packageManager = new PackageManager(solrClient, solrBaseUrl, zkHost); + try { + repositoryManager = new RepositoryManager(solrClient, packageManager); + + switch (cmd) { + case "add-repo": + String repoName = cli.getArgs()[1]; + String repoUrl = cli.getArgs()[2]; + repositoryManager.addRepository(repoName, repoUrl); + PackageUtils.printGreen("Added repository: " + repoName); + break; + case "list-installed": + PackageUtils.printGreen("Installed packages:\n-----"); + for (SolrPackageInstance pkg: packageManager.fetchInstalledPackageInstances()) { + PackageUtils.printGreen(pkg); + } + break; + case "list-available": + PackageUtils.printGreen("Available packages:\n-----"); + for (SolrPackage pkg: repositoryManager.getPackages()) { + PackageUtils.printGreen(pkg.name + " \t\t"+pkg.description); + for (SolrPackageRelease version: pkg.versions) { + PackageUtils.printGreen("\tVersion: "+version.version); + } + } + break; + case "list-deployed": + if (cli.hasOption('c')) { + String collection = cli.getArgs()[1]; + Map packages = packageManager.getPackagesDeployed(collection); + PackageUtils.printGreen("Packages deployed on " + collection + ":"); + for (String packageName: packages.keySet()) { + PackageUtils.printGreen("\t" + packages.get(packageName)); + } + } else { + String packageName = cli.getArgs()[1]; + Map deployedCollections = packageManager.getDeployedCollections(packageName); + PackageUtils.printGreen("Collections on which package " + packageName + " was deployed:"); + for (String collection: deployedCollections.keySet()) { + PackageUtils.printGreen("\t" + collection + "("+packageName+":"+deployedCollections.get(collection)+")"); + } + } + break; + case "install": + { + Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString()); + String packageName = parsedVersion.first(); + String version = parsedVersion.second(); + repositoryManager.install(packageName, version); + PackageUtils.printGreen(repositoryManager.toString() + " installed."); + break; + } + case "deploy": + { + Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString()); + String packageName = parsedVersion.first(); + String version = parsedVersion.second(); + boolean noprompt = cli.hasOption('y'); + boolean isUpdate = cli.hasOption("update") || cli.hasOption('u'); + packageManager.deploy(packageName, version, PackageUtils.validateCollections(cli.getOptionValues("collections")), cli.getOptionValues("param"), isUpdate, noprompt); + break; + } + case "undeploy": + { + Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString()); + if (parsedVersion.second() != null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Only package name expected, without a version. Actual: " + cli.getArgList().get(1)); + } + String packageName = parsedVersion.first(); + packageManager.undeploy(packageName, cli.getOptionValues("collections")); + break; + } + case "help": + case "usage": + print("Package Manager\n---------------"); + printGreen("./solr package add-repo "); + print("Add a repository to Solr."); + print(""); + printGreen("./solr package install [:] "); + print("Install a package into Solr. This copies over the artifacts from the repository into Solr's internal package store and sets up classloader for this package to be used."); + print(""); + printGreen("./solr package deploy [:] [-y] [--update] -collections [-p = -p = ...] "); + print("Bootstraps a previously installed package into the specified collections. It the package accepts parameters for its setup commands, they can be specified (as per package documentation)."); + print(""); + printGreen("./solr package list-installed"); + print("Print a list of packages installed in Solr."); + print(""); + printGreen("./solr package list-available"); + print("Print a list of packages available in the repositories."); + print(""); + printGreen("./solr package list-deployed -c "); + print("Print a list of packages deployed on a given collection."); + print(""); + printGreen("./solr package list-deployed "); + print("Print a list of collections on which a given package has been deployed."); + print(""); + printGreen("./solr package undeploy -collections "); + print("Undeploys a package from specified collection(s)"); + print("\n"); + print("Note: (a) Please add '-solrUrl http://host:port' parameter if needed (usually on Windows)."); + print(" (b) Please make sure that all Solr nodes are started with '-Denable.packages=true' parameter."); + print("\n"); + break; + default: + throw new RuntimeException("Unrecognized command: "+cmd); + }; + } finally { + packageManager.close(); + } + } + } + log.info("Finished: "+cmd); + + } catch (Exception ex) { + ex.printStackTrace(); // We need to print this since SolrCLI drops the stack trace in favour of brevity. Package tool should surely print full stacktraces! + throw ex; + } + } + + /** + * Parses package name and version in the format "name:version" or "name" + * @return A pair of package name (first) and version (second) + */ + private Pair parsePackageVersion(String arg) { + String splits[] = arg.split(":"); + if (splits.length > 2) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid package name: " + arg + + ". Didn't match the pattern: : or "); + } + + String packageName = splits[0]; + String version = splits.length == 2? splits[1]: null; + return new Pair(packageName, version); + } + + @SuppressWarnings("static-access") + public Option[] getOptions() { + return new Option[] { + OptionBuilder + .withArgName("URL") + .hasArg() + .isRequired(true) + .withDescription("Address of the Solr Web application, defaults to: " + SolrCLI.DEFAULT_SOLR_URL) + .create("solrUrl"), + + OptionBuilder + .withArgName("COLLECTIONS") + .hasArgs() + .isRequired(false) + .withDescription("List of collections. Run './solr package help' for more details.") + .create("collections"), + + OptionBuilder + .withArgName("PARAMS") + .hasArgs() + .isRequired(false) + .withDescription("List of parameters to be used with deploy command. Run './solr package help' for more details.") + .withLongOpt("param") + .create("p"), + + OptionBuilder + .isRequired(false) + .withDescription("If a deployment is an update over a previous deployment. Run './solr package help' for more details.") + .withLongOpt("update") + .create("u"), + + OptionBuilder + .isRequired(false) + .withDescription("Run './solr package help' for more details.") + .withLongOpt("collection") + .create("c"), + + OptionBuilder + .isRequired(false) + .withDescription("Run './solr package help' for more details.") + .withLongOpt("noprompt") + .create("y") + }; + } + + private String getZkHost(CommandLine cli) throws Exception { + String zkHost = cli.getOptionValue("zkHost"); + if (zkHost != null) + return zkHost; + + String systemInfoUrl = solrUrl+"/admin/info/system"; + CloseableHttpClient httpClient = SolrCLI.getHttpClient(); + try { + // hit Solr to get system info + Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); + + // convert raw JSON into user-friendly output + StatusTool statusTool = new StatusTool(); + Map status = statusTool.reportStatus(solrUrl+"/", systemInfo, httpClient); + Map cloud = (Map)status.get("cloud"); + if (cloud != null) { + String zookeeper = (String) cloud.get("ZooKeeper"); + if (zookeeper.endsWith("(embedded)")) { + zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length()); + } + zkHost = zookeeper; + } + } finally { + HttpClientUtil.close(httpClient); + } + + return zkHost; + } + +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index 3f97b824385..6c04b4480a4 100755 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -417,6 +417,10 @@ public class SolrCLI { return new AuthTool(); else if ("autoscaling".equals(toolType)) return new AutoscalingTool(); + else if ("export".equals(toolType)) + return new ExportTool(); + else if ("package".equals(toolType)) + return new PackageTool(); // If you add a built-in tool to this class, add it here to avoid // classpath scanning diff --git a/solr/core/src/test-files/solr/question-answer-repository-private-key.pem b/solr/core/src/test-files/solr/question-answer-repository-private-key.pem new file mode 100644 index 00000000000..d755d2d5c97 --- /dev/null +++ b/solr/core/src/test-files/solr/question-answer-repository-private-key.pem @@ -0,0 +1,9 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIBPAIBAAJBAOkIQ4lxVDPPIR2noh4MrTVw3JASYXd6k0wajrSDFYlwepVoSO/f +LK1AHYomrub8C8dXZwxzPMqpDVQ4MABdgcECAwEAAQJBAKzsLPG43zry4SgYVPzn +e0DE12cxvJHkq5k1u9/HxhuNqUdfhKzkMW5BqayDT74UqSLLTT9zFpGPsRUHD37V +aAECIQD7HV0vP9ipMskCMA+x8H8X/F2V2b6RWGlfIDc6XEiE8QIhAO2Q197gMnzP +z3VluoeL9pORw55aqx7kuAXwqmUqRInRAiBX3wWdpBTX2EqYdmL3nDWNGiVRa5mQ +2MQ+olJRHLvPsQIhAL0w/LmiEpMTbEQyH7qS3GvpScBytJSF0YfpgcnPP4YBAiEA +14io0rN1QgVWykmckeZZSzaPLqpkE2wCUZUOemoBByg= +-----END RSA PRIVATE KEY----- diff --git a/solr/core/src/test-files/solr/question-answer-repository/publickey.der b/solr/core/src/test-files/solr/question-answer-repository/publickey.der new file mode 100644 index 0000000000000000000000000000000000000000..8ca82e87862ccc0a0cacc08ebb07498f6dfcea7a GIT binary patch literal 94 zcmV-k0HOadTrdp=2`Yw2hW8Bt0RaU714{rfNCH6s=?FuKaa1$UAswfp9t^EDaNLj* zVRw3yOd5`~gB6KzdX;EM@82w~Kpl!EuIBs;$5&?zb3Dqa4OBQV09}E>0s{d60da00 AzW@LL literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..239a8687accc9aa2b64565dfa4f490862e0c35aa GIT binary patch literal 5579 zcmbtYby$?`(qB5Hc0piikX&E|Q5vMXTac0#mImosx}`&ur6i;lgaxHj2?3?M1eHbs z$q!$T`kq(6bN)ET41J5=(M?RfIH)@GpT;0NY0jKw&fX}B3fx#ZnLC-=Z94RInp^qGr!IGr#eZR*B({Bu3=oNaK?y3taIlm!(;@QMho~+9PF>lkrpQ zOlEov+VLiacD$wYLo;1Txb8yHd^f~q=UL>M%p;QBtI#vi`g2{NDe0~UaGu0PtMdv| zGsltH{>Ru42rqc0*k=nnY!6A!@pzy4&sdm@H#~~77kCg3LjF`52o23(rsFm=DPqvmC zSFt+PjJPM#>v@Chb2o=$icI+GTV-5YHez=2w+~`tGVsEOR79EQZ|;?|Gg9vbW%ii} z#}A?zNyjv;M1p%G%TI6L%l@1(Wskw69Bk`b@iFvh5(6pQUOf!T7$U7o-h8W~_+se%tlQZai5|XEJxM^pOBCNcS`z zR3SytamOOApqpof(L@|HH3UUak1L^o#^?b|i%*Zh(z?ncNOsn9mHfT3tiqG>fG&DA zp}&lyYU)LCE+D^YCj~1A;V{;njbPic`LOn)QmCwZ1cEBT0SZD5UwtWp1g@z53MTc zOl4;|#(oKZ*VFyFWI8(K;q_Ar%VH;0{R-2ue#x-uo$VC{W>W%fUa0_~*D-HvYaVmB zUF6>Xa?Ss`F5)9!cWF)=gN;zAmfrZg;h|yO_my^r+x6ifrFyWUhK!d)^gSQ-2mSmv(lxr3 zz)%K%H`*;X@+o0z6m9X$_LN!7Wt(5y9JD-*JooBBpqswQV#I~gYQ{l_G$*QwI0FTe%xs&Md?}Rp{`R)iIxK3= zu9-6@Frm)6)y5%eDpG1US$EM4Yw{gr2}w@4DtxR=gw%>0!GuG$r?Af(TUyIsp&j+{ znCNlEFu8^HS20+G5?foUIZRDdrQ={?O(#A+Y5Qgz_MIoP1UbPa&7-U*V?0(*sK+VR zRv{YcnF!Vv9D8we{w+) zPisxDf2j80hEw;s+$v{Y6Rl`7`fz<&+rdzuI%mbEsV&-^3|pIB7`=_NNJF$}K-njV z0bf&+E=(g@$>@`x{EhO|O3TlQCgJS&IYt0?dXc25AYY$H?*YhoiKgaP;&_CaraH1 zroO)o`CQ>AEyq3&rKa5ueeyV6qEkt#yD&7muNt!q*L&0;VN}Bz3mdoQMK!svCfv80 z2uCyKL^DDf1$vW9$2J^9Q4PF~cqlHBl|ekf6-llar&osuQ<1}&jvsKqAv~w5L&7Jz z;}`B9a8;PI;rY|V;iV*=4p=%}S-iNqA$7g{=yoxk6#X$?LufmJo?YQRkxG2SO^HWC z5G3h}4e{*#AX$VY+q-RR{fBcop{GsHz}O!++ZoA26pt|t3T!#>O}kvtUhz#_GfNMW zM0;V{!G$J`e(!s+R}2TI)7C)Kk=gR&`IT-eZFmcPX2?r7tPxbM*e zMZ?dmTKWosoN~^0^oOAEAs%P~D#1MUO-_3@jiD?)@X3ok!BXQ4gTZ196DI zon`*=tKb;9Ss!ISE4_uWxExHp``9m5cyJ;S?qVe_=Z~mwJ4vI7jr&R&HyK;aqYpd? zda=+kYakhCdF=~|5^*6s6CvUPNLWC7)C%{?EL|WocjsDu`W8&FyEXhcuz7&d&r3io z{7%FsR6IPLXVNbL_O;3Od38H(rm8L4InIxo^Lt4=9#=_hrFmXZnUb$|RDBA{dP#UrZz@l5DTnu^{XM*Gj(ud*VRc?-DM%n?vuLWE>S(By z3O-y{NQQW$arcGO?OIOb4?d|HH@os*%hh_sncqDcN%a+I&z-h5(+&R&ufRQ|#Mex* zIT=tg4zHNk*+L}F4w-s{jhprM8rSo=WiD<^HC3#6Sdwh%@~1mqQ5t?I0Bc!{O3b@4 zZakkFLoi*qOhZiZjEy)NAdGo)|91N_AZUpMTyzKgqlKNqP$DC=b5KI z$Y@xBQte&grk`fbG*w+NRc7Zk@KpR=FpAlzj^@@v7I%(u-n&-eFO#@3@|Ge^1(W)E4B(BKkz59Z_5j81So(;(%fq-X3%klq@Tc%DC;*cx^iy99?H4a_`kXWko!0!0V2CwM11W zU^V1R`v?!&nC(cQAW=M)aja=NayMd#Kg2iO~wm?+G0r1Mw_vo6H8jv_3O^ST*=1u{OGdPD_t>R(c9@ z5w1eq{bnVtb)WgEdt?NcVQ<@0B--SZ7myg2q>8ez^Ge1r0J=V+Vt*Z&5jsZJUod9< zDs3byq}0$GT!_{h#S)86?d#ED%aZw2ND1D`->N(|0m2gXXwu8xJ zKr@qGvmD*L#`Sq3w2XK(6=HxT#}WwA0f@$iCZFS*&;lkX0lkD6y=P?V?eFde$Tu|8 zm!`(puPzyWkmN3^_`+~vUt3xdTejiyhy!^}Oc9DjD0zcYh>Xa2o0{T(LspJ0FWsCG_{+~$sM53F3d zUB7p&+_vA_)}Pkk0?aSlcD|aeaX&Nwp!qUSN&GKjot1dvqPS$W;)r40CW5@Un7op{knv;0~6+WR~F9hZemHup2T*J1oRJPLV& zbyw{-SK}<-Idhy3LjzKFq^7tZOhTpEjF}r?lK1aOd?8Yro^}gFYa<;9Q(~EZ-_Uyq z)3v)cnz>N6;!sL3uWh}tTZXkMp$%~-hUhM&ewL8c3}zDSZPbc6R8)q(K2<0`adWqR zAekXl3|#=$hPN1fO^K;5eNFEWW@gi4@^D>l=e$AHei6=j*r9z@@Y?sisgw6ELn+j$58L%uCcRc?i& zc)}BRJcnXVf^GQ(*h`A+tOB=pj=cuLGB8uoMfFhkSojA7DNu#@%~*jSr=-{7d1mA% zL<0F!nXgCrwz$1#DR~(&q|Va)Oi>S>*B7cjyQ9tiiJ#l`JjnpjG65En0qtXGD5Ig1 zVf_CSsmsM(5(!{a^4tAq@cn}Qc|!G#`5P4kvt3?)!u~d|`X%AdL!`gpmx|9MyDuOA z9=SgZ{;v6-L#6LH0H8K_>)&vH9x?s0;?HHpFKNP8{%*xTrTw$W_$BxE4D&6EeJ2vY z#l^MD*Ps6Rk<)%G@ptgIto946@jLiWp88$Pw=DNdOvHZ@^WSsc4{QIN^?oR-4W9d7 w)ch+4{&MV}Bk`9u6!164{_@R#Thupgzv5Cu84LTHVEoI2^UaB>g*?lBBY{j5Jh(TS?}NQeUs40vGo%o&p#9K;K}6I?o6G zg%t-*$UqmT0v8AqV>wqV;RZ)Dqg!1%gx?Y3l)+w1$``>;{N?J02z3$#b!amv8ZE4w z{3Ui%80@7xc85p42Qwia+gn4nNm^0P16)K@WRzQ-TOHtsd4RvnyRzu#xZaz&v$eIO zldXgMKekb@i3i5NB6xlS0HFOhIyFgkn7uR1(Miq(Ze|U0;C^Ur;^>&6<8Dm4LU!pr zsbpSG9{n1*B$WG!S;gcVG_}v@Y;>sc#11657E5NXak$pzNvRJ&7FBswTwnK#Fs88C z$L6pRAt{fyveZ3|t#+nb=Z#F>8kdk*Qc^80I?Jc8-SDQQD7b#TgOI1n>?nWcXooAPPNvsZ+d9sK(O{W1I86 zfiUX=oB(q7jQZk;f$5lad*v1Qo4rn(H}y;mhVV(KUyS=3LV|GBdv2ICH}|kgmg1R? zNyYnVUJ&g*AV@-(T)0WK4V#{NDz@d&N8`9f;uJ_7gaV1}176Ubx6Z6|N=Jqsr-!mg z-D6w$LaiTCaPJtosf)%4!R=Lts4X*r0EsLnMT*N_#=u zYNQ%y5!3Wd42XLACKkjQvJdUT6Wn&LvHo%xTSy1(nAJ1W4{fkf4MlX`MIMX0H6{fO zviK}qg4A$TrB~so(;p0#s^dah-k2kM+JIH~sD-#@fw(mqHSG2A;g)0Fx|}7zUKu>C zVg^Bwtc{hPHRNs)BjElk>&E zEeBWq#TE~ZXxFZgo+Hcb>2+xL2~PfQh+=IWT8MF`q>x9e11S0{n+Y_AaUNwsdX(gx`&S!cF47BtwqGIU}DoO1* z&cSbYiQO`1T?>QGu!iCG79O_}HLLUCAKZdneaR!SBdYHq+^Qu2Cr@p!iixTvK6ZKo@ zdU{df;|I+VxX-5=)tKJ5Dt3d`kJH1FEd1GINVA+^{U zxT}$ZagSSLG2Bn&ETq$dxb-;}(Vei5!-8HQ9^rC4i%*i~-FrllXf(z-YkSCcA>4i9 ziQ70oXD@`jf@~PK>-a5T5n-9gC2baCnv$S_b!&ooG5I*r&q(1IuY_z@59?j)4*3nm zzM+s@ikLmt8)E`h<1*ifFZ?91kK=gGEW@V!%TBQL$6&N6V%+wH|&6M75h1;7yipAIhh}ZEiR*4E0 z2$xSLi%Y?oE~S=Ur)F_$7?WmS5|%VekP=gg z6x*LB__3QMvujR;3T6`r#-X=M%oWlC4RBQ1r;rO8ab{;Z@<>Z_OVI|RxRK6pE zLai6?tX)&*@v}|$mNf=2XGu-97v4R_sF(lpdm~wh!TD14uSQO>?xH0H69|Kl6CV`xq;?8#Xq|7Jc9v!w; z>0FiSZ$*yr@Ub_irsa9Zht#qXjeE$J>kGoBmYFIvyBmAJWW`ah2Sv+sdypG;HF)hb zqluAA_RcXNDwP+?Pe(8+x#OpMHp{K0SArgQ@7Y%}L1*qjGRts1cE}1=YCrL=GIQu9 ztQ?y)S*?U8;;ycC@(Y%QJy5O@Hq`uZOd6$ILQgr|q!!lfG|eFmd#yc2l5e(8H`=^O z5A;?uOYX?DWIJDOe4dNXKWMpIL=9&a(9;kKpD{{Qi&x4A$?qDZlMdi0`ta{kZ^u0h z>L{n0u}{d~&d{*=da`gZPn2M<{(N$bt$cRb{E>=}07dG$c>D8@>sXTx^jQ|mYA|kJ zmZu}d73{JU`;ByLtOv*?!zaMEDRT|+qmP7L7iEi>h2yZ=NTWlOwqr44pzF-{Vi_g@ zjIo^EtMQ|xLS0^WSed$+!Fi2TgP9x&P@WwPjZ{~!46iA4?9|*kJ6+2~6IEFvaS8r; zGrD&@&uRK(V5Kk?Ur{>A3Zcm_sv^rD(A);g=26{;6e*+}MwIV>!?K2ILaOC85vVN5 z-Hr24!Pc@RIE^K|Wpvt7oq?9-NVn6lhb<>QSc$#mS2HJKceqa%BS6zy2(dzlBo>zJ zAU;#r@ZL$BA9X2oBGxA#Uu|m@u#WLoURh&3Q69C3J$%qMM3l0wXH#x_2>n`T?`>~b zQa&Hueo!`xn(5~wU>1-yJ7*)!OLJnuMHs~l^?(=5iqu-5_2JO5 zstSBgt@n|ywv0-96yh+jM9~m@!6)9H?_4z^?=w#M8VEAs3C?~<*kFp?{&1=x7Xve! zkZf5EGq13+YQU9oRQCiTBlH;-N-uM7WXIv@4soiN4=S}oMQ+&$$F0`aq~=f9p(!Ny z@Q-*t@y+1T*?6JmISp-yu>?DEgz(1D1)s2S(k6G<4^nm>Mp4uDoZu22Vcu{#odMU< zs;@bE-psQzF>6?lkekq!B9QITKHx-)kw)#Lq@ZWWR-x)J4G9!}udM)l-ZCOrR+wMb&eX4!Tq9uD>&Rn z3NsIji5!#%&g_D*a*W6Zk+$3^ZaG6YGFd~_qV`^^iqf}y>P`F5SM$@roC4LTO$OYy zu()EV{?wT~%d6hnse#1D!w{M16&86iF*|oB@oTzeO%sN3(0j5qmcH$Fk4;&@?f2+Z zGI-lyuIL5Gy(p1%keFe!c|$1rDZtW9T%e5h*#k1&i5r!C_M-( zl;d8^I=%9_5saRp`Yzc(+QXG(|EPi)m|Kb|tAok>nZ)>Mb{?x2S}z*opy>-gF=TYq zO>eRaR9x%{S9FE~`9r@sk~~P|;ETmzfq=XW$(jEKl$GLDA#xAkud&`zJosPj=es08Uj@TtA4Sc){cPFig zo-**ly=_u{LOs3jIc)kcn23*Z821~&kDT*+O8haZlq3rU05HS=0NDR)F3G4SscPcn z1ap9Y&nlxd6dWm~2`^+ytHw#BS$U&3QIUCN9`vfwq9bFWKirfuzUyL!i)E)%%Df*k zR#Hw8Tgq)!B@2Tk}pb90xM|RLJJtVpwc8-$#S_ z#0cE!yd^4vo3E+w+^W`}F_+`Qz5`s**VYA!ud)wk$t3jVPufk0@5wKF#IaMVnY} zSPrQCA#ck|tzw#@+Mgn|je&eBTA#d@QLmaxaN!wOhC%j+W|6~5OetACAhS$Xoq`{X z(3>}%J1XeWcp!W&R8yN*c#n}SSp&6`H2>}Z;zlRx^Ock~@@V=2;Kag2n@u!SxSuzM zxT>zaNptos*v9@tS^FMQH5PYg=W);y{wwQ5v(&UsI5dkLQ8(>}=PRcNVdu>F~>NZhI&^T#d?znk( z(#W%bB3*a#JfvnnuY`}ef*(6|iodapSkIrLy#!UyJgHOBlgBH{gvznc28TUaI93n5 zOB38_%xwQOd`Oe_6DOW|^y`PlzJZji@!~W$cZN#BpXUdlkI|H>DO~^?6hq8tSiVli z1Dfcynq;WwRd3JZBc((mDG~t8*bv0*t$=WJ?)VFA!&`s}AfO8mrR$tjrR9UZk8E8N zO;KWm)e1uQvm{r3$sz5TRb|o3$XA=*>6;AiXbJ|_ifNp5sq64Ym|0c3KUx}NBQr2d zWG*fl{dm!TFOdnZUi62?+(Sb| zEBWB7EjyGZ6ehB3HcBT=5*8Rkr|<87tx^IhJu7T%=dYNv{TL(wq4rNb*3bq?Fkw7= zCfFgkb)R-h)dj$)8D?Zu?H0zDzyx;i9`RB<@{TzT9_+8M4w0x%6VlTeO~A1!G>(ZR;1TvY#-WAKf2F}PB1yWAi|`K_G8bijvlzx7)Q)itXZ$N?&qkKEi*?Ssu@j%9pK$V2_h@yk zZ##eo%pXwiCp$!O0bhY~&lR!zz99?9_zXf5pv5MP40HjpW?;}ww3U&~C{{0Q8NXF9 z6)7b!V0n-^)^9vFu6QSwQWX1r>yAJA6XD_8wM6~O&bL&1A+z^GLI;#nHFhZ-gY>iN z&KPC7*g_&?NbbB7yDaa?p4RKM!GHCr9A4{9-8;^X{?$esb-rdhiM$BHFGbLXhWyCzAWqE8Cc)hxlzLTI$?6jv6E8s##(;ff|yn%{#Ko9i7l@< z^SxVJXzW(HIk;bq7mLfRb7(#hgL>3lY5O-6OJ{bW!YeKmaR)HL9$#LZFoM^+DA)VC zZYp5{hOT_JM{^M=>&j=dQ2_uv02RQ-1a4`5)tKRa>}U(u)W8Cuyn*%`Tu)~=8~_sP z+LdqrwKQtHhfnk1Hm-7@X6taEI|8RTd$}-?2l7dB%9rK$DOz{m#i++I zMbGM~z2IEm+Y^g(5hJ{$IacT}*Oe~Q8Yk~d2-HKGmtt&cU8BJ?&A#dt*{~TwQ%rr}&o9SOK(E zh`s+#2DX1@90iJ~ahw!}nyiwz*9T9{!Gd9@{cpQV_SF5%5tb+nNM%wRO5m<986env z*Gzt1!nh^W;4;1M==Eq06bUO&2hy2Z<(Dy7dOs3yK5a|Da~gHqwkbS>ATN)cq7R&P zOF$_mK7B^ZlYz(cq{;YDm$(^c122{*_t8=AvC-0ptyY7zUrD zBc)05A7iL0AR&{Y{O8`|l?z-ALco^f_4Oyr{$l%cFY=oBdgS`ET&+LZ{??iNCE?HQ z!oS$B6rW3WT;2X1qkkCuUGqQNhu`@CfJ*=Ef8+aeGx3)de@-BONfW{OyA}VG_RnZ$A`O`p^9@YW@}NejV(e5%-t2w?w}k?60%=|2B24?HB%DE2T#HPjCqQOXn3j kUin-cG2oxzi1r;E{{f7u3TWupd^lGhw=1;txRwt1KU%auxc~qF literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/solr/question-answer-repository/repository.json b/solr/core/src/test-files/solr/question-answer-repository/repository.json new file mode 100644 index 00000000000..51609723bb2 --- /dev/null +++ b/solr/core/src/test-files/solr/question-answer-repository/repository.json @@ -0,0 +1,56 @@ +[ + { + "name": "question-answer", + "description": "A natural language question answering plugin", + "versions": [ + { + "version": "1.0.0", + "date": "2019-01-01", + "artifacts": [ + { + "url": "question-answer-request-handler-1.0.jar", + "sig": "TTzgh5/usbyWg7oZ7lRwz4eQfh1FeXWvv4U85tsVthVz0MRDz9t7SmonDkegZ7OyqeoiQ4I207pifpVW+DRd9Q==" + } + ], + "manifest": { + "version-constraint": "8 - 9", + "plugins": [ + { + "name": "request-handler", + "setup-command": { + "path": "/api/collections/${collection}/config", + "payload": {"add-requesthandler": {"name": "${RH-HANDLER-PATH}", "class": "question-answer:fullstory.QARequestHandler"}}, + "method": "POST" + }, + "uninstall-command": { + "path": "/api/collections/${collection}/config", + "payload": {"delete-requesthandler": "${RH-HANDLER-PATH}"}, + "method": "POST" + }, + "verify-command": { + "path": "/api/collections/${collection}/config/requestHandler?componentName=${RH-HANDLER-PATH}&meta=true", + "method": "GET", + "condition": "$['config'].['requestHandler'].['${RH-HANDLER-PATH}'].['_packageinfo_'].['version']", + "expected": "${package-version}" + } + } + ], + "parameter-defaults": { + "RH-HANDLER-PATH": "/mypath" + } + } + }, + { + "version": "1.1.0", + "date": "2019-01-01", + "artifacts": [ + { + "url": "question-answer-request-handler-1.1.jar", + "sig": "LypqlmbJ76AWa5jx0XhjKxO4lrcQAvpSuYddfzcE6TnX0VDPFhrlQHSSX6cZLtvNbQ+74xUMKgsoX1IUnEnKYw==" + } + ] + } + ] + } +] + diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java new file mode 100644 index 00000000000..a9bf9075826 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.Arrays; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.core.TestSolrConfigHandler; +import org.apache.solr.util.PackageTool; +import org.apache.solr.util.SolrCLI; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PackageManagerCLITest extends SolrCloudTestCase { + + // Note for those who want to modify the jar files used in the packages used in this test: + // You need to re-sign the jars for install step, as follows: + // $ openssl dgst -sha1 -sign ./solr/core/src/test-files/solr/question-answer-repository-private-key.pem ./solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar | openssl enc -base64 + // You can place the new signature thus obtained (removing any whitespaces) in the repository.json. + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static LocalWebServer repositoryServer; + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("enable.packages", "true"); + + configureCluster(1) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .addConfig("conf2", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + repositoryServer = new LocalWebServer(TEST_PATH().resolve("question-answer-repository").toString()); + repositoryServer.start(); + } + + @AfterClass + public static void teardown() throws Exception { + repositoryServer.stop(); + System.clearProperty("enable.packages"); + } + + @Test + @SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.") + public void testPackageManager() throws Exception { + PackageTool tool = new PackageTool(); + + // Enable the logger for this test. Need to do this since the tool disables logger. + Configurator.setRootLevel(Level.INFO); + + String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString(); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "add-repo", "fullstory", "http://localhost:" + repositoryServer.getPort()}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-available"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer:1.0.0"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"}); + + CollectionAdminRequest.createCollection("abc", "conf1", 1, 1).process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection("def", "conf2", 1, 1).process(cluster.getSolrClient()); + + String rhPath = "/mypath2"; + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer", "-y", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath}); + assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0"); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "-c", "abc"}); + + // Should we test the "auto-update to latest" functionality or the default explicit deploy functionality + boolean autoUpdateToLatest = random().nextBoolean(); + + if (autoUpdateToLatest) { + log.info("Testing auto-update to latest installed"); + + // This command pegs the version to the latest available + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer:latest", "-y", "-collections", "abc"}); + assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.0.0"); + + run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer"}); + assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.1.0"); + } else { + log.info("Testing explicit deployment to a different/newer version"); + + run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer"}); + assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0"); + + if (random().nextBoolean()) { // even if parameters are not passed in, they should be picked up from previous deployment + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "-y", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath}); + } else { + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "-y", "question-answer", "-collections", "abc"}); + } + assertPackageVersion("abc", "question-answer", "1.1.0", rhPath, "1.1.0"); + } + + log.info("Running undeploy..."); + run(tool, new String[] {"-solrUrl", solrUrl, "undeploy", "question-answer", "-collections", "abc"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"}); + + } + + void assertPackageVersion(String collection, String pkg, String version, String component, String componentVersion) throws Exception { + TestSolrConfigHandler.testForResponseElement( + null, + cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection, + "/config/params?meta=true", + cluster.getSolrClient(), + Arrays.asList("response", "params", "PKG_VERSIONS", pkg), + version, + 1); + + TestSolrConfigHandler.testForResponseElement( + null, + cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection, + "/config/requestHandler?componentName=" + component + "&meta=true", + cluster.getSolrClient(), + Arrays.asList("config", "requestHandler", component, "_packageinfo_", "version"), + componentVersion, + 1); + } + + private void run(PackageTool tool, String[] args) throws Exception { + int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res); + } + + static class LocalWebServer { + private int port = 0; + final private String resourceDir; + Server server; + ServerConnector connector; + + public LocalWebServer(String resourceDir) { + this.resourceDir = resourceDir; + } + + public int getPort() { + return connector != null? connector.getLocalPort(): port; + } + + public void start() throws Exception { + server = new Server(); + + connector = new ServerConnector(server); + connector.setPort(port); + server.addConnector(connector); + server.setStopAtShutdown(true); + + ResourceHandler resourceHandler = new ResourceHandler(); + resourceHandler.setResourceBase(resourceDir); + resourceHandler.setDirectoriesListed(true); + + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] {resourceHandler, new DefaultHandler()}); + server.setHandler(handlers); + + server.start(); + } + + public void stop() throws Exception { + server.stop(); + } + } +} diff --git a/solr/licenses/java-semver-0.9.0.jar.sha1 b/solr/licenses/java-semver-0.9.0.jar.sha1 new file mode 100644 index 00000000000..e59c2edfaee --- /dev/null +++ b/solr/licenses/java-semver-0.9.0.jar.sha1 @@ -0,0 +1 @@ +59a83ca73c72a5e25b3f0b1bb305230a11000329 diff --git a/solr/licenses/java-semver-LICENSE-MIT.txt b/solr/licenses/java-semver-LICENSE-MIT.txt new file mode 100644 index 00000000000..66a6a512a83 --- /dev/null +++ b/solr/licenses/java-semver-LICENSE-MIT.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright 2012-2014 Zafar Khaja . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java index bda1078ed04..d3fe6473839 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java @@ -33,6 +33,10 @@ public class Package { public String version; @JsonProperty(required = true) public List files; + @JsonProperty + public String manifest; + @JsonProperty + public String manifestSHA512; }