From d9f41f8a5a31e7dd8f4ccee729d479ce07175c1a Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Thu, 14 Nov 2019 18:21:35 +0530 Subject: [PATCH] SOLR-13662: Package manager (CLI) --- lucene/ivy-versions.properties | 2 + solr/CHANGES.txt | 12 +- solr/bin/solr | 55 +++ solr/bin/solr.cmd | 10 + solr/core/ivy.xml | 3 + .../DefaultPackageRepository.java | 116 +++++ .../solr/packagemanager/PackageManager.java | 416 ++++++++++++++++++ .../packagemanager/PackageRepository.java | 53 +++ .../solr/packagemanager/PackageUtils.java | 238 ++++++++++ .../packagemanager/RepositoryManager.java | 317 +++++++++++++ .../solr/packagemanager/SolrPackage.java | 140 ++++++ .../packagemanager/SolrPackageInstance.java | 66 +++ .../solr/packagemanager/package-info.java | 21 + .../java/org/apache/solr/pkg/PackageAPI.java | 8 + .../org/apache/solr/util/PackageTool.java | 293 ++++++++++++ .../java/org/apache/solr/util/SolrCLI.java | 2 + ...question-answer-repository-private-key.pem | 9 + .../question-answer-repository/publickey.der | Bin 0 -> 94 bytes .../question-answer-request-handler-1.0.jar | Bin 0 -> 5652 bytes .../question-answer-request-handler-1.1.jar | Bin 0 -> 6324 bytes .../repository.json | 56 +++ .../solr/cloud/PackageManagerCLITest.java | 201 +++++++++ solr/licenses/java-semver-0.9.0.jar.sha1 | 1 + solr/licenses/java-semver-LICENSE-MIT.txt | 21 + .../client/solrj/request/beans/Package.java | 4 + 25 files changed, 2041 insertions(+), 3 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java create mode 100644 solr/core/src/java/org/apache/solr/packagemanager/package-info.java create mode 100644 solr/core/src/java/org/apache/solr/util/PackageTool.java create mode 100644 solr/core/src/test-files/solr/question-answer-repository-private-key.pem create mode 100644 solr/core/src/test-files/solr/question-answer-repository/publickey.der create mode 100644 solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar create mode 100644 solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar create mode 100644 solr/core/src/test-files/solr/question-answer-repository/repository.json create mode 100644 solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java create mode 100644 solr/licenses/java-semver-0.9.0.jar.sha1 create mode 100644 solr/licenses/java-semver-LICENSE-MIT.txt diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 4d3b6c0caac..a1f6db3689d 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -26,6 +26,8 @@ com.fasterxml.jackson.core.version = 2.9.9 /com.github.ben-manes.caffeine/caffeine = 2.8.0 /com.github.virtuald/curvesapi = 1.04 +/com.github.zafarkhaja/java-semver = 0.9.0 + /com.google.guava/guava = 25.1-jre /com.google.protobuf/protobuf-java = 3.6.1 /com.google.re2j/re2j = 1.2 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8790ee5c504..81462b0cc62 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -108,9 +108,15 @@ New Features --------------------- * SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya) -* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store - the configuration, APIs to read/edit them and isolated classloaders to load the classes from - hose packages if the 'class' attribute is prefixed with `:` (noble, Ishan Chattopadhyaya) +* SOLR-13822: A Package management system with the following features: + (a) A packages.json in ZK to store the configuration, + (b) APIs to read/edit them, and + (c) Isolated classloaders to load the classes from those packages when the 'class' + attribute is prefixed with ':' + (noble, Ishan Chattopadhyaya) + +* SOLR-13662: A CLI based Package Manager ("bin/solr package help" for more details). + (Ishan Chattopadhyaya, noble, David Smiley, Jan Hoydahl) * SOLR-10786: Add DBSCAN clustering Streaming Evaluator (Joel Bernstein) diff --git a/solr/bin/solr b/solr/bin/solr index 596242fac4b..4b6236537bf 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -764,6 +764,56 @@ function get_info() { return $CODE } # end get_info +function run_package() { + runningSolrUrl="" + + numSolrs=`find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | wc -l | tr -d ' '` + if [ "$numSolrs" != "0" ]; then + #echo -e "\nFound $numSolrs Solr nodes: " + while read PIDF + do + ID=`cat "$PIDF"` + port=`jetty_port "$ID"` + if [ "$port" != "" ]; then + #echo -e "\nSolr process $ID running on port $port" + runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr" + break + CODE=$? + echo "" + else + echo -e "\nSolr process $ID from $PIDF not found." + CODE=1 + fi + done < <(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f) + else + # no pid files but check using ps just to be sure + numSolrs=`ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//'` + if [ "$numSolrs" != "0" ]; then + echo -e "\nFound $numSolrs Solr nodes: " + PROCESSES=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | awk '{print $2}' | sort -r) + for ID in $PROCESSES + do + port=`jetty_port "$ID"` + if [ "$port" != "" ]; then + echo "" + echo "Solr process $ID running on port $port" + runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr" + break + CODE=$? + echo "" + fi + done + else + echo -e "\nNo Solr nodes are running.\n" + exit 1 + CODE=3 + fi + fi + + run_tool package -solrUrl "$runningSolrUrl" $@ + #exit $? +} + # tries to gracefully stop Solr using the Jetty # stop command and if that fails, then uses kill -9 function stop_solr() { @@ -1359,6 +1409,11 @@ if [[ "$SCRIPT_CMD" == "export" ]]; then exit $? fi +if [[ "$SCRIPT_CMD" == "package" ]]; then + run_package $@ + exit $? +fi + if [[ "$SCRIPT_CMD" == "auth" ]]; then VERBOSE="" diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 221b514f1e7..792e621a05e 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -215,6 +215,7 @@ IF "%1"=="-version" goto get_version IF "%1"=="assert" goto run_assert IF "%1"=="autoscaling" goto run_autoscaling IF "%1"=="export" goto run_export +IF "%1"=="package" goto run_package REM Only allow the command to be the first argument, assume start if not supplied IF "%1"=="start" goto set_script_cmd @@ -1422,6 +1423,15 @@ goto done: org.apache.solr.util.SolrCLI %* goto done: +:run_package +REM TODO: Compute the running Solr URL and populate it as a parameter (as has been done for the shell script) +REM Without that, users will have to supply -solrUrl parameter in every request. Life can be so hard for Windows users! +"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.util.SolrCLI %* +goto done: + :parse_config_args IF [%1]==[] goto run_config IF "%1"=="-z" goto set_config_zk diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 9fba66347ac..f7843a1eff8 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -138,6 +138,9 @@ + + + diff --git a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java new file mode 100644 index 00000000000..dee55a5047c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +/** + * This is a serializable bean (for the JSON that is stored in /repository.json) representing a repository of Solr packages. + * Supports standard repositories based on a webservice. + */ +public class DefaultPackageRepository extends PackageRepository { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public DefaultPackageRepository() { // this is needed for deserialization from JSON + + } + + public DefaultPackageRepository(String repositoryName, String repositoryURL) { + this.name = repositoryName; + this.repositoryURL = repositoryURL; + } + + @Override + public void refresh() { + packages = null; + } + + @JsonIgnore + private Map packages; + + @Override + public Map getPackages() { + if (packages == null) { + initPackages(); + } + + return packages; + } + + @Override + public SolrPackage getPackage(String packageName) { + return getPackages().get(packageName); + } + + @Override + public boolean hasPackage(String packageName) { + return getPackages().containsKey(packageName); + } + + @Override + public Path download(String artifactName) throws SolrException, IOException { + Path tmpDirectory = Files.createTempDirectory("solr-packages"); + tmpDirectory.toFile().deleteOnExit(); + URL url = new URL(new URL(repositoryURL), artifactName); + String fileName = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); + Path destination = tmpDirectory.resolve(fileName); + + switch (url.getProtocol()) { + case "http": + case "https": + case "ftp": + FileUtils.copyURLToFile(url, destination.toFile()); + break; + default: + throw new SolrException(ErrorCode.BAD_REQUEST, "URL protocol " + url.getProtocol() + " not supported"); + } + + return destination; + } + + private void initPackages() { + try (CloseableHttpClient client = HttpClientBuilder.create().build()) { + SolrPackage[] items = PackageUtils.getJson(client, repositoryURL + "/repository.json", SolrPackage[].class); + + packages = new HashMap<>(items.length); + for (SolrPackage pkg : items) { + pkg.setRepository(name); + packages.put(pkg.name, pkg); + } + } catch (IOException ex) { + throw new SolrException(ErrorCode.INVALID_STATE, ex); + } + log.debug("Found {} packages in repository '{}'", packages.size(), name); + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java new file mode 100644 index 00000000000..73f50dc0cfa --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import static org.apache.solr.packagemanager.PackageUtils.getMapper; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.common.NavigableObject; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.Utils; +import org.apache.solr.packagemanager.SolrPackage.Command; +import org.apache.solr.packagemanager.SolrPackage.Manifest; +import org.apache.solr.packagemanager.SolrPackage.Plugin; +import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.util.SolrCLI; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Strings; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; + +/** + * Handles most of the management of packages that are already installed in Solr. + */ +public class PackageManager implements Closeable { + + final String solrBaseUrl; + final HttpSolrClient solrClient; + final SolrZkClient zkClient; + + private Map> packages = null; + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + + public PackageManager(HttpSolrClient solrClient, String solrBaseUrl, String zkHost) { + this.solrBaseUrl = solrBaseUrl; + this.solrClient = solrClient; + this.zkClient = new SolrZkClient(zkHost, 30000); + log.info("Done initializing a zkClient instance..."); + } + + @Override + public void close() throws IOException { + if (zkClient != null) { + zkClient.close(); + } + } + + public List fetchInstalledPackageInstances() throws SolrException { + log.info("Getting packages from packages.json..."); + List ret = new ArrayList(); + packages = new HashMap>(); + try { + Map packagesZnodeMap = null; + + if (zkClient.exists("/packages.json", true) == true) { + packagesZnodeMap = (Map)getMapper().readValue( + new String(zkClient.getData("/packages.json", null, null, true), "UTF-8"), Map.class).get("packages"); + for (Object packageName: packagesZnodeMap.keySet()) { + List pkg = (List)packagesZnodeMap.get(packageName); + for (Map pkgVersion: (List)pkg) { + Manifest manifest = PackageUtils.fetchManifest(solrClient, solrBaseUrl, pkgVersion.get("manifest").toString(), pkgVersion.get("manifestSHA512").toString()); + List solrplugins = manifest.plugins; + SolrPackageInstance pkgInstance = new SolrPackageInstance(packageName.toString(), null, + pkgVersion.get("version").toString(), manifest, solrplugins, manifest.parameterDefaults); + List list = packages.containsKey(packageName)? packages.get(packageName): new ArrayList(); + list.add(pkgInstance); + packages.put(packageName.toString(), list); + ret.add(pkgInstance); + } + } + } + } catch (Exception e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + log.info("Got packages: "+ret); + return ret; + } + + public Map getPackagesDeployed(String collection) { + Map packages = null; + try { + NavigableObject result = (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), + solrBaseUrl+"/api/collections/"+collection+"/config/params/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER); + packages = (Map) result._get("/response/params/PKG_VERSIONS", Collections.emptyMap()); + } catch (PathNotFoundException ex) { + // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager. + } + if (packages == null) return Collections.emptyMap(); + Map ret = new HashMap(); + for (String packageName: packages.keySet()) { + if (Strings.isNullOrEmpty(packageName) == false) { // There can be an empty key, storing the version here + ret.put(packageName, getPackageInstance(packageName, packages.get(packageName))); + } + } + return ret; + } + + private boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, boolean noprompt, + List collections, String overrides[]) { + for (String collection: collections) { + + SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageInstance.name); + if (packageInstance.equals(deployedPackage)) { + if (!pegToLatest) { + PackageUtils.printRed("Package " + packageInstance + " already deployed on "+collection); + continue; + } + } else { + if (deployedPackage != null && !isUpdate) { + PackageUtils.printRed("Package " + deployedPackage + " already deployed on "+collection+". To update to "+packageInstance+", pass --update parameter."); + continue; + } + } + + Map collectionParameterOverrides = getCollectionParameterOverrides(packageInstance, isUpdate, overrides, collection); + + // Get package params + try { + boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/abc/config/params/packages", Map.class) + .getOrDefault("response", Collections.emptyMap())).containsKey("params"); + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + getMapper().writeValueAsString(Collections.singletonMap(packageParamsExist? "update": "set", + Collections.singletonMap("packages", Collections.singletonMap(packageInstance.name, collectionParameterOverrides))))); + } catch (Exception e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + + // Set the package version in the collection's parameters + try { + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + + // If updating, refresh the package version for this to take effect + if (isUpdate || pegToLatest) { + try { + SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageInstance.name + "\"}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } + + // If it is a fresh deploy on a collection, run setup commands all the plugins in the package + if (!isUpdate) { + Map systemParams = Map.of("collection", collection, "package-name", packageInstance.name, "package-version", packageInstance.version); + + for (Plugin plugin: packageInstance.plugins) { + Command cmd = plugin.setupCommand; + if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) { + if ("POST".equalsIgnoreCase(cmd.method)) { + try { + String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), packageInstance.parameterDefaults, collectionParameterOverrides, systemParams); + String path = PackageUtils.resolve(cmd.path, packageInstance.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Executing " + payload + " for path:" + path); + boolean shouldExecute = true; + if (!noprompt) { // show a prompt asking user to execute the setup command for the plugin + PackageUtils.print(PackageUtils.YELLOW, "Execute this command (y/n): "); + String userInput = new Scanner(System.in, "UTF-8").next(); + if (!"yes".equalsIgnoreCase(userInput) && !"y".equalsIgnoreCase(userInput)) { + shouldExecute = false; + PackageUtils.printRed("Skipping setup command for deploying (deployment verification may fail)." + + " Please run this step manually or refer to package documentation."); + } + } + if (shouldExecute) { + SolrCLI.postJsonToSolr(solrClient, path, payload); + } + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for setup commands"); + } + } else { + PackageUtils.printRed("There is no setup command to execute for plugin: " + plugin.name); + } + } + } + + // Set the package version in the collection's parameters + try { + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } + + // Verify that package was successfully deployed + boolean success = verify(packageInstance, collections); + if (success) { + PackageUtils.printGreen("Deployed and verified package: " + packageInstance.name + ", version: " + packageInstance.version); + } + return success; + } + + private Map getCollectionParameterOverrides(SolrPackageInstance packageInstance, boolean isUpdate, + String[] overrides, String collection) { + Map collectionParameterOverrides = isUpdate? getPackageParams(packageInstance.name, collection): new HashMap(); + if (overrides != null) { + for (String override: overrides) { + collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]); + } + } + return collectionParameterOverrides; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + Map getPackageParams(String packageName, String collection) { + try { + return (Map)((Map)((Map)((Map) + PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/packages", Map.class) + .get("response")) + .get("params")) + .get("packages")).get(packageName); + } catch (Exception ex) { + // This should be because there are no parameters. Be tolerant here. + return Collections.emptyMap(); + } + } + + /** + * Given a package and list of collections, verify if the package is installed + * in those collections. It uses the verify command of every plugin in the package (if defined). + */ + public boolean verify(SolrPackageInstance pkg, List collections) { + boolean success = true; + for (Plugin plugin: pkg.plugins) { + PackageUtils.printGreen(plugin.verifyCommand); + for (String collection: collections) { + Map collectionParameterOverrides = getPackageParams(pkg.name, collection); + Command cmd = plugin.verifyCommand; + + Map systemParams = Map.of("collection", collection, "package-name", pkg.name, "package-version", pkg.version); + String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Executing " + url + " for collection:" + collection); + + if ("GET".equalsIgnoreCase(cmd.method)) { + String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url); + PackageUtils.printGreen(response); + String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration()) + .read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams)); + String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue); + if (!expectedValue.equals(actualValue)) { + PackageUtils.printRed("Failed to deploy plugin: " + plugin.name); + success = false; + } + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for setup commands"); + } + } + } + return success; + } + + /** + * Get the installed instance of a specific version of a package. If version is null, PackageUtils.LATEST or PackagePluginHolder.LATEST, + * then it returns the highest version available in the system for the package. + */ + public SolrPackageInstance getPackageInstance(String packageName, String version) { + fetchInstalledPackageInstances(); + List versions = packages.get(packageName); + SolrPackageInstance latest = null; + if (versions != null && !versions.isEmpty()) { + latest = versions.get(0); + for (int i=0; i collectionParameterOverrides = getPackageParams(packageName, collection); + + // Run the uninstall command for all plugins + Map systemParams = Map.of("collection", collection, "package-name", deployedPackage.name, "package-version", deployedPackage.version); + + for (Plugin plugin: deployedPackage.plugins) { + Command cmd = plugin.uninstallCommand; + if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) { + if ("POST".equalsIgnoreCase(cmd.method)) { + try { + String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams); + String path = PackageUtils.resolve(cmd.path, deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams); + PackageUtils.printGreen("Executing " + payload + " for path:" + path); + SolrCLI.postJsonToSolr(solrClient, path, payload); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for uninstall commands"); + } + } else { + PackageUtils.printRed("There is no uninstall command to execute for plugin: " + plugin.name); + } + } + + // Set the package version in the collection's parameters + try { + SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", "{set: {PKG_VERSIONS: {"+packageName+": null}}}"); + SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageName + "\"}"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.SERVER_ERROR, ex); + } + + // TODO: Also better to remove the package parameters + } + } + + /** + * Given a package, return a map of collections where this package is + * installed to the installed version (which can be {@link PackagePluginHolder#LATEST}) + */ + public Map getDeployedCollections(String packageName) { + List allCollections; + try { + allCollections = zkClient.getChildren("/collections", null, true); + } catch (KeeperException | InterruptedException e) { + throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e); + } + Map deployed = new HashMap(); + for (String collection: allCollections) { + // Check package version installed + String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/PKG_VERSIONS?omitHeader=true"); + String version = null; + try { + version = JsonPath.parse(paramsJson, PackageUtils.jsonPathConfiguration()) + .read("$['response'].['params'].['PKG_VERSIONS'].['"+packageName+"'])"); + } catch (PathNotFoundException ex) { + // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager. + } + if (version != null) { + deployed.put(collection, version); + } + } + return deployed; + } + +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java new file mode 100644 index 00000000000..43b4ada23a1 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageRepository.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.packagemanager; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; + +/** + * Abstract class for a repository, holding {@link SolrPackage} items. + */ +public abstract class PackageRepository { + + @JsonProperty("name") + public String name = null; + + @JsonProperty("url") + public String repositoryURL = null; + + public abstract void refresh(); + + /** + * Returns a map of package name to {@link SolrPackage}s. + */ + public abstract Map getPackages(); + + public abstract SolrPackage getPackage(String packageName); + + public abstract boolean hasPackage(String packageName); + + /** + * Provides a method to download an artifact from this repository. + */ + public abstract Path download(String artifactName) throws SolrException, IOException; + +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java new file mode 100644 index 00000000000..77ae057ca86 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.packagemanager; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.response.V2Response; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.BlobRepository; +import org.apache.solr.packagemanager.SolrPackage.Manifest; +import org.apache.solr.util.SolrJacksonAnnotationInspector; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.zafarkhaja.semver.Version; +import com.google.common.base.Strings; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import com.jayway.jsonpath.spi.mapper.MappingProvider; + +public class PackageUtils { + + /** + * Represents a version which denotes the latest version available at the moment. + */ + public static String LATEST = "latest"; + + public static Configuration jsonPathConfiguration() { + MappingProvider provider = new JacksonMappingProvider(); + JsonProvider jsonProvider = new JacksonJsonProvider(); + Configuration c = Configuration.builder().jsonProvider(jsonProvider).mappingProvider(provider).options(com.jayway.jsonpath.Option.REQUIRE_PROPERTIES).build(); + return c; + } + + public static ObjectMapper getMapper() { + return new ObjectMapper().setAnnotationIntrospector(new SolrJacksonAnnotationInspector()); + } + + /** + * Uploads a file to the package store / file store of Solr. + * + * @param client A Solr client + * @param buffer File contents + * @param name Name of the file as it will appear in the file store (can be hierarchical) + * @param sig Signature digest (public key should be separately uploaded to ZK) + */ + public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig) + throws SolrServerException, IOException { + String resource = "/api/cluster/files" + name; + ModifiableSolrParams params = new ModifiableSolrParams(); + if (sig != null) { + params.add("sig", sig); + } + V2Response rsp = new V2Request.Builder(resource) + .withMethod(SolrRequest.METHOD.PUT) + .withPayload(buffer) + .forceV2(true) + .withMimeType("application/octet-stream") + .withParams(params) + .build() + .process(client); + if (!name.equals(rsp.getResponse().get(CommonParams.FILE))) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Mismatch in file uploaded. Uploaded: " + + rsp.getResponse().get(CommonParams.FILE)+", Original: "+name); + } + } + + /** + * Download JSON from the url and deserialize into klass. + */ + public static T getJson(HttpClient client, String url, Class klass) { + try { + return getMapper().readValue(getJsonStringFromUrl(client, url), klass); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Search through the list of jar files for a given file. Returns string of + * the file contents or null if file wasn't found. This is suitable for looking + * for manifest or property files within pre-downloaded jar files. + * Please note that the first instance of the file found is returned. + */ + public static String getFileFromJarsAsString(List jars, String filename) { + for (Path jarfile: jars) { + try (ZipFile zipFile = new ZipFile(jarfile.toFile())) { + ZipEntry entry = zipFile.getEntry(filename); + if (entry == null) continue; + return IOUtils.toString(zipFile.getInputStream(entry), "UTF-8"); + } catch (Exception ex) { + throw new SolrException(ErrorCode.BAD_REQUEST, ex); + } + } + return null; + } + + /** + * Returns JSON string from a given URL + */ + public static String getJsonStringFromUrl(HttpClient client, String url) { + try { + return IOUtils.toString(client.execute(new HttpGet(url)).getEntity().getContent(), "UTF-8"); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Checks whether a given version satisfies the constraint (defined by a semver expression) + */ + public static boolean checkVersionConstraint(String ver, String constraint) { + return Strings.isNullOrEmpty(constraint) || Version.valueOf(ver).satisfies(constraint); + } + + /** + * Fetches a manifest file from the File Store / Package Store. A SHA512 check is enforced after fetching. + */ + public static Manifest fetchManifest(HttpSolrClient solrClient, String solrBaseUrl, String manifestFilePath, String expectedSHA512) throws MalformedURLException, IOException { + String manifestJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/node/files" + manifestFilePath); + String calculatedSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8"))); + if (expectedSHA512.equals(calculatedSHA512) == false) { + throw new SolrException(ErrorCode.UNAUTHORIZED, "The manifest SHA512 doesn't match expected SHA512. Possible unauthorized manipulation. " + + "Expected: " + expectedSHA512 + ", calculated: " + calculatedSHA512 + ", manifest location: " + manifestFilePath); + } + Manifest manifest = getMapper().readValue(manifestJson, Manifest.class); + return manifest; + } + + /** + * Replace a templatized string with parameter substituted string. First applies the overrides, then defaults and then systemParams. + */ + public static String resolve(String str, Map defaults, Map overrides, Map systemParams) { + // TODO: Should perhaps use Matchers etc. instead of this clumsy replaceAll(). + + if (str == null) return null; + for (String param: defaults.keySet()) { + str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param)); + } + for (String param: overrides.keySet()) { + str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param)); + } + for (String param: systemParams.keySet()) { + str = str.replaceAll("\\$\\{"+param+"\\}", systemParams.get(param)); + } + return str; + } + + /** + * Compares two versions v1 and v2. Returns negative if v1 isLessThan v2, positive if v1 isGreaterThan v2 and 0 if equal. + */ + public static int compareVersions(String v1, String v2) { + return Version.valueOf(v1).compareTo(Version.valueOf(v2)); + } + + public static String BLACK = "\u001B[30m"; + public static String RED = "\u001B[31m"; + public static String GREEN = "\u001B[32m"; + public static String YELLOW = "\u001B[33m"; + public static String BLUE = "\u001B[34m"; + public static String PURPLE = "\u001B[35m"; + public static String CYAN = "\u001B[36m"; + public static String WHITE = "\u001B[37m"; + + /** + * Console print using green color + */ + public static void printGreen(Object message) { + PackageUtils.print(PackageUtils.GREEN, message); + } + + /** + * Console print using red color + */ + public static void printRed(Object message) { + PackageUtils.print(PackageUtils.RED, message); + } + + public static void print(Object message) { + print(null, message); + } + + @SuppressForbidden(reason = "Need to use System.out.println() instead of log4j/slf4j for cleaner output") + public static void print(String color, Object message) { + String RESET = "\u001B[0m"; + + if (color != null) { + System.out.println(color + String.valueOf(message) + RESET); + } else { + System.out.println(message); + } + } + + public static String[] validateCollections(String collections[]) { + String collectionNameRegex = "^[a-zA-Z0-9_-]*$"; + for (String c: collections) { + if (c.matches(collectionNameRegex) == false) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid collection name: " + c + + ". Didn't match the pattern: '"+collectionNameRegex+"'"); + } + } + return collections; + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java new file mode 100644 index 00000000000..5dd503ebe0c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import static org.apache.solr.packagemanager.PackageUtils.getMapper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.util.Version; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.request.beans.Package; +import org.apache.solr.client.solrj.response.V2Response; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.core.BlobRepository; +import org.apache.solr.packagemanager.SolrPackage.Artifact; +import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; +import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handles most of the management of repositories and packages present in external repositories. + */ +public class RepositoryManager { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + final private PackageManager packageManager; + + public static final String systemVersion = Version.LATEST.toString(); + + final HttpSolrClient solrClient; + + public RepositoryManager(HttpSolrClient solrClient, PackageManager packageManager) { + this.packageManager = packageManager; + this.solrClient = solrClient; + } + + public List getPackages() { + List list = new ArrayList<>(getPackagesMap().values()); + Collections.sort(list); + return list; + } + + /** + * Get a map of package name to {@link SolrPackage} objects + */ + public Map getPackagesMap() { + Map packagesMap = new HashMap<>(); + for (PackageRepository repository: getRepositories()) { + packagesMap.putAll(repository.getPackages()); + } + + return packagesMap; + } + + /** + * List of added repositories + */ + public List getRepositories() { + // TODO: Instead of fetching again and again, we should look for caching this + PackageRepository items[]; + try { + items = getMapper().readValue(getRepositoriesJson(packageManager.zkClient), DefaultPackageRepository[].class); + } catch (IOException | KeeperException | InterruptedException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + List repositories = Arrays.asList(items); + + for (PackageRepository updateRepository: repositories) { + updateRepository.refresh(); + } + + return repositories; + } + + /** + * Add a repository to Solr + */ + public void addRepository(String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException { + String existingRepositoriesJson = getRepositoriesJson(packageManager.zkClient); + log.info(existingRepositoriesJson); + + List repos = getMapper().readValue(existingRepositoriesJson, List.class); + repos.add(new DefaultPackageRepository(name, uri)); + if (packageManager.zkClient.exists("/repositories.json", true) == false) { + packageManager.zkClient.create("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true); + } else { + packageManager.zkClient.setData("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), true); + } + + if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe/"+name+".der", true)==false) { + packageManager.zkClient.create("/keys/exe/"+name+".der", new byte[0], CreateMode.PERSISTENT, true); + } + packageManager.zkClient.setData("/keys/exe/"+name+".der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true); + } + + private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException { + if (zkClient.exists("/repositories.json", true)) { + return new String(zkClient.getData("/repositories.json", null, null, true), "UTF-8"); + } + return "[]"; + } + + /** + * Install a given package and version from the available repositories to Solr. + * The various steps for doing so are, briefly, a) find upload a manifest to package store, + * b) download the artifacts and upload to package store, c) call {@link PackageAPI} to register + * the package. + */ + private boolean installPackage(String packageName, String version) throws SolrException { + SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); + if (existingPlugin != null && existingPlugin.version.equals(version)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Plugin already installed."); + } + + SolrPackageRelease release = getPackageRelease(packageName, version); + List downloaded = downloadPackageArtifacts(packageName, version); + // TODO: Should we introduce a checksum to validate the downloading? + // Currently, not a big problem since signature based checking happens anyway + + try { + // post the manifest + PackageUtils.printGreen("Posting manifest..."); + + if (release.manifest == null) { + String manifestJson = PackageUtils.getFileFromJarsAsString(downloaded, "manifest.json"); + if (manifestJson == null) { + throw new SolrException(ErrorCode.NOT_FOUND, "No manifest found for package: " + packageName + ", version: " + version); + } + release.manifest = getMapper().readValue(manifestJson, SolrPackage.Manifest.class); + } + String manifestJson = getMapper().writeValueAsString(release.manifest); + String manifestSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8"))); + PackageUtils.postFile(solrClient, ByteBuffer.wrap(manifestJson.getBytes("UTF-8")), + String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json"), null); + + // post the artifacts + PackageUtils.printGreen("Posting artifacts..."); + for (int i=0; i String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, file.getFileName().toString())).collect(Collectors.toList()); + add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; + add.manifestSHA512 = manifestSHA512; + + V2Request req = new V2Request.Builder("/api/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + + try { + V2Response resp = req.process(solrClient); + PackageUtils.printGreen("Response: "+resp.jsonStr()); + } catch (SolrServerException | IOException e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + + } catch (SolrServerException | IOException e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + return false; + } + + private List downloadPackageArtifacts(String packageName, String version) throws SolrException { + try { + SolrPackageRelease release = getPackageRelease(packageName, version); + List downloadedPaths = new ArrayList(release.artifacts.size()); + + for (PackageRepository repo: getRepositories()) { + if (repo.hasPackage(packageName)) { + for (Artifact art: release.artifacts) { + downloadedPaths.add(repo.download(art.url)); + } + return downloadedPaths; + } + } + } catch (IOException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Error during download of package " + packageName, e); + } + throw new SolrException(ErrorCode.NOT_FOUND, "Package not found in any repository."); + } + + /** + * Given a package name and version, find the release/version object as found in the repository + */ + private SolrPackageRelease getPackageRelease(String packageName, String version) throws SolrException { + SolrPackage pkg = getPackagesMap().get(packageName); + if (pkg == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Package "+packageName+" not found in any repository"); + } + if (version == null || PackageUtils.LATEST.equals(version)) { + return getLastPackageRelease(pkg); + } + for (SolrPackageRelease release : pkg.versions) { + if (PackageUtils.compareVersions(version, release.version) == 0) { + return release; + } + } + throw new SolrException(ErrorCode.BAD_REQUEST, "Package " + packageName + ":" + version + " does not exist in any repository."); + } + + public SolrPackageRelease getLastPackageRelease(String packageName) { + SolrPackage pkg = getPackagesMap().get(packageName); + if (pkg == null) { + return null; + } + return getLastPackageRelease(pkg); + } + + private SolrPackageRelease getLastPackageRelease(SolrPackage pkg) { + SolrPackageRelease latest = null; + for (SolrPackageRelease release: pkg.versions) { + if (latest == null) { + latest = release; + } else { + if (PackageUtils.compareVersions(latest.version, release.version) < 0) { + latest = release; + } + } + } + return latest; + } + + /** + * Is there a version of the package available in the repositories that is more + * latest than our latest installed version of the package? + */ + public boolean hasPackageUpdate(String packageName) { + SolrPackage pkg = getPackagesMap().get(packageName); + if (pkg == null) { + return false; + } + String installedVersion = packageManager.getPackageInstance(packageName, null).version; + SolrPackageRelease last = getLastPackageRelease(packageName); + return last != null && PackageUtils.compareVersions(last.version, installedVersion) > 0; + } + + /** + * Install a version of the package. Also, run verify commands in case some + * collection was using {@link PackagePluginHolder#LATEST} version of this package and got auto-updated. + */ + public void install(String packageName, String version) throws SolrException { + String latestVersion = getLastPackageRelease(packageName).version; + + Map collectionsDeployedIn = packageManager.getDeployedCollections(packageName); + List peggedToLatest = collectionsDeployedIn.keySet().stream(). + filter(collection -> collectionsDeployedIn.get(collection).equals(PackagePluginHolder.LATEST)).collect(Collectors.toList()); + if (!peggedToLatest.isEmpty()) { + PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+peggedToLatest); + } + + if (version == null || version.equals(PackageUtils.LATEST)) { + installPackage(packageName, latestVersion); + } else { + installPackage(packageName, version); + } + + SolrPackageInstance updatedPackage = packageManager.getPackageInstance(packageName, PackageUtils.LATEST); + boolean res = packageManager.verify(updatedPackage, peggedToLatest); + PackageUtils.printGreen("Verifying version " + updatedPackage.version + + " on " + peggedToLatest + ", result: " + res); + if (!res) throw new SolrException(ErrorCode.BAD_REQUEST, "Failed verification after deployment"); + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java new file mode 100644 index 00000000000..eaa4334107f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.util.ReflectMapWriter; + +/** + * Describes a package (along with all released versions) as it appears in a repository. + */ +public class SolrPackage implements Comparable, ReflectMapWriter { + + @JsonProperty("name") + public String name; + + @JsonProperty("description") + public String description; + + @JsonProperty("versions") + public List versions; + + @JsonProperty("repository") + private String repository; + + @Override + public String toString() { + return jsonStr(); + } + + public static class SolrPackageRelease implements ReflectMapWriter { + @JsonProperty("version") + public String version; + + @JsonProperty("date") + public Date date; + + @JsonProperty("artifacts") + public List artifacts; + + @JsonProperty("manifest") + public Manifest manifest; + + @Override + public String toString() { + return jsonStr(); + } + } + + public static class Artifact implements ReflectMapWriter { + @JsonProperty("url") + public String url; + + @JsonProperty("sig") + public String sig; + } + + public static class Manifest implements ReflectMapWriter { + @JsonProperty("version-constraint") + public String versionConstraint; + + @JsonProperty("plugins") + public List plugins; + + @JsonProperty("parameter-defaults") + public Map parameterDefaults; + } + + public static class Plugin implements ReflectMapWriter { + public String name; + @JsonProperty("setup-command") + public Command setupCommand; + + @JsonProperty("uninstall-command") + public Command uninstallCommand; + + @JsonProperty("verify-command") + public Command verifyCommand; + + @Override + public String toString() { + return jsonStr(); + } + } + + @Override + public int compareTo(SolrPackage o) { + return name.compareTo(o.name); + } + + public String getRepository() { + return repository; + } + + public void setRepository(String repository) { + this.repository = repository; + } + + public static class Command implements ReflectMapWriter { + @JsonProperty("path") + public String path; + + @JsonProperty("method") + public String method; + + @JsonProperty("payload") + public Map payload; + + @JsonProperty("condition") + public String condition; + + @JsonProperty("expected") + public String expected; + + @Override + public String toString() { + return jsonStr(); + } + } +} + diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java new file mode 100644 index 00000000000..f913bfb22c3 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import java.util.List; +import java.util.Map; + +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.packagemanager.SolrPackage.Manifest; +import org.apache.solr.packagemanager.SolrPackage.Plugin; + +/** + * Describes one instance of a package as it exists in Solr when installed. + */ +public class SolrPackageInstance implements ReflectMapWriter { + @JsonProperty("name") + final public String name; + + final public String description; + + @JsonProperty("version") + final public String version; + + final public Manifest manifest; + + final public List plugins; + + final public Map parameterDefaults; + + public SolrPackageInstance(String id, String description, String version, Manifest manifest, + List plugins, Map params) { + this.name = id; + this.description = description; + this.version = version; + this.manifest = manifest; + this.plugins = plugins; + this.parameterDefaults = params; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) return false; + return name.equals(((SolrPackageInstance)obj).name) && version.equals(((SolrPackageInstance)obj).version); + } + + @Override + public String toString() { + return jsonStr(); + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/package-info.java b/solr/core/src/java/org/apache/solr/packagemanager/package-info.java new file mode 100644 index 00000000000..66f135a2715 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/packagemanager/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains Package Manager (CLI) implementation + */ +package org.apache.solr.packagemanager; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 8327d3662f9..3b005445dac 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -180,12 +180,20 @@ public class PackageAPI { @JsonProperty public List files; + @JsonProperty + public String manifest; + + @JsonProperty + public String manifestSHA512; + public PkgVersion() { } public PkgVersion(Package.AddVersion addVersion) { this.version = addVersion.version; this.files = addVersion.files; + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; } @Override diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java new file mode 100644 index 00000000000..8464ec14597 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util; + +import static org.apache.solr.packagemanager.PackageUtils.printGreen; +import static org.apache.solr.packagemanager.PackageUtils.print; + +import java.lang.invoke.MethodHandles; +import java.util.Map; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.Pair; +import org.apache.solr.packagemanager.PackageManager; +import org.apache.solr.packagemanager.PackageUtils; +import org.apache.solr.packagemanager.RepositoryManager; +import org.apache.solr.packagemanager.SolrPackage; +import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; +import org.apache.solr.packagemanager.SolrPackageInstance; +import org.apache.solr.util.SolrCLI.StatusTool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class PackageTool extends SolrCLI.ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.") + public PackageTool() { + // Need a logging free, clean output going through to the user. + Configurator.setRootLevel(Level.OFF); + } + + @Override + public String getName() { + return "package"; + } + + public static String solrUrl = null; + public static String solrBaseUrl = null; + public PackageManager packageManager; + public RepositoryManager repositoryManager; + + @Override + @SuppressForbidden(reason = "We really need to print the stacktrace here, otherwise " + + "there shall be little else information to debug problems. Other SolrCLI tools " + + "don't print stack traces, hence special treatment is needed here.") + protected void runImpl(CommandLine cli) throws Exception { + try { + solrUrl = cli.getOptionValues("solrUrl")[cli.getOptionValues("solrUrl").length-1]; + solrBaseUrl = solrUrl.replaceAll("\\/solr$", ""); // strip out ending "/solr" + log.info("Solr url: "+solrUrl+", solr base url: "+solrBaseUrl); + String zkHost = getZkHost(cli); + + log.info("ZK: "+zkHost); + String cmd = cli.getArgList().size() == 0? "help": cli.getArgs()[0]; + + try (HttpSolrClient solrClient = new HttpSolrClient.Builder(solrBaseUrl).build()) { + if (cmd != null) { + packageManager = new PackageManager(solrClient, solrBaseUrl, zkHost); + try { + repositoryManager = new RepositoryManager(solrClient, packageManager); + + switch (cmd) { + case "add-repo": + String repoName = cli.getArgs()[1]; + String repoUrl = cli.getArgs()[2]; + repositoryManager.addRepository(repoName, repoUrl); + PackageUtils.printGreen("Added repository: " + repoName); + break; + case "list-installed": + PackageUtils.printGreen("Installed packages:\n-----"); + for (SolrPackageInstance pkg: packageManager.fetchInstalledPackageInstances()) { + PackageUtils.printGreen(pkg); + } + break; + case "list-available": + PackageUtils.printGreen("Available packages:\n-----"); + for (SolrPackage pkg: repositoryManager.getPackages()) { + PackageUtils.printGreen(pkg.name + " \t\t"+pkg.description); + for (SolrPackageRelease version: pkg.versions) { + PackageUtils.printGreen("\tVersion: "+version.version); + } + } + break; + case "list-deployed": + if (cli.hasOption('c')) { + String collection = cli.getArgs()[1]; + Map packages = packageManager.getPackagesDeployed(collection); + PackageUtils.printGreen("Packages deployed on " + collection + ":"); + for (String packageName: packages.keySet()) { + PackageUtils.printGreen("\t" + packages.get(packageName)); + } + } else { + String packageName = cli.getArgs()[1]; + Map deployedCollections = packageManager.getDeployedCollections(packageName); + PackageUtils.printGreen("Collections on which package " + packageName + " was deployed:"); + for (String collection: deployedCollections.keySet()) { + PackageUtils.printGreen("\t" + collection + "("+packageName+":"+deployedCollections.get(collection)+")"); + } + } + break; + case "install": + { + Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString()); + String packageName = parsedVersion.first(); + String version = parsedVersion.second(); + repositoryManager.install(packageName, version); + PackageUtils.printGreen(repositoryManager.toString() + " installed."); + break; + } + case "deploy": + { + Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString()); + String packageName = parsedVersion.first(); + String version = parsedVersion.second(); + boolean noprompt = cli.hasOption('y'); + boolean isUpdate = cli.hasOption("update") || cli.hasOption('u'); + packageManager.deploy(packageName, version, PackageUtils.validateCollections(cli.getOptionValues("collections")), cli.getOptionValues("param"), isUpdate, noprompt); + break; + } + case "undeploy": + { + Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString()); + if (parsedVersion.second() != null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Only package name expected, without a version. Actual: " + cli.getArgList().get(1)); + } + String packageName = parsedVersion.first(); + packageManager.undeploy(packageName, cli.getOptionValues("collections")); + break; + } + case "help": + case "usage": + print("Package Manager\n---------------"); + printGreen("./solr package add-repo "); + print("Add a repository to Solr."); + print(""); + printGreen("./solr package install [:] "); + print("Install a package into Solr. This copies over the artifacts from the repository into Solr's internal package store and sets up classloader for this package to be used."); + print(""); + printGreen("./solr package deploy [:] [-y] [--update] -collections [-p = -p = ...] "); + print("Bootstraps a previously installed package into the specified collections. It the package accepts parameters for its setup commands, they can be specified (as per package documentation)."); + print(""); + printGreen("./solr package list-installed"); + print("Print a list of packages installed in Solr."); + print(""); + printGreen("./solr package list-available"); + print("Print a list of packages available in the repositories."); + print(""); + printGreen("./solr package list-deployed -c "); + print("Print a list of packages deployed on a given collection."); + print(""); + printGreen("./solr package list-deployed "); + print("Print a list of collections on which a given package has been deployed."); + print(""); + printGreen("./solr package undeploy -collections "); + print("Undeploys a package from specified collection(s)"); + print("\n"); + print("Note: (a) Please add '-solrUrl http://host:port' parameter if needed (usually on Windows)."); + print(" (b) Please make sure that all Solr nodes are started with '-Denable.packages=true' parameter."); + print("\n"); + break; + default: + throw new RuntimeException("Unrecognized command: "+cmd); + }; + } finally { + packageManager.close(); + } + } + } + log.info("Finished: "+cmd); + + } catch (Exception ex) { + ex.printStackTrace(); // We need to print this since SolrCLI drops the stack trace in favour of brevity. Package tool should surely print full stacktraces! + throw ex; + } + } + + /** + * Parses package name and version in the format "name:version" or "name" + * @return A pair of package name (first) and version (second) + */ + private Pair parsePackageVersion(String arg) { + String splits[] = arg.split(":"); + if (splits.length > 2) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid package name: " + arg + + ". Didn't match the pattern: : or "); + } + + String packageName = splits[0]; + String version = splits.length == 2? splits[1]: null; + return new Pair(packageName, version); + } + + @SuppressWarnings("static-access") + public Option[] getOptions() { + return new Option[] { + OptionBuilder + .withArgName("URL") + .hasArg() + .isRequired(true) + .withDescription("Address of the Solr Web application, defaults to: " + SolrCLI.DEFAULT_SOLR_URL) + .create("solrUrl"), + + OptionBuilder + .withArgName("COLLECTIONS") + .hasArgs() + .isRequired(false) + .withDescription("List of collections. Run './solr package help' for more details.") + .create("collections"), + + OptionBuilder + .withArgName("PARAMS") + .hasArgs() + .isRequired(false) + .withDescription("List of parameters to be used with deploy command. Run './solr package help' for more details.") + .withLongOpt("param") + .create("p"), + + OptionBuilder + .isRequired(false) + .withDescription("If a deployment is an update over a previous deployment. Run './solr package help' for more details.") + .withLongOpt("update") + .create("u"), + + OptionBuilder + .isRequired(false) + .withDescription("Run './solr package help' for more details.") + .withLongOpt("collection") + .create("c"), + + OptionBuilder + .isRequired(false) + .withDescription("Run './solr package help' for more details.") + .withLongOpt("noprompt") + .create("y") + }; + } + + private String getZkHost(CommandLine cli) throws Exception { + String zkHost = cli.getOptionValue("zkHost"); + if (zkHost != null) + return zkHost; + + String systemInfoUrl = solrUrl+"/admin/info/system"; + CloseableHttpClient httpClient = SolrCLI.getHttpClient(); + try { + // hit Solr to get system info + Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); + + // convert raw JSON into user-friendly output + StatusTool statusTool = new StatusTool(); + Map status = statusTool.reportStatus(solrUrl+"/", systemInfo, httpClient); + Map cloud = (Map)status.get("cloud"); + if (cloud != null) { + String zookeeper = (String) cloud.get("ZooKeeper"); + if (zookeeper.endsWith("(embedded)")) { + zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length()); + } + zkHost = zookeeper; + } + } finally { + HttpClientUtil.close(httpClient); + } + + return zkHost; + } + +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index d077d7d4e05..3feb0daf3c7 100755 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -419,6 +419,8 @@ public class SolrCLI implements CLIO { return new AutoscalingTool(); else if ("export".equals(toolType)) return new ExportTool(); + else if ("package".equals(toolType)) + return new PackageTool(); // If you add a built-in tool to this class, add it here to avoid // classpath scanning diff --git a/solr/core/src/test-files/solr/question-answer-repository-private-key.pem b/solr/core/src/test-files/solr/question-answer-repository-private-key.pem new file mode 100644 index 00000000000..d755d2d5c97 --- /dev/null +++ b/solr/core/src/test-files/solr/question-answer-repository-private-key.pem @@ -0,0 +1,9 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIBPAIBAAJBAOkIQ4lxVDPPIR2noh4MrTVw3JASYXd6k0wajrSDFYlwepVoSO/f +LK1AHYomrub8C8dXZwxzPMqpDVQ4MABdgcECAwEAAQJBAKzsLPG43zry4SgYVPzn +e0DE12cxvJHkq5k1u9/HxhuNqUdfhKzkMW5BqayDT74UqSLLTT9zFpGPsRUHD37V +aAECIQD7HV0vP9ipMskCMA+x8H8X/F2V2b6RWGlfIDc6XEiE8QIhAO2Q197gMnzP +z3VluoeL9pORw55aqx7kuAXwqmUqRInRAiBX3wWdpBTX2EqYdmL3nDWNGiVRa5mQ +2MQ+olJRHLvPsQIhAL0w/LmiEpMTbEQyH7qS3GvpScBytJSF0YfpgcnPP4YBAiEA +14io0rN1QgVWykmckeZZSzaPLqpkE2wCUZUOemoBByg= +-----END RSA PRIVATE KEY----- diff --git a/solr/core/src/test-files/solr/question-answer-repository/publickey.der b/solr/core/src/test-files/solr/question-answer-repository/publickey.der new file mode 100644 index 0000000000000000000000000000000000000000..8ca82e87862ccc0a0cacc08ebb07498f6dfcea7a GIT binary patch literal 94 zcmV-k0HOadTrdp=2`Yw2hW8Bt0RaU714{rfNCH6s=?FuKaa1$UAswfp9t^EDaNLj* zVRw3yOd5`~gB6KzdX;EM@82w~Kpl!EuIBs;$5&?zb3Dqa4OBQV09}E>0s{d60da00 AzW@LL literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..46394e4e3410aacf456c77e639ef183f96381773 GIT binary patch literal 5652 zcmbtYbzD?y^IuSK>0AkEK|&VkuBBUINoiPMap?|`5{V@wBvo2cB&54mQc+TnSdbEg z1yMkOA6&0;-|PL|_m6k>bIxW~;@zqf=iXF9QHf|G}pN(S^Br!8|-w ztekBTFn1msgq4TKBSSw+$`7EEJ5!o={4@z=*v}$)?%P&RJ&!K(45I`>bmg2wvA|m1 z?0xNacIoMC;o}Bisf`c%BsCytbPK4PD9v#*M>oLo_SU!7t%nS`tKaOrliu5k5FJ-m z&wZW!8n@Ll7qGj0@b;!8jwNRjTa9IWu#Xx?$76TFMYF6O=O}|MPSlWZMdVjiUm0np z`7mnJ&k0+!a71#R8s zGFQ>YWUrz`dC&takEt*qG7At!h|Ge@O|JWtg&CcRZBDrjj@YTyFOPOAEgWVNS#NY4 zXQv#{1>^Fq@jHm+!85D$qVf@7GWZ+rrnrj|HPU94dWHSZ5p)55dM;cKo+G;GqHD?> z4Sn*V1LjXxG2xCpbV5dTp`$=EqD}f(@DQQ2p{m@-Hx~NZ;7Jhz^$WweBLP?UYL z-X6c=R^?-mR9HGx9KtZR0re;6q7t@N%T?zSYJEl8;@ze+{6r>7#I|z)%o`r688Jnr zlc-i^5o@H>iR?y(V9EPbjFk}C|#OPu|qk9`L^eTg^V2!wXTv|xkXtrf!&d@qdc z-A1dDV46K4h?_<4bJYkuf1f`noKe73ut2vUas#RvlgO1vkZJq) zt=4Kqdg{@QW7)B|l3)tyv1NF%)KybG*FMc&TZ2wX{@p6w^b|gbW$s`l>(Ndb_hMX7!AweHuZ*M~!1EC7MNqJSTQjn&ug0E~pTItnzy^R7( zU?qM8f$9_MOM+joNoBylqCmHZs9T-|`M2Q+jONf;!=BGk6H-TsrVhUT?pKoBYRXpK zFkMI+dBt%Y6JD#w7B}jmosWdvDeZBB`S{gZtLav!C=fdxv)09uIUtf|-9ONXJ@XrE zi0WTn7L$Ik)nJS1AzT5W1YT5NjE4n9%x{kFKp$n4m!wr5>qN~%uUuDzm8lDn7gFvp zPd8?XJ){Nq>Mp>tlI?vyLVMsYZ+BNDX~Vgl>ee;vW5W?_=}@no zqS}Nu3YGLiV?!pQp4nJRO8>p_D}k$7%3Cv0YXa|^wy%0E()Hn5lPfDsyQ+qxRJ|w| zUOor}kgF_H?e;lEmx820&_vsq;t_LI+Yt~Chmj|430S;LOL7x%PvZ@xXhF?3uR|X2 zNy;O{PJ+dX2C+eP zHp*MO>ls^1eZPNeO%y7I^^EX30on?`7A5@{-!s|zDO~X7H=y+@zSb9*@JL;+>Nis@ zi!9@!4G|lLJ0^7ijvU6=RLfOQ?<+dP_oYk-6!OxOA7{eruo+CS1mwB|6Oh^Y0o0FC z3sOrePB~APR!x96E-%oU^nLO(jS+b(v$%U($x!0alKq{W46EdbHyjd#9qffV(uG6H z{=uYbI`RznbfQd@KXM!L=cQCoeoiotphkCJJz^j^;7RJEQet|L2^xAKeZX5-z$ayz z*=wA0b(^7!Fd~5278#$NNPHM*|I|%uh;PtV#m0cPy03ulQ-hf4$tqTE7K(H)&Qhyx zP6D}`R}fU;;!I1W;Gu7!(s|79U$1*ywZh@c%~e19nv@1~gytKxnT*K`6ne4%=Z<{% zZJ=SYvt@GSMm+wfPq{_OV@;7C!7`y-hF&!+D7%DRR{#BygA(gO|6-SCUsb~!rGkpD z#%5?L0PTSwW4Utm?3Xak7y;#QK^EfjUZ2V~!9dDXI{*DKAyas|2h;7ls$Yg%)y>}0 zzc;t2&ubd3`|L%N6Bt0}ITH?$x^XYEARIXW1Q}XNKP1~(e~vqlgH|IXwezuJbMEGv zxj!};?=KmQOV_UNi~G?PJJy%%<62A+9{b4PcLyuxnkw;{N+Jh)eDNt^gzYqZd!jOZ z6Lr~}dg8gg8(0Fey6B*h8@lAe9FPe-VXvfUsVQMPx-@b>NRB6mH%8wCKG|CuYB2qP zsU~_u^k7Q9j@FWIl^3|zG&z>m~6!7LDoa#S_CV<%yQu9?Wi$JtB1mQSZdVdXogxyn_e*YrE!8+&GJ zxA>4E3|;-?2wIymu3jTEq>cr$vXDA2L}2`~rKvu$^4aLbYJeVA@1ZraFtFDsjf&z+ zaMJ=x%1AEOR%)+aH0Q*BCcf!8N&Gy74kiB?=WcMT2g$PScj&c6mNE6(6^nUSAY&_0 zywyAAr!nyhY;wXb?%Zt`oQMsmnX75a+i(9YMt_u!%P8EzM;8D9qe}n)*T0sItU3^F zD^E|DyYr7CGFIQrheDA$=$gee;yx+%i+mhKP9ekvNO5%Nn8_`mU;*2UXmh6|X0w@% zh@p#Xm_@mY<}dvx^NLa^1!AgM|G2ikMa-kAOylW}#4p1`{al5cp`aV<@9mW^vkc(6-IDXj;;^xFuk`JiW<(7#ML5XJU zc*SdzS+=R8OG!3S-UaORb>xu`eRYjVG$&|!@3GT21PE*g(Bm4wRAknpEK_qxl^SX% zN=9MxzTgiGz$t=ekaE3d1aii$QNWOTxJ-Jr!P)Sz!x?6Sd7MpY<~Q&q;~9{H%cBcJ z*Uj(=^C7LkfFEgHEZuKq!Z@!Z_w_?2GUjpkEd^Hdr!xx@OVKKXDRY zxzn|bXR|j|s436tO1(T6MgiAbo=2IduNJIKRBt!XORVE)nR19Y%q{$ROU9*BaTwdLf40r zzi4Q#HGbdxsxl;i&I{ol7z&QNiJ?K=c_9(y7gw4lA`7rLw`C$cI7yg$AH+esbGPK^ zGCRpa3=mwr=-JM&B0?L~l*d``xo@gGD=0upKj9pyI035{X4%Jyj~bap`Fn}>y@{hA z1%b`wk?oatnO3)DT$>A>lZ`a%I%8yXN*MH!&fTL6-xujxv~c_f-h94!A>uIa;A2W$ zuz8H*(8?Iz$m7or^W1%@YaiKJ6LR*Ng(fal2Pt=oPb7g29ap5-hm zNtw`z1{^NsHD4=l!0L7UAoIX3VBAFoRP_OCP>P>Ts3=s6x!9BB8bjFPb)sccYKE}% zZbdG+)Znyg!8EZqaUn-U&abZ%dnxCl&8kXP;am>dM;O|%C7G~P)dzI*lMDe2?GxSo zA}<$jt$%1@2E649UIWTLWZ}uzpk zApgB(COHlAUoEqfl{d`!S2OuLGtKYJzxSZO!^Hj#_D4GkcX0+=IeYlP+`;ZY8dR{u zj}G;hHb4OEv{@FY{xlJY1pu_3<|x_!MXalf6OXSGA~tErIgFPyd`l7s9j6&Yth3}^ zkzAC*rQf3XaEkP)rQWpc&X*UOa+*m+Eeq+!TXnY~v~2P18i6I(bDkuI@eR=~W_UA^ zI&nw<5=*)(Zro~Cpy^L!Zjgpc7k+D1KI&XqHWuz;oF2*qm%;n*2}5~Raz$%Fw>Lk; z+K#$%9*^(@C2z~U1N%(z$g`WVHaI~Xjb-n_g~s`PE4ueswH; zzH|vuLO7>qw}C0e*_74e@+9HXpHKNLtEdygBKopXH|DFVI#2(1mGW;Mo_0QvG_fL{ zd9s>_wp$0uF?A*VjEH-d_Kn(DM3LAR7qcIcs;xkP!fps0ktDFMMB1mswBe8P6RMXT z?%5@EG?%9%m_)oOL+@{e;NBMJ}x>NEd3x-z!9Y5^zuK)Oy?^9 zeI)h6kTdKV^5g0H*O32-`{%*a8S0E`LMTpS_8j-mW2kfe|G5_VA?r+<7Qw$s`+H?_ zF8BA;be8^p5Gml~MBw!HmuG(_!=EPp4n9kV=U~g3`Yg&*XV-)IW#t`ED=Cesk=(Z~mK6XSvEQm!}H literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar new file mode 100644 index 0000000000000000000000000000000000000000..436b0f3aa920023ab40303e1c2881d55f2a47b5f GIT binary patch literal 6324 zcmbtZbzGDC{vV@D@X(FYAURS{kd&CfV1R@;8idg>KoCR`rBhlu1d)(N!qF)y-Q6)# zggcMFd+~Vuo%_fA?)B{T?0G)VTgHJ^%{d=+tf-)+%?DQ82X}Rt_!^3q$oaHwR0d7(i4p){{tKPv9f-N3 ztGTnwT^QWV*4&BD)E4IKoUDtFpf{)Sx!7U{jS^+qrWMhbMEBbk7DmfDYK4n}ty#eX znP6~GQ~MSx%}AH6B|9<3)#j+TZP%1AFfi^YJeN(>xQ+En>-lhEdSSqp=RUQrph4P#O3@6?ukBg^ODm98ML8E4x=M627%Z zKS0ZDU=-Dz7`vLxL#jOFMkt@SnUpqn*XK>$9Eux{J}Zqt(2r$Yt<#E$aSfF8&eM1* z{Ey9ol#EuR-n9U^tL)YtXWXRWLVLdo=`RvYwN`;@Y%1nGm=AV_>#ZVlzhsZ?cw zdr3&i1p1gg>@|oT=qg9QTsei6^_LgHNhIcW6(1JKiTRQ3f;N&+q>vRbA(~8fQ3oag zA+Vu-#l;y?og$c-M(O$t$(TKC3M2(fPTs+~HD|RV2j-^5(s3d3V$=jvA`=#XlB>kD z?NXa}4@^*2+>Xx>t33e0{CEy(BFpgC z*9z7*CURwZZEqy|zS){l_E3=~M1qnjTfYHN~dxG#j{FV(?L#12iip09?1)Frsf zi(Vy&xtE(o3Vz5?)#>Z7u|>LJV+o5)Ygi*hJOc-OXGO+KLXsli@) zbh!3FjtnK+ScGS3`W6dON_yqpkpjvF!Rz8i)q5olsFY)251TZ@=DsOb-LHj-NpOmN zFJVBQ*k54Ob7c_K({No+zo$O*Y{%F7-2iR^Yw&BUHq*dsnZezm%AqOSpBcOZuhXi$ z+SxCf;ol|3*Ae3?!q(~e_&6#V^!3}hc$FXyVU3wK1Ji`Zdj6(PBO|L@a}nKZZfdRw zqD9e1)XnByw@+^r)oO#dC{n%@`&8IIS{>^kV{L2s(z#LTqzH+o&{QM1PiH^qKrz*n zDKN_5>gMjp)||QXC`%;M`M_XD6x-fB&eB<=mO-~P;l@0~-a`L*d)^EKuye^-zNH9v zyWfF$svA?xl5UHAvhzypYPsaX+PbhIzn?aKrm-#elw5e1p?@>w(7K4WV;`6n9;;+g zX1W8Y5RFs3onyk^`fgQ_YTu2oeY9zsHhrN z`U1*Uk)(jkl$iOa>%AHZ0pp-<&lZeecL$k98R||v*g1|VE_@onP zmqoib2-y$z#48 z!yc|~k3_jwcE;Bwx6&|%b2<>mZnVdB0ZcP58jHnmr305^HV*fTa>UDZ=Ub2Sp1z5u zFYDOs-vK2xZ^VlhpB~UAupO(u7RfNn9bHO^5eO!tRb!k44`=HP4rNDrmzxf~Jr>Ey zOpUO#(I2ZQhi~R77up%WK`9a{zE9*TV(NJOjW=J&^5|emY82-Qzt%Dda5Aq7&R2TX zHrZM2mheI+a$nu4;oE+&f1Y?yvE?`DjM-_XmokVm{Pw^uj&rpo>*jfb62+5czXSrB z46T(5v(H-ohj;sXmKV;$&+Pr0gyT;ew5=}|@pxb^!I@GLa?JZnaR07l5B4i#1xj8` zYp&2*W+)YYlj}bLh%6~ zVmzsx*;C!E*M5-ek+oFw>Bz=vfxhwm;^U6`j^j^CMOJU>MfvKzJ!ymF?#n*NN8Vb` zz2ihR6fz%IjZZ%g3T?umm#*?kvquWab6woMokh+&;!oxn$seXWuknDxU72CG`qYti zy`=~ROQL+wBsZVgByNu3ipopz5uV``K83u-l9bEa+axM@8oV1X9#lJ&sc_<2K1i;) zTBEqAuEs2v@#TY9ORooYP|Hq7K#QR8m(A8)++xic9nLfwGOO*E?!g(Q=gj2!5K|0=JD zAB@1P(47p!C4=JbTKK6yR96J&I(?HoGO?HK=4|{@79YpIiNgh+45P#Ji(gAmFkod- zv^mo`SJhVME(omYYjyNeXT7>ki)0vAu9RferounM!7|a#oS@>sXP?3B6D%P}h2X!ZTR9U^nZ=1eBFE}w(8z9v8C`t9>ba>4AY(maK-kr%xWwRHtq zddx%%S|BZ+5(y(U!>Lru4*|=c8a*nLqSxbV#4>0qQn4#`A>IJGUB~ij^?9IFH_txE zit6fv=rF|>;vw#HLo4J|0+j~EU|Zx#I+bLT$OzZgr{^Zp`~_k2*QE&w3pg#Hf`lh- zq+#ukm&dVgv#+ZM&pPVUH2RK&a^#A1%##5|xe_ejEVx^p!id zJjw1a$)Am&%kbvgnk}V$laWA_-fLR*w2N>g>tO{@wQ-Zbq{)Qf*;6}$;V6&5jGYk; zasN39DVgzeSqo*gD0+^1wwN` zeM-AA@-9Tv1Xcx$+id9}xVss67Q?7A;p3b1ts{f8IL4E2WirFAomE(p?t^oWA`E8g zl?_=YynsHT#>XOc1tV$HJVQ#-*MqEqUY=`n^#y?!+~}9C7D7#jm-Z@HpM@ z=W)P=e@aG>x;wLf#ODg zPj(GU*D_210EP^_J=AUhkI2=KH7%(pt{Cblu9aPu7Tnd2Pm0+*>H_~jLnx|{RU3*sqW?)VYx z!SU@=HOnc^b=xWX$=}xB*C=0iNgP zp-LF4N?;{k9@l_{>M!IO%)|8bttEFKzA~jBAbNSGVwBwFK;4#W6lzL-zbtW{T23^` zqt;>ux<>=y3$IDr$dzNr_sP!$d%i1Dj6J`;kZp}KC^*m<`k8~FkZg>%Q z<{8wzn=&^0Cby#w}fxwO^UFpbNDdpRIh!Et0$R+(*vV=9YRrXQ4AUVSC;} z^(SNZ0%lS_KS`oeB(}2HigKi$T^<>KxUi&)*J(i(N}Bi}LvxUaIQ>EE;7!4_09`LZ z+NpU|Qffsjb)RO?k_|3L*YwUSiRngV(Cdwv=y+9Efk%`+t*cikQ^LpbVpk;c<>-MU zIof>LGZF7Xu%#O4T%7I<+;0+^8pNWE;2>b}%xPmo!JpeIf7HT)K6^P77pru;<57#@ zX1WH~*-LUs;U~>`?IvkK6(p;iX?bdEsNtcG+G!mMLk2Z6>*iZmYLwM^lX0{`cB;Hs{ zo>)sJzzlLn%q$=x+N;F1mMz+%3MpToBmBL*08LOJD$&L-ZhIXH4r$$b+7>zy2+pM_2zARCHt;k8x zhyOnnxgE^S9R5>1`ztZc---WP-u@_qe`S;SC$>K;U2A(dFAVPNZtleE^rQCWwfa%~ z{!}$bqH!xpt!B;#0|2N)W0%bTBG$p)jt^mH8xqwG_ZJ|2wk?G{7Yg>dqPgJoJ~}ss z8~RZpc9b;dq1L#}-eDP77Mzg#u^Ci4W&h=wunMDJ{IRyKtTY?0X@W?L$fgYQn1&mG z75dWHxE%3PD4I>q$-Dot+L3qI>GR(1QriIO@Y<>f6)Vp>uv_m{#?bRk4jLAV`r+ zwEosbNk{s)L95-Be9IDer8i^eFgMPjoi5gN#a1lMTNMHKPQKV>{2Hb6mGe>$6!B$D z`o3il?zCA>=r#31UrPV$+B!JdJD5AUSerXDv=S)!kikFuJpzcCMc zdBCuYlkT&^ zIpbej%&ocB-Ko}VxT__;$o7g&*;pt4jOr`mVgNnB4hFZjKzDceY@F@kP;GnwW^s0xQ84;)MGyiouvXAp`_HeB+KupW0iut~ zJXq;^e2%44dc^@qj(qWEmw3%6qiPo%%3HBwOgDf##bVYrDI*BHeD0 zUN9Z2wpi#Z6zdGrbVd6cU`)%i)*)Bu_1c`GxH>C3ON&Td*PCRXOL$z7d!kapHJ_HOaAkAt-&^vrv$7RD2i$+0c8(U_6wgg%soN7ELDbOO2$(lzbQ| zkE^K!zWsR63)-90=dyRUGw;aQ%^qZp$%0WVzYdmb+fSA?pKhDT&Wawk)-t+C>N+YI z$k4*T&(c$AjW6>}?tL#44EQ!~Pa<#{!f#*qDu5&_i-xvKE{#DL92S){q2t9O;B`{w zODMyEsnR^)#JR}fCVovhv>Lp%zNv&*SnTd!!z2y`KZ-o+GPF|YxV0ZS;cE04){IzI zQ}%u*G3w!Ih{C9}iaa?@Q|g?3(B$Cyk-q%-GmVPJc<%{=jUDCURSs^-UztWZ4CzL> zVS8npNyo~;UG7@nyv0=Ou3vU>#mC#@ko~6Jd9(10tT(mFgXbqn7>Rd4$G94*7(fER z?=kOtr}MAlKPda7?}-5<|K8*Le>$I+tUN!TgrEALKP&%Thw@TFGy3+g{mM(Tf8|6w z^V8Ty-~SBomp1>dgL%nFh2HL$e82B!ez*VMGw{2N^io<3;Xg_HH9!4%UVaTk--F%d zh(!PV&!YT0^j%KPugu?r-{q+PJM*Q-|764WV0bwW|4Gb$4~>^o_`BeEsRm5?H&-vM z{cmdiFI4`#V84UwOKn_#{Eyl$!TY;XM$8}QpU0muNqLM!$UX#E9N WHB_;2zVi{GKjmnwUHSe`1K?k-H+Xyi literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/solr/question-answer-repository/repository.json b/solr/core/src/test-files/solr/question-answer-repository/repository.json new file mode 100644 index 00000000000..157787fbd50 --- /dev/null +++ b/solr/core/src/test-files/solr/question-answer-repository/repository.json @@ -0,0 +1,56 @@ +[ + { + "name": "question-answer", + "description": "A natural language question answering plugin", + "versions": [ + { + "version": "1.0.0", + "date": "2019-01-01", + "artifacts": [ + { + "url": "question-answer-request-handler-1.0.jar", + "sig": "C9UWKkucmY3UNzqn0VLneVMe9kCbJjw7Urc76vGenoRwp32xvNn5ZIGZ7G34xZP7cVjqn/ltDlLWBZ/C3eAtuw==" + } + ], + "manifest": { + "version-constraint": "8 - 9", + "plugins": [ + { + "name": "request-handler", + "setup-command": { + "path": "/api/collections/${collection}/config", + "payload": {"add-requesthandler": {"name": "${RH-HANDLER-PATH}", "class": "question-answer:fullstory.QARequestHandler"}}, + "method": "POST" + }, + "uninstall-command": { + "path": "/api/collections/${collection}/config", + "payload": {"delete-requesthandler": "${RH-HANDLER-PATH}"}, + "method": "POST" + }, + "verify-command": { + "path": "/api/collections/${collection}/config/requestHandler?componentName=${RH-HANDLER-PATH}&meta=true", + "method": "GET", + "condition": "$['config'].['requestHandler'].['${RH-HANDLER-PATH}'].['_packageinfo_'].['version']", + "expected": "${package-version}" + } + } + ], + "parameter-defaults": { + "RH-HANDLER-PATH": "/mypath" + } + } + }, + { + "version": "1.1.0", + "date": "2019-01-01", + "artifacts": [ + { + "url": "question-answer-request-handler-1.1.jar", + "sig": "qdNsVe5G8clkBDQwcX2MViA1XU6bhqEwFCPiqPxot7YQw5oj/pFvDPkzWaVa6rNbUzK3WUBbhV5TedhF7hZMOQ==" + } + ] + } + ] + } +] + diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java new file mode 100644 index 00000000000..a9bf9075826 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.lang.invoke.MethodHandles; +import java.util.Arrays; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.core.TestSolrConfigHandler; +import org.apache.solr.util.PackageTool; +import org.apache.solr.util.SolrCLI; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PackageManagerCLITest extends SolrCloudTestCase { + + // Note for those who want to modify the jar files used in the packages used in this test: + // You need to re-sign the jars for install step, as follows: + // $ openssl dgst -sha1 -sign ./solr/core/src/test-files/solr/question-answer-repository-private-key.pem ./solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar | openssl enc -base64 + // You can place the new signature thus obtained (removing any whitespaces) in the repository.json. + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static LocalWebServer repositoryServer; + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("enable.packages", "true"); + + configureCluster(1) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .addConfig("conf2", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + repositoryServer = new LocalWebServer(TEST_PATH().resolve("question-answer-repository").toString()); + repositoryServer.start(); + } + + @AfterClass + public static void teardown() throws Exception { + repositoryServer.stop(); + System.clearProperty("enable.packages"); + } + + @Test + @SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.") + public void testPackageManager() throws Exception { + PackageTool tool = new PackageTool(); + + // Enable the logger for this test. Need to do this since the tool disables logger. + Configurator.setRootLevel(Level.INFO); + + String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString(); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "add-repo", "fullstory", "http://localhost:" + repositoryServer.getPort()}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-available"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer:1.0.0"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"}); + + CollectionAdminRequest.createCollection("abc", "conf1", 1, 1).process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection("def", "conf2", 1, 1).process(cluster.getSolrClient()); + + String rhPath = "/mypath2"; + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer", "-y", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath}); + assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0"); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "-c", "abc"}); + + // Should we test the "auto-update to latest" functionality or the default explicit deploy functionality + boolean autoUpdateToLatest = random().nextBoolean(); + + if (autoUpdateToLatest) { + log.info("Testing auto-update to latest installed"); + + // This command pegs the version to the latest available + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer:latest", "-y", "-collections", "abc"}); + assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.0.0"); + + run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer"}); + assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.1.0"); + } else { + log.info("Testing explicit deployment to a different/newer version"); + + run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer"}); + assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0"); + + if (random().nextBoolean()) { // even if parameters are not passed in, they should be picked up from previous deployment + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "-y", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath}); + } else { + run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "-y", "question-answer", "-collections", "abc"}); + } + assertPackageVersion("abc", "question-answer", "1.1.0", rhPath, "1.1.0"); + } + + log.info("Running undeploy..."); + run(tool, new String[] {"-solrUrl", solrUrl, "undeploy", "question-answer", "-collections", "abc"}); + + run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"}); + + } + + void assertPackageVersion(String collection, String pkg, String version, String component, String componentVersion) throws Exception { + TestSolrConfigHandler.testForResponseElement( + null, + cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection, + "/config/params?meta=true", + cluster.getSolrClient(), + Arrays.asList("response", "params", "PKG_VERSIONS", pkg), + version, + 1); + + TestSolrConfigHandler.testForResponseElement( + null, + cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection, + "/config/requestHandler?componentName=" + component + "&meta=true", + cluster.getSolrClient(), + Arrays.asList("config", "requestHandler", component, "_packageinfo_", "version"), + componentVersion, + 1); + } + + private void run(PackageTool tool, String[] args) throws Exception { + int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res); + } + + static class LocalWebServer { + private int port = 0; + final private String resourceDir; + Server server; + ServerConnector connector; + + public LocalWebServer(String resourceDir) { + this.resourceDir = resourceDir; + } + + public int getPort() { + return connector != null? connector.getLocalPort(): port; + } + + public void start() throws Exception { + server = new Server(); + + connector = new ServerConnector(server); + connector.setPort(port); + server.addConnector(connector); + server.setStopAtShutdown(true); + + ResourceHandler resourceHandler = new ResourceHandler(); + resourceHandler.setResourceBase(resourceDir); + resourceHandler.setDirectoriesListed(true); + + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] {resourceHandler, new DefaultHandler()}); + server.setHandler(handlers); + + server.start(); + } + + public void stop() throws Exception { + server.stop(); + } + } +} diff --git a/solr/licenses/java-semver-0.9.0.jar.sha1 b/solr/licenses/java-semver-0.9.0.jar.sha1 new file mode 100644 index 00000000000..e59c2edfaee --- /dev/null +++ b/solr/licenses/java-semver-0.9.0.jar.sha1 @@ -0,0 +1 @@ +59a83ca73c72a5e25b3f0b1bb305230a11000329 diff --git a/solr/licenses/java-semver-LICENSE-MIT.txt b/solr/licenses/java-semver-LICENSE-MIT.txt new file mode 100644 index 00000000000..66a6a512a83 --- /dev/null +++ b/solr/licenses/java-semver-LICENSE-MIT.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright 2012-2014 Zafar Khaja . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java index bda1078ed04..d3fe6473839 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java @@ -33,6 +33,10 @@ public class Package { public String version; @JsonProperty(required = true) public List files; + @JsonProperty + public String manifest; + @JsonProperty + public String manifestSHA512; }