mirror of https://github.com/apache/lucene.git
SOLR-13662: Package manager (CLI)
This commit is contained in:
Ishan Chattopadhyaya
parent
3c9140f243
commit
6edbda7429
|
|
@ -26,6 +26,8 @@ com.fasterxml.jackson.core.version = 2.9.9
|
|||
/com.github.ben-manes.caffeine/caffeine = 2.8.0
|
||||
/com.github.virtuald/curvesapi = 1.04
|
||||
|
||||
/com.github.zafarkhaja/java-semver = 0.9.0
|
||||
|
||||
/com.google.guava/guava = 25.1-jre
|
||||
/com.google.protobuf/protobuf-java = 3.6.1
|
||||
/com.google.re2j/re2j = 1.2
|
||||
|
|
|
|||
|
|
@ -39,10 +39,16 @@ New Features
|
|||
|
||||
* SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya)
|
||||
|
||||
* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store
|
||||
the configuration, APIs to read/edit them and isolated classloaders to load the classes from
|
||||
hose packages if the 'class' attribute is prefixed with `<package-name>:` (noble, Ishan Chattopadhyaya)
|
||||
|
||||
* SOLR-13822: A Package management system with the following features:
|
||||
(a) A packages.json in ZK to store the configuration,
|
||||
(b) APIs to read/edit them, and
|
||||
(c) Isolated classloaders to load the classes from those packages when the 'class'
|
||||
attribute is prefixed with '<package-name>:'
|
||||
(noble, Ishan Chattopadhyaya)
|
||||
|
||||
* SOLR-13662: A CLI based Package Manager ("bin/solr package help" for more details).
|
||||
(Ishan Chattopadhyaya, noble, David Smiley, Jan Hoydahl)
|
||||
|
||||
* SOLR-10786: Add DBSCAN clustering Streaming Evaluator (Joel Bernstein)
|
||||
|
||||
Improvements
|
||||
|
|
|
|||
|
|
@ -764,6 +764,56 @@ function get_info() {
|
|||
return $CODE
|
||||
} # end get_info
|
||||
|
||||
function run_package() {
|
||||
runningSolrUrl=""
|
||||
|
||||
numSolrs=`find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | wc -l | tr -d ' '`
|
||||
if [ "$numSolrs" != "0" ]; then
|
||||
#echo -e "\nFound $numSolrs Solr nodes: "
|
||||
while read PIDF
|
||||
do
|
||||
ID=`cat "$PIDF"`
|
||||
port=`jetty_port "$ID"`
|
||||
if [ "$port" != "" ]; then
|
||||
#echo -e "\nSolr process $ID running on port $port"
|
||||
runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr"
|
||||
break
|
||||
CODE=$?
|
||||
echo ""
|
||||
else
|
||||
echo -e "\nSolr process $ID from $PIDF not found."
|
||||
CODE=1
|
||||
fi
|
||||
done < <(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f)
|
||||
else
|
||||
# no pid files but check using ps just to be sure
|
||||
numSolrs=`ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//'`
|
||||
if [ "$numSolrs" != "0" ]; then
|
||||
echo -e "\nFound $numSolrs Solr nodes: "
|
||||
PROCESSES=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | awk '{print $2}' | sort -r)
|
||||
for ID in $PROCESSES
|
||||
do
|
||||
port=`jetty_port "$ID"`
|
||||
if [ "$port" != "" ]; then
|
||||
echo ""
|
||||
echo "Solr process $ID running on port $port"
|
||||
runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr"
|
||||
break
|
||||
CODE=$?
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo -e "\nNo Solr nodes are running.\n"
|
||||
exit 1
|
||||
CODE=3
|
||||
fi
|
||||
fi
|
||||
|
||||
run_tool package -solrUrl "$runningSolrUrl" $@
|
||||
#exit $?
|
||||
}
|
||||
|
||||
# tries to gracefully stop Solr using the Jetty
|
||||
# stop command and if that fails, then uses kill -9
|
||||
function stop_solr() {
|
||||
|
|
@ -1359,6 +1409,11 @@ if [[ "$SCRIPT_CMD" == "export" ]]; then
|
|||
exit $?
|
||||
fi
|
||||
|
||||
if [[ "$SCRIPT_CMD" == "package" ]]; then
|
||||
run_package $@
|
||||
exit $?
|
||||
fi
|
||||
|
||||
if [[ "$SCRIPT_CMD" == "auth" ]]; then
|
||||
|
||||
VERBOSE=""
|
||||
|
|
|
|||
|
|
@ -215,6 +215,7 @@ IF "%1"=="-version" goto get_version
|
|||
IF "%1"=="assert" goto run_assert
|
||||
IF "%1"=="autoscaling" goto run_autoscaling
|
||||
IF "%1"=="export" goto run_export
|
||||
IF "%1"=="package" goto run_package
|
||||
|
||||
REM Only allow the command to be the first argument, assume start if not supplied
|
||||
IF "%1"=="start" goto set_script_cmd
|
||||
|
|
@ -1422,6 +1423,15 @@ goto done:
|
|||
org.apache.solr.util.SolrCLI %*
|
||||
goto done:
|
||||
|
||||
:run_package
|
||||
REM TODO: Compute the running Solr URL and populate it as a parameter (as has been done for the shell script)
|
||||
REM Without that, users will have to supply -solrUrl parameter in every request. Life can be so hard for Windows users!
|
||||
"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^
|
||||
-Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^
|
||||
-classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^
|
||||
org.apache.solr.util.SolrCLI %*
|
||||
goto done:
|
||||
|
||||
:parse_config_args
|
||||
IF [%1]==[] goto run_config
|
||||
IF "%1"=="-z" goto set_config_zk
|
||||
|
|
|
|||
|
|
@ -138,6 +138,9 @@
|
|||
<dependency org="com.google.protobuf" name="protobuf-java" rev="${/com.google.protobuf/protobuf-java}" conf="compile"/>
|
||||
<dependency org="com.jayway.jsonpath" name="json-path" rev="${/com.jayway.jsonpath/json-path}" conf="compile"/>
|
||||
|
||||
<!-- Package manager -->
|
||||
<dependency org="com.github.zafarkhaja" name="java-semver" rev="${/com.github.zafarkhaja/java-semver}" conf="compile"/>
|
||||
|
||||
<dependency org="org.rrd4j" name="rrd4j" rev="${/org.rrd4j/rrd4j}" conf="compile"/>
|
||||
|
||||
<!-- JWT Auth plugin -->
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
||||
/**
|
||||
* This is a serializable bean (for the JSON that is stored in /repository.json) representing a repository of Solr packages.
|
||||
* Supports standard repositories based on a webservice.
|
||||
*/
|
||||
public class DefaultPackageRepository extends PackageRepository {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
public DefaultPackageRepository() { // this is needed for deserialization from JSON
|
||||
|
||||
}
|
||||
|
||||
public DefaultPackageRepository(String repositoryName, String repositoryURL) {
|
||||
this.name = repositoryName;
|
||||
this.repositoryURL = repositoryURL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
packages = null;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
private Map<String, SolrPackage> packages;
|
||||
|
||||
@Override
|
||||
public Map<String, SolrPackage> getPackages() {
|
||||
if (packages == null) {
|
||||
initPackages();
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SolrPackage getPackage(String packageName) {
|
||||
return getPackages().get(packageName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasPackage(String packageName) {
|
||||
return getPackages().containsKey(packageName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path download(String artifactName) throws SolrException, IOException {
|
||||
Path tmpDirectory = Files.createTempDirectory("solr-packages");
|
||||
tmpDirectory.toFile().deleteOnExit();
|
||||
URL url = new URL(new URL(repositoryURL), artifactName);
|
||||
String fileName = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
|
||||
Path destination = tmpDirectory.resolve(fileName);
|
||||
|
||||
switch (url.getProtocol()) {
|
||||
case "http":
|
||||
case "https":
|
||||
case "ftp":
|
||||
FileUtils.copyURLToFile(url, destination.toFile());
|
||||
break;
|
||||
default:
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "URL protocol " + url.getProtocol() + " not supported");
|
||||
}
|
||||
|
||||
return destination;
|
||||
}
|
||||
|
||||
private void initPackages() {
|
||||
try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
|
||||
SolrPackage[] items = PackageUtils.getJson(client, repositoryURL + "/repository.json", SolrPackage[].class);
|
||||
|
||||
packages = new HashMap<>(items.length);
|
||||
for (SolrPackage pkg : items) {
|
||||
pkg.setRepository(name);
|
||||
packages.put(pkg.name, pkg);
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
throw new SolrException(ErrorCode.INVALID_STATE, ex);
|
||||
}
|
||||
log.debug("Found {} packages in repository '{}'", packages.size(), name);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,416 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
import static org.apache.solr.packagemanager.PackageUtils.getMapper;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Scanner;
|
||||
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.common.NavigableObject;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Command;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Manifest;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Plugin;
|
||||
import org.apache.solr.pkg.PackagePluginHolder;
|
||||
import org.apache.solr.util.SolrCLI;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import com.jayway.jsonpath.PathNotFoundException;
|
||||
|
||||
/**
|
||||
* Handles most of the management of packages that are already installed in Solr.
|
||||
*/
|
||||
public class PackageManager implements Closeable {
|
||||
|
||||
final String solrBaseUrl;
|
||||
final HttpSolrClient solrClient;
|
||||
final SolrZkClient zkClient;
|
||||
|
||||
private Map<String, List<SolrPackageInstance>> packages = null;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
|
||||
public PackageManager(HttpSolrClient solrClient, String solrBaseUrl, String zkHost) {
|
||||
this.solrBaseUrl = solrBaseUrl;
|
||||
this.solrClient = solrClient;
|
||||
this.zkClient = new SolrZkClient(zkHost, 30000);
|
||||
log.info("Done initializing a zkClient instance...");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (zkClient != null) {
|
||||
zkClient.close();
|
||||
}
|
||||
}
|
||||
|
||||
public List<SolrPackageInstance> fetchInstalledPackageInstances() throws SolrException {
|
||||
log.info("Getting packages from packages.json...");
|
||||
List<SolrPackageInstance> ret = new ArrayList<SolrPackageInstance>();
|
||||
packages = new HashMap<String, List<SolrPackageInstance>>();
|
||||
try {
|
||||
Map packagesZnodeMap = null;
|
||||
|
||||
if (zkClient.exists("/packages.json", true) == true) {
|
||||
packagesZnodeMap = (Map)getMapper().readValue(
|
||||
new String(zkClient.getData("/packages.json", null, null, true), "UTF-8"), Map.class).get("packages");
|
||||
for (Object packageName: packagesZnodeMap.keySet()) {
|
||||
List pkg = (List)packagesZnodeMap.get(packageName);
|
||||
for (Map pkgVersion: (List<Map>)pkg) {
|
||||
Manifest manifest = PackageUtils.fetchManifest(solrClient, solrBaseUrl, pkgVersion.get("manifest").toString(), pkgVersion.get("manifestSHA512").toString());
|
||||
List<Plugin> solrplugins = manifest.plugins;
|
||||
SolrPackageInstance pkgInstance = new SolrPackageInstance(packageName.toString(), null,
|
||||
pkgVersion.get("version").toString(), manifest, solrplugins, manifest.parameterDefaults);
|
||||
List<SolrPackageInstance> list = packages.containsKey(packageName)? packages.get(packageName): new ArrayList<SolrPackageInstance>();
|
||||
list.add(pkgInstance);
|
||||
packages.put(packageName.toString(), list);
|
||||
ret.add(pkgInstance);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
log.info("Got packages: "+ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
public Map<String, SolrPackageInstance> getPackagesDeployed(String collection) {
|
||||
Map<String, String> packages = null;
|
||||
try {
|
||||
NavigableObject result = (NavigableObject) Utils.executeGET(solrClient.getHttpClient(),
|
||||
solrBaseUrl+"/api/collections/"+collection+"/config/params/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER);
|
||||
packages = (Map<String, String>) result._get("/response/params/PKG_VERSIONS", Collections.emptyMap());
|
||||
} catch (PathNotFoundException ex) {
|
||||
// Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
|
||||
}
|
||||
if (packages == null) return Collections.emptyMap();
|
||||
Map<String, SolrPackageInstance> ret = new HashMap<String, SolrPackageInstance>();
|
||||
for (String packageName: packages.keySet()) {
|
||||
if (Strings.isNullOrEmpty(packageName) == false) { // There can be an empty key, storing the version here
|
||||
ret.put(packageName, getPackageInstance(packageName, packages.get(packageName)));
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, boolean noprompt,
|
||||
List<String> collections, String overrides[]) {
|
||||
for (String collection: collections) {
|
||||
|
||||
SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageInstance.name);
|
||||
if (packageInstance.equals(deployedPackage)) {
|
||||
if (!pegToLatest) {
|
||||
PackageUtils.printRed("Package " + packageInstance + " already deployed on "+collection);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
if (deployedPackage != null && !isUpdate) {
|
||||
PackageUtils.printRed("Package " + deployedPackage + " already deployed on "+collection+". To update to "+packageInstance+", pass --update parameter.");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Map<String,String> collectionParameterOverrides = getCollectionParameterOverrides(packageInstance, isUpdate, overrides, collection);
|
||||
|
||||
// Get package params
|
||||
try {
|
||||
boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/abc/config/params/packages", Map.class)
|
||||
.getOrDefault("response", Collections.emptyMap())).containsKey("params");
|
||||
SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params",
|
||||
getMapper().writeValueAsString(Collections.singletonMap(packageParamsExist? "update": "set",
|
||||
Collections.singletonMap("packages", Collections.singletonMap(packageInstance.name, collectionParameterOverrides)))));
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, e);
|
||||
}
|
||||
|
||||
// Set the package version in the collection's parameters
|
||||
try {
|
||||
SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params",
|
||||
"{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}");
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
|
||||
}
|
||||
|
||||
// If updating, refresh the package version for this to take effect
|
||||
if (isUpdate || pegToLatest) {
|
||||
try {
|
||||
SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageInstance.name + "\"}");
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
|
||||
}
|
||||
}
|
||||
|
||||
// If it is a fresh deploy on a collection, run setup commands all the plugins in the package
|
||||
if (!isUpdate) {
|
||||
Map<String, String> systemParams = PackageUtils.map("collection", collection, "package-name", packageInstance.name, "package-version", packageInstance.version);
|
||||
|
||||
for (Plugin plugin: packageInstance.plugins) {
|
||||
Command cmd = plugin.setupCommand;
|
||||
if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) {
|
||||
if ("POST".equalsIgnoreCase(cmd.method)) {
|
||||
try {
|
||||
String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
|
||||
String path = PackageUtils.resolve(cmd.path, packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
|
||||
PackageUtils.printGreen("Executing " + payload + " for path:" + path);
|
||||
boolean shouldExecute = true;
|
||||
if (!noprompt) { // show a prompt asking user to execute the setup command for the plugin
|
||||
PackageUtils.print(PackageUtils.YELLOW, "Execute this command (y/n): ");
|
||||
String userInput = new Scanner(System.in, "UTF-8").next();
|
||||
if (!"yes".equalsIgnoreCase(userInput) && !"y".equalsIgnoreCase(userInput)) {
|
||||
shouldExecute = false;
|
||||
PackageUtils.printRed("Skipping setup command for deploying (deployment verification may fail)."
|
||||
+ " Please run this step manually or refer to package documentation.");
|
||||
}
|
||||
}
|
||||
if (shouldExecute) {
|
||||
SolrCLI.postJsonToSolr(solrClient, path, payload);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
|
||||
}
|
||||
} else {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for setup commands");
|
||||
}
|
||||
} else {
|
||||
PackageUtils.printRed("There is no setup command to execute for plugin: " + plugin.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set the package version in the collection's parameters
|
||||
try {
|
||||
SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params",
|
||||
"{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}");
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that package was successfully deployed
|
||||
boolean success = verify(packageInstance, collections);
|
||||
if (success) {
|
||||
PackageUtils.printGreen("Deployed and verified package: " + packageInstance.name + ", version: " + packageInstance.version);
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
private Map<String,String> getCollectionParameterOverrides(SolrPackageInstance packageInstance, boolean isUpdate,
|
||||
String[] overrides, String collection) {
|
||||
Map<String, String> collectionParameterOverrides = isUpdate? getPackageParams(packageInstance.name, collection): new HashMap<String,String>();
|
||||
if (overrides != null) {
|
||||
for (String override: overrides) {
|
||||
collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]);
|
||||
}
|
||||
}
|
||||
return collectionParameterOverrides;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
Map<String, String> getPackageParams(String packageName, String collection) {
|
||||
try {
|
||||
return (Map<String, String>)((Map)((Map)((Map)
|
||||
PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/packages", Map.class)
|
||||
.get("response"))
|
||||
.get("params"))
|
||||
.get("packages")).get(packageName);
|
||||
} catch (Exception ex) {
|
||||
// This should be because there are no parameters. Be tolerant here.
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a package and list of collections, verify if the package is installed
|
||||
* in those collections. It uses the verify command of every plugin in the package (if defined).
|
||||
*/
|
||||
public boolean verify(SolrPackageInstance pkg, List<String> collections) {
|
||||
boolean success = true;
|
||||
for (Plugin plugin: pkg.plugins) {
|
||||
PackageUtils.printGreen(plugin.verifyCommand);
|
||||
for (String collection: collections) {
|
||||
Map<String, String> collectionParameterOverrides = getPackageParams(pkg.name, collection);
|
||||
Command cmd = plugin.verifyCommand;
|
||||
|
||||
Map<String, String> systemParams = PackageUtils.map("collection", collection, "package-name", pkg.name, "package-version", pkg.version);
|
||||
String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
|
||||
PackageUtils.printGreen("Executing " + url + " for collection:" + collection);
|
||||
|
||||
if ("GET".equalsIgnoreCase(cmd.method)) {
|
||||
String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url);
|
||||
PackageUtils.printGreen(response);
|
||||
String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration())
|
||||
.read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams));
|
||||
String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
|
||||
PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue);
|
||||
if (!expectedValue.equals(actualValue)) {
|
||||
PackageUtils.printRed("Failed to deploy plugin: " + plugin.name);
|
||||
success = false;
|
||||
}
|
||||
} else {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for setup commands");
|
||||
}
|
||||
}
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the installed instance of a specific version of a package. If version is null, PackageUtils.LATEST or PackagePluginHolder.LATEST,
|
||||
* then it returns the highest version available in the system for the package.
|
||||
*/
|
||||
public SolrPackageInstance getPackageInstance(String packageName, String version) {
|
||||
fetchInstalledPackageInstances();
|
||||
List<SolrPackageInstance> versions = packages.get(packageName);
|
||||
SolrPackageInstance latest = null;
|
||||
if (versions != null && !versions.isEmpty()) {
|
||||
latest = versions.get(0);
|
||||
for (int i=0; i<versions.size(); i++) {
|
||||
SolrPackageInstance pkg = versions.get(i);
|
||||
if (pkg.version.equals(version)) {
|
||||
return pkg;
|
||||
}
|
||||
if (PackageUtils.compareVersions(latest.version, pkg.version) <= 0) {
|
||||
latest = pkg;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackagePluginHolder.LATEST)) {
|
||||
return latest;
|
||||
} else return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deploys a version of a package to a list of collections.
|
||||
* @param version If null, the most recent version is deployed.
|
||||
* EXPERT FEATURE: If version is PackageUtils.LATEST, this collection will be auto updated whenever a newer version of this package is installed.
|
||||
* @param isUpdate Is this a fresh deployment or is it an update (i.e. there is already a version of this package deployed on this collection)
|
||||
* @param noprompt If true, don't prompt before executing setup commands.
|
||||
*/
|
||||
public void deploy(String packageName, String version, String[] collections, String[] parameters,
|
||||
boolean isUpdate, boolean noprompt) throws SolrException {
|
||||
boolean pegToLatest = PackageUtils.LATEST.equals(version); // User wants to peg this package's version to the latest installed (for auto-update, i.e. no explicit deploy step)
|
||||
SolrPackageInstance packageInstance = getPackageInstance(packageName, version);
|
||||
if (packageInstance == null) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Package instance doesn't exist: " + packageName + ":" + null +
|
||||
". Use install command to install this version first.");
|
||||
}
|
||||
if (version == null) version = packageInstance.version;
|
||||
|
||||
Manifest manifest = packageInstance.manifest;
|
||||
if (PackageUtils.checkVersionConstraint(RepositoryManager.systemVersion, manifest.versionConstraint) == false) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Version incompatible! Solr version: "
|
||||
+ RepositoryManager.systemVersion + ", package version constraint: " + manifest.versionConstraint);
|
||||
}
|
||||
|
||||
boolean res = deployPackage(packageInstance, pegToLatest, isUpdate, noprompt,
|
||||
Arrays.asList(collections), parameters);
|
||||
PackageUtils.print(res? PackageUtils.GREEN: PackageUtils.RED, res? "Deployment successful": "Deployment failed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Undeploys a packge from given collections.
|
||||
*/
|
||||
public void undeploy(String packageName, String[] collections) throws SolrException {
|
||||
for (String collection: collections) {
|
||||
SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageName);
|
||||
Map<String, String> collectionParameterOverrides = getPackageParams(packageName, collection);
|
||||
|
||||
// Run the uninstall command for all plugins
|
||||
Map<String, String> systemParams = PackageUtils.map("collection", collection, "package-name", deployedPackage.name, "package-version", deployedPackage.version);
|
||||
|
||||
for (Plugin plugin: deployedPackage.plugins) {
|
||||
Command cmd = plugin.uninstallCommand;
|
||||
if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) {
|
||||
if ("POST".equalsIgnoreCase(cmd.method)) {
|
||||
try {
|
||||
String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams);
|
||||
String path = PackageUtils.resolve(cmd.path, deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams);
|
||||
PackageUtils.printGreen("Executing " + payload + " for path:" + path);
|
||||
SolrCLI.postJsonToSolr(solrClient, path, payload);
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
|
||||
}
|
||||
} else {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for uninstall commands");
|
||||
}
|
||||
} else {
|
||||
PackageUtils.printRed("There is no uninstall command to execute for plugin: " + plugin.name);
|
||||
}
|
||||
}
|
||||
|
||||
// Set the package version in the collection's parameters
|
||||
try {
|
||||
SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", "{set: {PKG_VERSIONS: {"+packageName+": null}}}");
|
||||
SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageName + "\"}");
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
|
||||
}
|
||||
|
||||
// TODO: Also better to remove the package parameters
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a package, return a map of collections where this package is
|
||||
* installed to the installed version (which can be {@link PackagePluginHolder#LATEST})
|
||||
*/
|
||||
public Map<String, String> getDeployedCollections(String packageName) {
|
||||
List<String> allCollections;
|
||||
try {
|
||||
allCollections = zkClient.getChildren("/collections", null, true);
|
||||
} catch (KeeperException | InterruptedException e) {
|
||||
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e);
|
||||
}
|
||||
Map<String, String> deployed = new HashMap<String, String>();
|
||||
for (String collection: allCollections) {
|
||||
// Check package version installed
|
||||
String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/PKG_VERSIONS?omitHeader=true");
|
||||
String version = null;
|
||||
try {
|
||||
version = JsonPath.parse(paramsJson, PackageUtils.jsonPathConfiguration())
|
||||
.read("$['response'].['params'].['PKG_VERSIONS'].['"+packageName+"'])");
|
||||
} catch (PathNotFoundException ex) {
|
||||
// Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
|
||||
}
|
||||
if (version != null) {
|
||||
deployed.put(collection, version);
|
||||
}
|
||||
}
|
||||
return deployed;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* Abstract class for a repository, holding {@link SolrPackage} items.
|
||||
*/
|
||||
public abstract class PackageRepository {
|
||||
|
||||
@JsonProperty("name")
|
||||
public String name = null;
|
||||
|
||||
@JsonProperty("url")
|
||||
public String repositoryURL = null;
|
||||
|
||||
public abstract void refresh();
|
||||
|
||||
/**
|
||||
* Returns a map of package name to {@link SolrPackage}s.
|
||||
*/
|
||||
public abstract Map<String, SolrPackage> getPackages();
|
||||
|
||||
public abstract SolrPackage getPackage(String packageName);
|
||||
|
||||
public abstract boolean hasPackage(String packageName);
|
||||
|
||||
/**
|
||||
* Provides a method to download an artifact from this repository.
|
||||
*/
|
||||
public abstract Path download(String artifactName) throws SolrException, IOException;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.request.V2Request;
|
||||
import org.apache.solr.client.solrj.response.V2Response;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.core.BlobRepository;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Manifest;
|
||||
import org.apache.solr.util.SolrJacksonAnnotationInspector;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.zafarkhaja.semver.Version;
|
||||
import com.google.common.base.Strings;
|
||||
import com.jayway.jsonpath.Configuration;
|
||||
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
|
||||
import com.jayway.jsonpath.spi.json.JsonProvider;
|
||||
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
|
||||
import com.jayway.jsonpath.spi.mapper.MappingProvider;
|
||||
|
||||
public class PackageUtils {
|
||||
|
||||
/**
|
||||
* Represents a version which denotes the latest version available at the moment.
|
||||
*/
|
||||
public static String LATEST = "latest";
|
||||
|
||||
public static Configuration jsonPathConfiguration() {
|
||||
MappingProvider provider = new JacksonMappingProvider();
|
||||
JsonProvider jsonProvider = new JacksonJsonProvider();
|
||||
Configuration c = Configuration.builder().jsonProvider(jsonProvider).mappingProvider(provider).options(com.jayway.jsonpath.Option.REQUIRE_PROPERTIES).build();
|
||||
return c;
|
||||
}
|
||||
|
||||
public static ObjectMapper getMapper() {
|
||||
return new ObjectMapper().setAnnotationIntrospector(new SolrJacksonAnnotationInspector());
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads a file to the package store / file store of Solr.
|
||||
*
|
||||
* @param client A Solr client
|
||||
* @param buffer File contents
|
||||
* @param name Name of the file as it will appear in the file store (can be hierarchical)
|
||||
* @param sig Signature digest (public key should be separately uploaded to ZK)
|
||||
*/
|
||||
public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
|
||||
throws SolrServerException, IOException {
|
||||
String resource = "/api/cluster/files" + name;
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
if (sig != null) {
|
||||
params.add("sig", sig);
|
||||
}
|
||||
V2Response rsp = new V2Request.Builder(resource)
|
||||
.withMethod(SolrRequest.METHOD.PUT)
|
||||
.withPayload(buffer)
|
||||
.forceV2(true)
|
||||
.withMimeType("application/octet-stream")
|
||||
.withParams(params)
|
||||
.build()
|
||||
.process(client);
|
||||
if (!name.equals(rsp.getResponse().get(CommonParams.FILE))) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Mismatch in file uploaded. Uploaded: " +
|
||||
rsp.getResponse().get(CommonParams.FILE)+", Original: "+name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download JSON from the url and deserialize into klass.
|
||||
*/
|
||||
public static <T> T getJson(HttpClient client, String url, Class<T> klass) {
|
||||
try {
|
||||
return getMapper().readValue(getJsonStringFromUrl(client, url), klass);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search through the list of jar files for a given file. Returns string of
|
||||
* the file contents or null if file wasn't found. This is suitable for looking
|
||||
* for manifest or property files within pre-downloaded jar files.
|
||||
* Please note that the first instance of the file found is returned.
|
||||
*/
|
||||
public static String getFileFromJarsAsString(List<Path> jars, String filename) {
|
||||
for (Path jarfile: jars) {
|
||||
try (ZipFile zipFile = new ZipFile(jarfile.toFile())) {
|
||||
ZipEntry entry = zipFile.getEntry(filename);
|
||||
if (entry == null) continue;
|
||||
return IOUtils.toString(zipFile.getInputStream(entry), "UTF-8");
|
||||
} catch (Exception ex) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, ex);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns JSON string from a given URL
|
||||
*/
|
||||
public static String getJsonStringFromUrl(HttpClient client, String url) {
|
||||
try {
|
||||
return IOUtils.toString(client.execute(new HttpGet(url)).getEntity().getContent(), "UTF-8");
|
||||
} catch (UnsupportedOperationException | IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether a given version satisfies the constraint (defined by a semver expression)
|
||||
*/
|
||||
public static boolean checkVersionConstraint(String ver, String constraint) {
|
||||
return Strings.isNullOrEmpty(constraint) || Version.valueOf(ver).satisfies(constraint);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a manifest file from the File Store / Package Store. A SHA512 check is enforced after fetching.
|
||||
*/
|
||||
public static Manifest fetchManifest(HttpSolrClient solrClient, String solrBaseUrl, String manifestFilePath, String expectedSHA512) throws MalformedURLException, IOException {
|
||||
String manifestJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/node/files" + manifestFilePath);
|
||||
String calculatedSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8")));
|
||||
if (expectedSHA512.equals(calculatedSHA512) == false) {
|
||||
throw new SolrException(ErrorCode.UNAUTHORIZED, "The manifest SHA512 doesn't match expected SHA512. Possible unauthorized manipulation. "
|
||||
+ "Expected: " + expectedSHA512 + ", calculated: " + calculatedSHA512 + ", manifest location: " + manifestFilePath);
|
||||
}
|
||||
Manifest manifest = getMapper().readValue(manifestJson, Manifest.class);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a templatized string with parameter substituted string. First applies the overrides, then defaults and then systemParams.
|
||||
*/
|
||||
public static String resolve(String str, Map<String, String> defaults, Map<String, String> overrides, Map<String, String> systemParams) {
|
||||
// TODO: Should perhaps use Matchers etc. instead of this clumsy replaceAll().
|
||||
|
||||
if (str == null) return null;
|
||||
for (String param: defaults.keySet()) {
|
||||
str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param));
|
||||
}
|
||||
for (String param: overrides.keySet()) {
|
||||
str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param));
|
||||
}
|
||||
for (String param: systemParams.keySet()) {
|
||||
str = str.replaceAll("\\$\\{"+param+"\\}", systemParams.get(param));
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares two versions v1 and v2. Returns negative if v1 isLessThan v2, positive if v1 isGreaterThan v2 and 0 if equal.
|
||||
*/
|
||||
public static int compareVersions(String v1, String v2) {
|
||||
return Version.valueOf(v1).compareTo(Version.valueOf(v2));
|
||||
}
|
||||
|
||||
public static String BLACK = "\u001B[30m";
|
||||
public static String RED = "\u001B[31m";
|
||||
public static String GREEN = "\u001B[32m";
|
||||
public static String YELLOW = "\u001B[33m";
|
||||
public static String BLUE = "\u001B[34m";
|
||||
public static String PURPLE = "\u001B[35m";
|
||||
public static String CYAN = "\u001B[36m";
|
||||
public static String WHITE = "\u001B[37m";
|
||||
|
||||
/**
|
||||
* Console print using green color
|
||||
*/
|
||||
public static void printGreen(Object message) {
|
||||
PackageUtils.print(PackageUtils.GREEN, message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Console print using red color
|
||||
*/
|
||||
public static void printRed(Object message) {
|
||||
PackageUtils.print(PackageUtils.RED, message);
|
||||
}
|
||||
|
||||
public static void print(Object message) {
|
||||
print(null, message);
|
||||
}
|
||||
|
||||
@SuppressForbidden(reason = "Need to use System.out.println() instead of log4j/slf4j for cleaner output")
|
||||
public static void print(String color, Object message) {
|
||||
String RESET = "\u001B[0m";
|
||||
|
||||
if (color != null) {
|
||||
System.out.println(color + String.valueOf(message) + RESET);
|
||||
} else {
|
||||
System.out.println(message);
|
||||
}
|
||||
}
|
||||
|
||||
public static String[] validateCollections(String collections[]) {
|
||||
String collectionNameRegex = "^[a-zA-Z0-9_-]*$";
|
||||
for (String c: collections) {
|
||||
if (c.matches(collectionNameRegex) == false) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid collection name: " + c +
|
||||
". Didn't match the pattern: '"+collectionNameRegex+"'");
|
||||
}
|
||||
}
|
||||
return collections;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replacement for Java 11's Map.of(). Borrowed from SolrTestCaseJ4's map().
|
||||
*/
|
||||
public static Map map(Object... params) {
|
||||
LinkedHashMap ret = new LinkedHashMap();
|
||||
for (int i=0; i<params.length; i+=2) {
|
||||
Object o = ret.put(params[i], params[i+1]);
|
||||
// TODO: handle multi-valued map?
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,317 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
import static org.apache.solr.packagemanager.PackageUtils.getMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.request.V2Request;
|
||||
import org.apache.solr.client.solrj.request.beans.Package;
|
||||
import org.apache.solr.client.solrj.response.V2Response;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.core.BlobRepository;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Artifact;
|
||||
import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
|
||||
import org.apache.solr.pkg.PackageAPI;
|
||||
import org.apache.solr.pkg.PackagePluginHolder;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Handles most of the management of repositories and packages present in external repositories.
|
||||
*/
|
||||
public class RepositoryManager {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
final private PackageManager packageManager;
|
||||
|
||||
public static final String systemVersion = Version.LATEST.toString();
|
||||
|
||||
final HttpSolrClient solrClient;
|
||||
|
||||
public RepositoryManager(HttpSolrClient solrClient, PackageManager packageManager) {
|
||||
this.packageManager = packageManager;
|
||||
this.solrClient = solrClient;
|
||||
}
|
||||
|
||||
public List<SolrPackage> getPackages() {
|
||||
List<SolrPackage> list = new ArrayList<>(getPackagesMap().values());
|
||||
Collections.sort(list);
|
||||
return list;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a map of package name to {@link SolrPackage} objects
|
||||
*/
|
||||
public Map<String, SolrPackage> getPackagesMap() {
|
||||
Map<String, SolrPackage> packagesMap = new HashMap<>();
|
||||
for (PackageRepository repository: getRepositories()) {
|
||||
packagesMap.putAll(repository.getPackages());
|
||||
}
|
||||
|
||||
return packagesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* List of added repositories
|
||||
*/
|
||||
public List<PackageRepository> getRepositories() {
|
||||
// TODO: Instead of fetching again and again, we should look for caching this
|
||||
PackageRepository items[];
|
||||
try {
|
||||
items = getMapper().readValue(getRepositoriesJson(packageManager.zkClient), DefaultPackageRepository[].class);
|
||||
} catch (IOException | KeeperException | InterruptedException e) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, e);
|
||||
}
|
||||
List<PackageRepository> repositories = Arrays.asList(items);
|
||||
|
||||
for (PackageRepository updateRepository: repositories) {
|
||||
updateRepository.refresh();
|
||||
}
|
||||
|
||||
return repositories;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a repository to Solr
|
||||
*/
|
||||
public void addRepository(String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException {
|
||||
String existingRepositoriesJson = getRepositoriesJson(packageManager.zkClient);
|
||||
log.info(existingRepositoriesJson);
|
||||
|
||||
List<PackageRepository> repos = getMapper().readValue(existingRepositoriesJson, List.class);
|
||||
repos.add(new DefaultPackageRepository(name, uri));
|
||||
if (packageManager.zkClient.exists("/repositories.json", true) == false) {
|
||||
packageManager.zkClient.create("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true);
|
||||
} else {
|
||||
packageManager.zkClient.setData("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), true);
|
||||
}
|
||||
|
||||
if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true);
|
||||
if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true);
|
||||
if (packageManager.zkClient.exists("/keys/exe/"+name+".der", true)==false) {
|
||||
packageManager.zkClient.create("/keys/exe/"+name+".der", new byte[0], CreateMode.PERSISTENT, true);
|
||||
}
|
||||
packageManager.zkClient.setData("/keys/exe/"+name+".der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true);
|
||||
}
|
||||
|
||||
private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException {
|
||||
if (zkClient.exists("/repositories.json", true)) {
|
||||
return new String(zkClient.getData("/repositories.json", null, null, true), "UTF-8");
|
||||
}
|
||||
return "[]";
|
||||
}
|
||||
|
||||
/**
|
||||
* Install a given package and version from the available repositories to Solr.
|
||||
* The various steps for doing so are, briefly, a) find upload a manifest to package store,
|
||||
* b) download the artifacts and upload to package store, c) call {@link PackageAPI} to register
|
||||
* the package.
|
||||
*/
|
||||
private boolean installPackage(String packageName, String version) throws SolrException {
|
||||
SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version);
|
||||
if (existingPlugin != null && existingPlugin.version.equals(version)) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Plugin already installed.");
|
||||
}
|
||||
|
||||
SolrPackageRelease release = getPackageRelease(packageName, version);
|
||||
List<Path> downloaded = downloadPackageArtifacts(packageName, version);
|
||||
// TODO: Should we introduce a checksum to validate the downloading?
|
||||
// Currently, not a big problem since signature based checking happens anyway
|
||||
|
||||
try {
|
||||
// post the manifest
|
||||
PackageUtils.printGreen("Posting manifest...");
|
||||
|
||||
if (release.manifest == null) {
|
||||
String manifestJson = PackageUtils.getFileFromJarsAsString(downloaded, "manifest.json");
|
||||
if (manifestJson == null) {
|
||||
throw new SolrException(ErrorCode.NOT_FOUND, "No manifest found for package: " + packageName + ", version: " + version);
|
||||
}
|
||||
release.manifest = getMapper().readValue(manifestJson, SolrPackage.Manifest.class);
|
||||
}
|
||||
String manifestJson = getMapper().writeValueAsString(release.manifest);
|
||||
String manifestSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8")));
|
||||
PackageUtils.postFile(solrClient, ByteBuffer.wrap(manifestJson.getBytes("UTF-8")),
|
||||
String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json"), null);
|
||||
|
||||
// post the artifacts
|
||||
PackageUtils.printGreen("Posting artifacts...");
|
||||
for (int i=0; i<release.artifacts.size(); i++) {
|
||||
PackageUtils.postFile(solrClient, ByteBuffer.wrap(FileUtils.readFileToByteArray(downloaded.get(i).toFile())),
|
||||
String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, downloaded.get(i).getFileName().toString()),
|
||||
release.artifacts.get(i).sig
|
||||
);
|
||||
}
|
||||
|
||||
// Call Package API to add this version of the package
|
||||
PackageUtils.printGreen("Executing Package API to register this package...");
|
||||
Package.AddVersion add = new Package.AddVersion();
|
||||
add.version = version;
|
||||
add.pkg = packageName;
|
||||
add.files = downloaded.stream().map(
|
||||
file -> String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, file.getFileName().toString())).collect(Collectors.toList());
|
||||
add.manifest = "/package/" + packageName + "/" + version + "/manifest.json";
|
||||
add.manifestSHA512 = manifestSHA512;
|
||||
|
||||
V2Request req = new V2Request.Builder("/api/cluster/package")
|
||||
.forceV2(true)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.withPayload(Collections.singletonMap("add", add))
|
||||
.build();
|
||||
|
||||
try {
|
||||
V2Response resp = req.process(solrClient);
|
||||
PackageUtils.printGreen("Response: "+resp.jsonStr());
|
||||
} catch (SolrServerException | IOException e) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
|
||||
} catch (SolrServerException | IOException e) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, e);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private List<Path> downloadPackageArtifacts(String packageName, String version) throws SolrException {
|
||||
try {
|
||||
SolrPackageRelease release = getPackageRelease(packageName, version);
|
||||
List<Path> downloadedPaths = new ArrayList<Path>(release.artifacts.size());
|
||||
|
||||
for (PackageRepository repo: getRepositories()) {
|
||||
if (repo.hasPackage(packageName)) {
|
||||
for (Artifact art: release.artifacts) {
|
||||
downloadedPaths.add(repo.download(art.url));
|
||||
}
|
||||
return downloadedPaths;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Error during download of package " + packageName, e);
|
||||
}
|
||||
throw new SolrException(ErrorCode.NOT_FOUND, "Package not found in any repository.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a package name and version, find the release/version object as found in the repository
|
||||
*/
|
||||
private SolrPackageRelease getPackageRelease(String packageName, String version) throws SolrException {
|
||||
SolrPackage pkg = getPackagesMap().get(packageName);
|
||||
if (pkg == null) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Package "+packageName+" not found in any repository");
|
||||
}
|
||||
if (version == null || PackageUtils.LATEST.equals(version)) {
|
||||
return getLastPackageRelease(pkg);
|
||||
}
|
||||
for (SolrPackageRelease release : pkg.versions) {
|
||||
if (PackageUtils.compareVersions(version, release.version) == 0) {
|
||||
return release;
|
||||
}
|
||||
}
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Package " + packageName + ":" + version + " does not exist in any repository.");
|
||||
}
|
||||
|
||||
public SolrPackageRelease getLastPackageRelease(String packageName) {
|
||||
SolrPackage pkg = getPackagesMap().get(packageName);
|
||||
if (pkg == null) {
|
||||
return null;
|
||||
}
|
||||
return getLastPackageRelease(pkg);
|
||||
}
|
||||
|
||||
private SolrPackageRelease getLastPackageRelease(SolrPackage pkg) {
|
||||
SolrPackageRelease latest = null;
|
||||
for (SolrPackageRelease release: pkg.versions) {
|
||||
if (latest == null) {
|
||||
latest = release;
|
||||
} else {
|
||||
if (PackageUtils.compareVersions(latest.version, release.version) < 0) {
|
||||
latest = release;
|
||||
}
|
||||
}
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Is there a version of the package available in the repositories that is more
|
||||
* latest than our latest installed version of the package?
|
||||
*/
|
||||
public boolean hasPackageUpdate(String packageName) {
|
||||
SolrPackage pkg = getPackagesMap().get(packageName);
|
||||
if (pkg == null) {
|
||||
return false;
|
||||
}
|
||||
String installedVersion = packageManager.getPackageInstance(packageName, null).version;
|
||||
SolrPackageRelease last = getLastPackageRelease(packageName);
|
||||
return last != null && PackageUtils.compareVersions(last.version, installedVersion) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Install a version of the package. Also, run verify commands in case some
|
||||
* collection was using {@link PackagePluginHolder#LATEST} version of this package and got auto-updated.
|
||||
*/
|
||||
public void install(String packageName, String version) throws SolrException {
|
||||
String latestVersion = getLastPackageRelease(packageName).version;
|
||||
|
||||
Map<String, String> collectionsDeployedIn = packageManager.getDeployedCollections(packageName);
|
||||
List<String> peggedToLatest = collectionsDeployedIn.keySet().stream().
|
||||
filter(collection -> collectionsDeployedIn.get(collection).equals(PackagePluginHolder.LATEST)).collect(Collectors.toList());
|
||||
if (!peggedToLatest.isEmpty()) {
|
||||
PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+peggedToLatest);
|
||||
}
|
||||
|
||||
if (version == null || version.equals(PackageUtils.LATEST)) {
|
||||
installPackage(packageName, latestVersion);
|
||||
} else {
|
||||
installPackage(packageName, version);
|
||||
}
|
||||
|
||||
SolrPackageInstance updatedPackage = packageManager.getPackageInstance(packageName, PackageUtils.LATEST);
|
||||
boolean res = packageManager.verify(updatedPackage, peggedToLatest);
|
||||
PackageUtils.printGreen("Verifying version " + updatedPackage.version +
|
||||
" on " + peggedToLatest + ", result: " + res);
|
||||
if (!res) throw new SolrException(ErrorCode.BAD_REQUEST, "Failed verification after deployment");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.common.annotation.JsonProperty;
|
||||
import org.apache.solr.common.util.ReflectMapWriter;
|
||||
|
||||
/**
|
||||
* Describes a package (along with all released versions) as it appears in a repository.
|
||||
*/
|
||||
public class SolrPackage implements Comparable<SolrPackage>, ReflectMapWriter {
|
||||
|
||||
@JsonProperty("name")
|
||||
public String name;
|
||||
|
||||
@JsonProperty("description")
|
||||
public String description;
|
||||
|
||||
@JsonProperty("versions")
|
||||
public List<SolrPackageRelease> versions;
|
||||
|
||||
@JsonProperty("repository")
|
||||
private String repository;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return jsonStr();
|
||||
}
|
||||
|
||||
public static class SolrPackageRelease implements ReflectMapWriter {
|
||||
@JsonProperty("version")
|
||||
public String version;
|
||||
|
||||
@JsonProperty("date")
|
||||
public Date date;
|
||||
|
||||
@JsonProperty("artifacts")
|
||||
public List<Artifact> artifacts;
|
||||
|
||||
@JsonProperty("manifest")
|
||||
public Manifest manifest;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return jsonStr();
|
||||
}
|
||||
}
|
||||
|
||||
public static class Artifact implements ReflectMapWriter {
|
||||
@JsonProperty("url")
|
||||
public String url;
|
||||
|
||||
@JsonProperty("sig")
|
||||
public String sig;
|
||||
}
|
||||
|
||||
public static class Manifest implements ReflectMapWriter {
|
||||
@JsonProperty("version-constraint")
|
||||
public String versionConstraint;
|
||||
|
||||
@JsonProperty("plugins")
|
||||
public List<Plugin> plugins;
|
||||
|
||||
@JsonProperty("parameter-defaults")
|
||||
public Map<String, String> parameterDefaults;
|
||||
}
|
||||
|
||||
public static class Plugin implements ReflectMapWriter {
|
||||
public String name;
|
||||
@JsonProperty("setup-command")
|
||||
public Command setupCommand;
|
||||
|
||||
@JsonProperty("uninstall-command")
|
||||
public Command uninstallCommand;
|
||||
|
||||
@JsonProperty("verify-command")
|
||||
public Command verifyCommand;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return jsonStr();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(SolrPackage o) {
|
||||
return name.compareTo(o.name);
|
||||
}
|
||||
|
||||
public String getRepository() {
|
||||
return repository;
|
||||
}
|
||||
|
||||
public void setRepository(String repository) {
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
public static class Command implements ReflectMapWriter {
|
||||
@JsonProperty("path")
|
||||
public String path;
|
||||
|
||||
@JsonProperty("method")
|
||||
public String method;
|
||||
|
||||
@JsonProperty("payload")
|
||||
public Map<String, Object> payload;
|
||||
|
||||
@JsonProperty("condition")
|
||||
public String condition;
|
||||
|
||||
@JsonProperty("expected")
|
||||
public String expected;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return jsonStr();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.packagemanager;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.common.annotation.JsonProperty;
|
||||
import org.apache.solr.common.util.ReflectMapWriter;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Manifest;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Plugin;
|
||||
|
||||
/**
|
||||
* Describes one instance of a package as it exists in Solr when installed.
|
||||
*/
|
||||
public class SolrPackageInstance implements ReflectMapWriter {
|
||||
@JsonProperty("name")
|
||||
final public String name;
|
||||
|
||||
final public String description;
|
||||
|
||||
@JsonProperty("version")
|
||||
final public String version;
|
||||
|
||||
final public Manifest manifest;
|
||||
|
||||
final public List<Plugin> plugins;
|
||||
|
||||
final public Map<String, String> parameterDefaults;
|
||||
|
||||
public SolrPackageInstance(String id, String description, String version, Manifest manifest,
|
||||
List<Plugin> plugins, Map<String, String> params) {
|
||||
this.name = id;
|
||||
this.description = description;
|
||||
this.version = version;
|
||||
this.manifest = manifest;
|
||||
this.plugins = plugins;
|
||||
this.parameterDefaults = params;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == null) return false;
|
||||
return name.equals(((SolrPackageInstance)obj).name) && version.equals(((SolrPackageInstance)obj).version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return jsonStr();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* This package contains Package Manager (CLI) implementation
|
||||
*/
|
||||
package org.apache.solr.packagemanager;
|
||||
|
|
@ -180,12 +180,20 @@ public class PackageAPI {
|
|||
@JsonProperty
|
||||
public List<String> files;
|
||||
|
||||
@JsonProperty
|
||||
public String manifest;
|
||||
|
||||
@JsonProperty
|
||||
public String manifestSHA512;
|
||||
|
||||
public PkgVersion() {
|
||||
}
|
||||
|
||||
public PkgVersion(Package.AddVersion addVersion) {
|
||||
this.version = addVersion.version;
|
||||
this.files = addVersion.files;
|
||||
this.manifest = addVersion.manifest;
|
||||
this.manifestSHA512 = addVersion.manifestSHA512;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,293 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.util;
|
||||
|
||||
import static org.apache.solr.packagemanager.PackageUtils.printGreen;
|
||||
import static org.apache.solr.packagemanager.PackageUtils.print;
|
||||
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.cli.OptionBuilder;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.core.config.Configurator;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
import org.apache.solr.client.solrj.impl.HttpClientUtil;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.util.Pair;
|
||||
import org.apache.solr.packagemanager.PackageManager;
|
||||
import org.apache.solr.packagemanager.PackageUtils;
|
||||
import org.apache.solr.packagemanager.RepositoryManager;
|
||||
import org.apache.solr.packagemanager.SolrPackage;
|
||||
import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
|
||||
import org.apache.solr.packagemanager.SolrPackageInstance;
|
||||
import org.apache.solr.util.SolrCLI.StatusTool;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
public class PackageTool extends SolrCLI.ToolBase {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
@SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.")
|
||||
public PackageTool() {
|
||||
// Need a logging free, clean output going through to the user.
|
||||
Configurator.setRootLevel(Level.OFF);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "package";
|
||||
}
|
||||
|
||||
public static String solrUrl = null;
|
||||
public static String solrBaseUrl = null;
|
||||
public PackageManager packageManager;
|
||||
public RepositoryManager repositoryManager;
|
||||
|
||||
@Override
|
||||
@SuppressForbidden(reason = "We really need to print the stacktrace here, otherwise "
|
||||
+ "there shall be little else information to debug problems. Other SolrCLI tools "
|
||||
+ "don't print stack traces, hence special treatment is needed here.")
|
||||
protected void runImpl(CommandLine cli) throws Exception {
|
||||
try {
|
||||
solrUrl = cli.getOptionValues("solrUrl")[cli.getOptionValues("solrUrl").length-1];
|
||||
solrBaseUrl = solrUrl.replaceAll("\\/solr$", ""); // strip out ending "/solr"
|
||||
log.info("Solr url: "+solrUrl+", solr base url: "+solrBaseUrl);
|
||||
String zkHost = getZkHost(cli);
|
||||
|
||||
log.info("ZK: "+zkHost);
|
||||
String cmd = cli.getArgList().size() == 0? "help": cli.getArgs()[0];
|
||||
|
||||
try (HttpSolrClient solrClient = new HttpSolrClient.Builder(solrBaseUrl).build()) {
|
||||
if (cmd != null) {
|
||||
packageManager = new PackageManager(solrClient, solrBaseUrl, zkHost);
|
||||
try {
|
||||
repositoryManager = new RepositoryManager(solrClient, packageManager);
|
||||
|
||||
switch (cmd) {
|
||||
case "add-repo":
|
||||
String repoName = cli.getArgs()[1];
|
||||
String repoUrl = cli.getArgs()[2];
|
||||
repositoryManager.addRepository(repoName, repoUrl);
|
||||
PackageUtils.printGreen("Added repository: " + repoName);
|
||||
break;
|
||||
case "list-installed":
|
||||
PackageUtils.printGreen("Installed packages:\n-----");
|
||||
for (SolrPackageInstance pkg: packageManager.fetchInstalledPackageInstances()) {
|
||||
PackageUtils.printGreen(pkg);
|
||||
}
|
||||
break;
|
||||
case "list-available":
|
||||
PackageUtils.printGreen("Available packages:\n-----");
|
||||
for (SolrPackage pkg: repositoryManager.getPackages()) {
|
||||
PackageUtils.printGreen(pkg.name + " \t\t"+pkg.description);
|
||||
for (SolrPackageRelease version: pkg.versions) {
|
||||
PackageUtils.printGreen("\tVersion: "+version.version);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "list-deployed":
|
||||
if (cli.hasOption('c')) {
|
||||
String collection = cli.getArgs()[1];
|
||||
Map<String, SolrPackageInstance> packages = packageManager.getPackagesDeployed(collection);
|
||||
PackageUtils.printGreen("Packages deployed on " + collection + ":");
|
||||
for (String packageName: packages.keySet()) {
|
||||
PackageUtils.printGreen("\t" + packages.get(packageName));
|
||||
}
|
||||
} else {
|
||||
String packageName = cli.getArgs()[1];
|
||||
Map<String, String> deployedCollections = packageManager.getDeployedCollections(packageName);
|
||||
PackageUtils.printGreen("Collections on which package " + packageName + " was deployed:");
|
||||
for (String collection: deployedCollections.keySet()) {
|
||||
PackageUtils.printGreen("\t" + collection + "("+packageName+":"+deployedCollections.get(collection)+")");
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "install":
|
||||
{
|
||||
Pair<String, String> parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString());
|
||||
String packageName = parsedVersion.first();
|
||||
String version = parsedVersion.second();
|
||||
repositoryManager.install(packageName, version);
|
||||
PackageUtils.printGreen(repositoryManager.toString() + " installed.");
|
||||
break;
|
||||
}
|
||||
case "deploy":
|
||||
{
|
||||
Pair<String, String> parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString());
|
||||
String packageName = parsedVersion.first();
|
||||
String version = parsedVersion.second();
|
||||
boolean noprompt = cli.hasOption('y');
|
||||
boolean isUpdate = cli.hasOption("update") || cli.hasOption('u');
|
||||
packageManager.deploy(packageName, version, PackageUtils.validateCollections(cli.getOptionValues("collections")), cli.getOptionValues("param"), isUpdate, noprompt);
|
||||
break;
|
||||
}
|
||||
case "undeploy":
|
||||
{
|
||||
Pair<String, String> parsedVersion = parsePackageVersion(cli.getArgList().get(1).toString());
|
||||
if (parsedVersion.second() != null) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Only package name expected, without a version. Actual: " + cli.getArgList().get(1));
|
||||
}
|
||||
String packageName = parsedVersion.first();
|
||||
packageManager.undeploy(packageName, cli.getOptionValues("collections"));
|
||||
break;
|
||||
}
|
||||
case "help":
|
||||
case "usage":
|
||||
print("Package Manager\n---------------");
|
||||
printGreen("./solr package add-repo <repository-name> <repository-url>");
|
||||
print("Add a repository to Solr.");
|
||||
print("");
|
||||
printGreen("./solr package install <package-name>[:<version>] ");
|
||||
print("Install a package into Solr. This copies over the artifacts from the repository into Solr's internal package store and sets up classloader for this package to be used.");
|
||||
print("");
|
||||
printGreen("./solr package deploy <package-name>[:<version>] [-y] [--update] -collections <comma-separated-collections> [-p <param1>=<val1> -p <param2>=<val2> ...] ");
|
||||
print("Bootstraps a previously installed package into the specified collections. It the package accepts parameters for its setup commands, they can be specified (as per package documentation).");
|
||||
print("");
|
||||
printGreen("./solr package list-installed");
|
||||
print("Print a list of packages installed in Solr.");
|
||||
print("");
|
||||
printGreen("./solr package list-available");
|
||||
print("Print a list of packages available in the repositories.");
|
||||
print("");
|
||||
printGreen("./solr package list-deployed -c <collection>");
|
||||
print("Print a list of packages deployed on a given collection.");
|
||||
print("");
|
||||
printGreen("./solr package list-deployed <package-name>");
|
||||
print("Print a list of collections on which a given package has been deployed.");
|
||||
print("");
|
||||
printGreen("./solr package undeploy <package-name> -collections <comma-separated-collections>");
|
||||
print("Undeploys a package from specified collection(s)");
|
||||
print("\n");
|
||||
print("Note: (a) Please add '-solrUrl http://host:port' parameter if needed (usually on Windows).");
|
||||
print(" (b) Please make sure that all Solr nodes are started with '-Denable.packages=true' parameter.");
|
||||
print("\n");
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Unrecognized command: "+cmd);
|
||||
};
|
||||
} finally {
|
||||
packageManager.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info("Finished: "+cmd);
|
||||
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace(); // We need to print this since SolrCLI drops the stack trace in favour of brevity. Package tool should surely print full stacktraces!
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses package name and version in the format "name:version" or "name"
|
||||
* @return A pair of package name (first) and version (second)
|
||||
*/
|
||||
private Pair<String, String> parsePackageVersion(String arg) {
|
||||
String splits[] = arg.split(":");
|
||||
if (splits.length > 2) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid package name: " + arg +
|
||||
". Didn't match the pattern: <packagename>:<version> or <packagename>");
|
||||
}
|
||||
|
||||
String packageName = splits[0];
|
||||
String version = splits.length == 2? splits[1]: null;
|
||||
return new Pair(packageName, version);
|
||||
}
|
||||
|
||||
@SuppressWarnings("static-access")
|
||||
public Option[] getOptions() {
|
||||
return new Option[] {
|
||||
OptionBuilder
|
||||
.withArgName("URL")
|
||||
.hasArg()
|
||||
.isRequired(true)
|
||||
.withDescription("Address of the Solr Web application, defaults to: " + SolrCLI.DEFAULT_SOLR_URL)
|
||||
.create("solrUrl"),
|
||||
|
||||
OptionBuilder
|
||||
.withArgName("COLLECTIONS")
|
||||
.hasArgs()
|
||||
.isRequired(false)
|
||||
.withDescription("List of collections. Run './solr package help' for more details.")
|
||||
.create("collections"),
|
||||
|
||||
OptionBuilder
|
||||
.withArgName("PARAMS")
|
||||
.hasArgs()
|
||||
.isRequired(false)
|
||||
.withDescription("List of parameters to be used with deploy command. Run './solr package help' for more details.")
|
||||
.withLongOpt("param")
|
||||
.create("p"),
|
||||
|
||||
OptionBuilder
|
||||
.isRequired(false)
|
||||
.withDescription("If a deployment is an update over a previous deployment. Run './solr package help' for more details.")
|
||||
.withLongOpt("update")
|
||||
.create("u"),
|
||||
|
||||
OptionBuilder
|
||||
.isRequired(false)
|
||||
.withDescription("Run './solr package help' for more details.")
|
||||
.withLongOpt("collection")
|
||||
.create("c"),
|
||||
|
||||
OptionBuilder
|
||||
.isRequired(false)
|
||||
.withDescription("Run './solr package help' for more details.")
|
||||
.withLongOpt("noprompt")
|
||||
.create("y")
|
||||
};
|
||||
}
|
||||
|
||||
private String getZkHost(CommandLine cli) throws Exception {
|
||||
String zkHost = cli.getOptionValue("zkHost");
|
||||
if (zkHost != null)
|
||||
return zkHost;
|
||||
|
||||
String systemInfoUrl = solrUrl+"/admin/info/system";
|
||||
CloseableHttpClient httpClient = SolrCLI.getHttpClient();
|
||||
try {
|
||||
// hit Solr to get system info
|
||||
Map<String,Object> systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true);
|
||||
|
||||
// convert raw JSON into user-friendly output
|
||||
StatusTool statusTool = new StatusTool();
|
||||
Map<String,Object> status = statusTool.reportStatus(solrUrl+"/", systemInfo, httpClient);
|
||||
Map<String,Object> cloud = (Map<String, Object>)status.get("cloud");
|
||||
if (cloud != null) {
|
||||
String zookeeper = (String) cloud.get("ZooKeeper");
|
||||
if (zookeeper.endsWith("(embedded)")) {
|
||||
zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length());
|
||||
}
|
||||
zkHost = zookeeper;
|
||||
}
|
||||
} finally {
|
||||
HttpClientUtil.close(httpClient);
|
||||
}
|
||||
|
||||
return zkHost;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -417,6 +417,10 @@ public class SolrCLI {
|
|||
return new AuthTool();
|
||||
else if ("autoscaling".equals(toolType))
|
||||
return new AutoscalingTool();
|
||||
else if ("export".equals(toolType))
|
||||
return new ExportTool();
|
||||
else if ("package".equals(toolType))
|
||||
return new PackageTool();
|
||||
|
||||
// If you add a built-in tool to this class, add it here to avoid
|
||||
// classpath scanning
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIBPAIBAAJBAOkIQ4lxVDPPIR2noh4MrTVw3JASYXd6k0wajrSDFYlwepVoSO/f
|
||||
LK1AHYomrub8C8dXZwxzPMqpDVQ4MABdgcECAwEAAQJBAKzsLPG43zry4SgYVPzn
|
||||
e0DE12cxvJHkq5k1u9/HxhuNqUdfhKzkMW5BqayDT74UqSLLTT9zFpGPsRUHD37V
|
||||
aAECIQD7HV0vP9ipMskCMA+x8H8X/F2V2b6RWGlfIDc6XEiE8QIhAO2Q197gMnzP
|
||||
z3VluoeL9pORw55aqx7kuAXwqmUqRInRAiBX3wWdpBTX2EqYdmL3nDWNGiVRa5mQ
|
||||
2MQ+olJRHLvPsQIhAL0w/LmiEpMTbEQyH7qS3GvpScBytJSF0YfpgcnPP4YBAiEA
|
||||
14io0rN1QgVWykmckeZZSzaPLqpkE2wCUZUOemoBByg=
|
||||
-----END RSA PRIVATE KEY-----
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,56 @@
|
|||
[
|
||||
{
|
||||
"name": "question-answer",
|
||||
"description": "A natural language question answering plugin",
|
||||
"versions": [
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"date": "2019-01-01",
|
||||
"artifacts": [
|
||||
{
|
||||
"url": "question-answer-request-handler-1.0.jar",
|
||||
"sig": "TTzgh5/usbyWg7oZ7lRwz4eQfh1FeXWvv4U85tsVthVz0MRDz9t7SmonDkegZ7OyqeoiQ4I207pifpVW+DRd9Q=="
|
||||
}
|
||||
],
|
||||
"manifest": {
|
||||
"version-constraint": "8 - 9",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "request-handler",
|
||||
"setup-command": {
|
||||
"path": "/api/collections/${collection}/config",
|
||||
"payload": {"add-requesthandler": {"name": "${RH-HANDLER-PATH}", "class": "question-answer:fullstory.QARequestHandler"}},
|
||||
"method": "POST"
|
||||
},
|
||||
"uninstall-command": {
|
||||
"path": "/api/collections/${collection}/config",
|
||||
"payload": {"delete-requesthandler": "${RH-HANDLER-PATH}"},
|
||||
"method": "POST"
|
||||
},
|
||||
"verify-command": {
|
||||
"path": "/api/collections/${collection}/config/requestHandler?componentName=${RH-HANDLER-PATH}&meta=true",
|
||||
"method": "GET",
|
||||
"condition": "$['config'].['requestHandler'].['${RH-HANDLER-PATH}'].['_packageinfo_'].['version']",
|
||||
"expected": "${package-version}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameter-defaults": {
|
||||
"RH-HANDLER-PATH": "/mypath"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"version": "1.1.0",
|
||||
"date": "2019-01-01",
|
||||
"artifacts": [
|
||||
{
|
||||
"url": "question-answer-request-handler-1.1.jar",
|
||||
"sig": "LypqlmbJ76AWa5jx0XhjKxO4lrcQAvpSuYddfzcE6TnX0VDPFhrlQHSSX6cZLtvNbQ+74xUMKgsoX1IUnEnKYw=="
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.cloud;
|
||||
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.core.config.Configurator;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.core.TestSolrConfigHandler;
|
||||
import org.apache.solr.util.PackageTool;
|
||||
import org.apache.solr.util.SolrCLI;
|
||||
import org.eclipse.jetty.server.Handler;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.server.ServerConnector;
|
||||
import org.eclipse.jetty.server.handler.DefaultHandler;
|
||||
import org.eclipse.jetty.server.handler.HandlerList;
|
||||
import org.eclipse.jetty.server.handler.ResourceHandler;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class PackageManagerCLITest extends SolrCloudTestCase {
|
||||
|
||||
// Note for those who want to modify the jar files used in the packages used in this test:
|
||||
// You need to re-sign the jars for install step, as follows:
|
||||
// $ openssl dgst -sha1 -sign ./solr/core/src/test-files/solr/question-answer-repository-private-key.pem ./solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar | openssl enc -base64
|
||||
// You can place the new signature thus obtained (removing any whitespaces) in the repository.json.
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private static LocalWebServer repositoryServer;
|
||||
|
||||
@BeforeClass
|
||||
public static void setupCluster() throws Exception {
|
||||
System.setProperty("enable.packages", "true");
|
||||
|
||||
configureCluster(1)
|
||||
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
|
||||
.addConfig("conf2", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
|
||||
.configure();
|
||||
|
||||
repositoryServer = new LocalWebServer(TEST_PATH().resolve("question-answer-repository").toString());
|
||||
repositoryServer.start();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void teardown() throws Exception {
|
||||
repositoryServer.stop();
|
||||
System.clearProperty("enable.packages");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.")
|
||||
public void testPackageManager() throws Exception {
|
||||
PackageTool tool = new PackageTool();
|
||||
|
||||
// Enable the logger for this test. Need to do this since the tool disables logger.
|
||||
Configurator.setRootLevel(Level.INFO);
|
||||
|
||||
String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "add-repo", "fullstory", "http://localhost:" + repositoryServer.getPort()});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-available"});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer:1.0.0"});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"});
|
||||
|
||||
CollectionAdminRequest.createCollection("abc", "conf1", 1, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("def", "conf2", 1, 1).process(cluster.getSolrClient());
|
||||
|
||||
String rhPath = "/mypath2";
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer", "-y", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath});
|
||||
assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0");
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "-c", "abc"});
|
||||
|
||||
// Should we test the "auto-update to latest" functionality or the default explicit deploy functionality
|
||||
boolean autoUpdateToLatest = random().nextBoolean();
|
||||
|
||||
if (autoUpdateToLatest) {
|
||||
log.info("Testing auto-update to latest installed");
|
||||
|
||||
// This command pegs the version to the latest available
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer:latest", "-y", "-collections", "abc"});
|
||||
assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.0.0");
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer"});
|
||||
assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.1.0");
|
||||
} else {
|
||||
log.info("Testing explicit deployment to a different/newer version");
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer"});
|
||||
assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0");
|
||||
|
||||
if (random().nextBoolean()) { // even if parameters are not passed in, they should be picked up from previous deployment
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "-y", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath});
|
||||
} else {
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "-y", "question-answer", "-collections", "abc"});
|
||||
}
|
||||
assertPackageVersion("abc", "question-answer", "1.1.0", rhPath, "1.1.0");
|
||||
}
|
||||
|
||||
log.info("Running undeploy...");
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "undeploy", "question-answer", "-collections", "abc"});
|
||||
|
||||
run(tool, new String[] {"-solrUrl", solrUrl, "list-deployed", "question-answer"});
|
||||
|
||||
}
|
||||
|
||||
void assertPackageVersion(String collection, String pkg, String version, String component, String componentVersion) throws Exception {
|
||||
TestSolrConfigHandler.testForResponseElement(
|
||||
null,
|
||||
cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection,
|
||||
"/config/params?meta=true",
|
||||
cluster.getSolrClient(),
|
||||
Arrays.asList("response", "params", "PKG_VERSIONS", pkg),
|
||||
version,
|
||||
1);
|
||||
|
||||
TestSolrConfigHandler.testForResponseElement(
|
||||
null,
|
||||
cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection,
|
||||
"/config/requestHandler?componentName=" + component + "&meta=true",
|
||||
cluster.getSolrClient(),
|
||||
Arrays.asList("config", "requestHandler", component, "_packageinfo_", "version"),
|
||||
componentVersion,
|
||||
1);
|
||||
}
|
||||
|
||||
private void run(PackageTool tool, String[] args) throws Exception {
|
||||
int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
|
||||
assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res);
|
||||
}
|
||||
|
||||
static class LocalWebServer {
|
||||
private int port = 0;
|
||||
final private String resourceDir;
|
||||
Server server;
|
||||
ServerConnector connector;
|
||||
|
||||
public LocalWebServer(String resourceDir) {
|
||||
this.resourceDir = resourceDir;
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return connector != null? connector.getLocalPort(): port;
|
||||
}
|
||||
|
||||
public void start() throws Exception {
|
||||
server = new Server();
|
||||
|
||||
connector = new ServerConnector(server);
|
||||
connector.setPort(port);
|
||||
server.addConnector(connector);
|
||||
server.setStopAtShutdown(true);
|
||||
|
||||
ResourceHandler resourceHandler = new ResourceHandler();
|
||||
resourceHandler.setResourceBase(resourceDir);
|
||||
resourceHandler.setDirectoriesListed(true);
|
||||
|
||||
HandlerList handlers = new HandlerList();
|
||||
handlers.setHandlers(new Handler[] {resourceHandler, new DefaultHandler()});
|
||||
server.setHandler(handlers);
|
||||
|
||||
server.start();
|
||||
}
|
||||
|
||||
public void stop() throws Exception {
|
||||
server.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
59a83ca73c72a5e25b3f0b1bb305230a11000329
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
The MIT License
|
||||
|
||||
Copyright 2012-2014 Zafar Khaja <zafarkhaja@gmail.com>.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
|
@ -33,6 +33,10 @@ public class Package {
|
|||
public String version;
|
||||
@JsonProperty(required = true)
|
||||
public List<String> files;
|
||||
@JsonProperty
|
||||
public String manifest;
|
||||
@JsonProperty
|
||||
public String manifestSHA512;
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue