mirror of https://github.com/apache/lucene.git
SOLR-13662, SOLR-13822: Fixing bug with refresh API, un-ignoring the test and new reference guide on Package Management
This commit is contained in:
Ishan Chattopadhyaya
parent
5200ee0ef9
commit
0857bb60d3
|
|
@ -235,7 +235,8 @@ public class PackageAPI {
|
|||
payload.addError("No such package: " + p);
|
||||
return;
|
||||
}
|
||||
|
||||
//first refresh my own
|
||||
packageLoader.notifyListeners(p);
|
||||
for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
|
||||
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
|
||||
coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&refreshPackage=" + p,
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ import org.eclipse.jetty.server.handler.HandlerList;
|
|||
import org.eclipse.jetty.server.handler.ResourceHandler;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -72,7 +71,6 @@ public class PackageManagerCLITest extends SolrCloudTestCase {
|
|||
|
||||
@Test
|
||||
@SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.")
|
||||
@Ignore
|
||||
public void testPackageManager() throws Exception {
|
||||
PackageTool tool = new PackageTool();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,24 +1,7 @@
|
|||
= Packages in Solr
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
= Package Manager Internals
|
||||
The package manager (CLI) internally uses various Solr APIs to install, deploy and update packages. This document contains an overview of those APIs.
|
||||
|
||||
A Package in Solr may contain one or more .jar files and any no:of plugins.
|
||||
|
||||
== Objectives
|
||||
== Design Objectives
|
||||
|
||||
* Zero disruption deployment (hot deployment): Should be possible to install and update packages without node restarts or core reloads, and hence deployments should be quick and without failed requests or stale caches.
|
||||
* Easy packaging:
|
||||
|
|
@ -26,8 +9,7 @@ A Package in Solr may contain one or more .jar files and any no:of plugins.
|
|||
** Artifacts (jars containing custom plugins) that users have already deployed (and are using in production) should be compatible, without needing to recompile or re-package, for greater adoption.
|
||||
** Should support single jar packages as well as multiple jar packages.
|
||||
** Use familiar / standard naming
|
||||
** Use industry standard concepts and terminology around package manager, similar to those like apt, dnf, homebrew etc.
|
||||
|
||||
** Use industry standard concepts and terminology around package manager, similar to those like apt, dnf, homebrew etc.s
|
||||
|
||||
== Classloaders
|
||||
|
||||
|
|
@ -35,25 +17,17 @@ At the heart of the system, we have classloader isolation. To achieve this, the
|
|||
The root classloader which has all the jars from Solr classpath. This requires Solr node restart to change anything.
|
||||
A set of named classloaders that inherit from the root classloader. The life cycles of the named classloaders are tied to the package configuration in ZK. As soon as the configuration is modified, the corresponding classloaders are reloaded and components are asked to reload.
|
||||
|
||||
|
||||
== A Plugin
|
||||
|
||||
A Plugin is an implementation of any of the standard Solr pluggable component. A Package has two components
|
||||
|
||||
* The jar file(s)
|
||||
* The configuration of the Package in Zookeeper in a file called `/packages.json`
|
||||
|
||||
=== Package loading security
|
||||
Packages are disabled by default. Start all your nodes with the system property `-Denable.package=true` to use this feature.
|
||||
== Package loading security
|
||||
Packages are disabled by default. Start all your nodes with the system property `-Denable.packages=true` to use this feature.
|
||||
|
||||
*Example*
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr -e cloud -noprompt -a "-Denable.packages=true"
|
||||
$ bin/solr -c -Denable.packages=true
|
||||
----
|
||||
|
||||
==== Upload your keys
|
||||
=== Upload your keys
|
||||
Package binaries must be signed with your private keys and ensure your public keys are published in Zookeeper
|
||||
|
||||
*Example*
|
||||
|
|
@ -93,9 +67,9 @@ The end points are
|
|||
* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the file
|
||||
* `GET /api/node/files/{full/path/to/}` get the list of files in `/full/path/to`
|
||||
|
||||
=== Signing your artifacts
|
||||
|
||||
|
||||
Use the following steps to upload a jar signed with your public key
|
||||
Use the following steps to upload a jar signed with your public key:
|
||||
|
||||
1) If you don't have a jar file with plugins, download a sample from github
|
||||
|
||||
|
|
@ -125,22 +99,22 @@ $ curl http://localhost:7574/api/node/files/mypkg/1.0?omitHeader=true
|
|||
----
|
||||
{
|
||||
"files":{"/mypkg/1.0":[{
|
||||
"name":"myplugins.jar",
|
||||
"timestamp":"2019-11-11T07:36:17.354Z",
|
||||
"sha512":"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
|
||||
"sig":["elNjhmWIOgTgbAzeZ+OcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ=="]}]}}
|
||||
"name":"myplugins.jar",
|
||||
"timestamp":"2019-11-11T07:36:17.354Z",
|
||||
"sha512":"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
|
||||
"sig":["elNjhmWIOgTgbAzeZ+OcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ=="]}]}}
|
||||
----
|
||||
|
||||
|
||||
== Packages
|
||||
A Package has the following attributes:
|
||||
A Package have the following attributes:
|
||||
|
||||
* A unique name
|
||||
* One or more versions with the following attributes:
|
||||
** `version` : The version string
|
||||
** `files` : An array of files from the package store
|
||||
|
||||
For every package/version in the packages definition, there is a unique `SolrResourceLoader` instance. This is a child of the `CoreContainer` resource loader.
|
||||
For every package / version in the packages definition, there is a unique `SolrResourceLoader` instance. This is a child of the `CoreContainer` resource loader.
|
||||
|
||||
=== `packages.json`
|
||||
|
||||
|
|
@ -164,20 +138,20 @@ The package configurations live in a file called /packages.json in Zookeeper. At
|
|||
|
||||
----
|
||||
|
||||
=== API end points
|
||||
== API end points
|
||||
|
||||
* `GET /api/cluster/package` Get the list of packages
|
||||
* `POST /api/cluster/package` edit packages
|
||||
** `add` command: add a version of a package
|
||||
** `delete` command : delete a version of a package
|
||||
|
||||
==== How to upgrade?
|
||||
=== How to upgrade?
|
||||
Use the add command to add a version that is bigger than the current version
|
||||
|
||||
==== How to downgrade ?
|
||||
=== How to downgrade ?
|
||||
Use the delete command to delete a version that is the highest and the next highest version will be chosen
|
||||
|
||||
==== Using multiple versions in parallel in each collection
|
||||
=== Using multiple versions in parallel in each collection
|
||||
We use the `params.json` in the collection config to store a version of a package it uses. by default it is the $LATEST.
|
||||
[source, json]
|
||||
----
|
||||
|
|
@ -190,14 +164,14 @@ We use the `params.json` in the collection config to store a version of a packag
|
|||
* for `mypkg` use the version `0.1` irrespective of whether there is a newer version is available or not
|
||||
* for `pkg2` use the latest. This is optional. The default is ‘$LATEST’
|
||||
|
||||
==== The workflow
|
||||
=== The workflow
|
||||
* A new version of a package is added
|
||||
* The package loader loads the classes and notifies every plugin holder of the availability of the new version
|
||||
* It checks if it is supposed to use a specific version, Ignore the update
|
||||
* If not, reload the plugin
|
||||
|
||||
|
||||
==== Using them in Plugins
|
||||
=== Using them in Plugins
|
||||
Any class name can be prefixed with the packagename e.g : `mypkg:fully.qualified.ClassName` and Solr would use the latest version of the package to load the classes from. The plugins loaded from packages cannot depend on core level classes.
|
||||
|
||||
*Plugin declaration in `solrconfig.xml`*
|
||||
|
|
@ -208,7 +182,7 @@ Any class name can be prefixed with the packagename e.g : `mypkg:fully.qualified
|
|||
</requestHandler>
|
||||
----
|
||||
|
||||
==== Full Working example
|
||||
=== Full working example
|
||||
|
||||
1) create a package
|
||||
|
||||
|
|
@ -365,7 +339,7 @@ $ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true
|
|||
----
|
||||
Note that the `Version` value is `"2"` . So the plugin is updated
|
||||
|
||||
==== How to avoid automatic upgrade?
|
||||
=== How to avoid automatic upgrade (while only using internal APIs)?
|
||||
|
||||
The default version used in any collection is always the latest. However, setting a per-collection property in the `params.json` ensures that the versions are always fixed irrespective of the new versions added
|
||||
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
= Package Management
|
||||
:page-children: package-manager-internals
|
||||
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
== Glossary of terms
|
||||
|
||||
=== Package
|
||||
A set of jar files (usually one) containing one or more <<solr-plugins.adoc#solr-plugins,Solr plugins>>. Each jar file is also accompanied by a signature string (which can be verified against a supplied public key).
|
||||
|
||||
=== Repository
|
||||
A location hosting one or many packages. Usually, this is a web service that serves meta information about packages as well as serves the package artifacts for downloading.
|
||||
|
||||
== Overview
|
||||
The package manager in Solr consists of the following internal components:
|
||||
|
||||
* Package Manager CLI
|
||||
* Package Manager internal APIs
|
||||
* Isolated classloaders
|
||||
* Package Store
|
||||
|
||||
In this guide, we will focus on the Package Manager CLI, which essentially uses the other APIs and components internally. For details on the other components (and hence details of inner workings of the package manager), please refer to <<package-manager-internals.adoc#package-manager-internals,Package Manager internals>>.
|
||||
|
||||
== Interacting with the package manager
|
||||
|
||||
Essentially, these are the phases in using the package manager:
|
||||
|
||||
* Starting Solr with support for package management
|
||||
* Adding trusted repositories
|
||||
* Listing and installing packages
|
||||
* Deploying packages on to collections
|
||||
* Updating packages
|
||||
|
||||
=== Starting Solr with package management support
|
||||
|
||||
Start all Solr nodes with the `-Denable.packages=true` parameter. There are security consequences in doing so. At a minimum, no unauthorized user should have write access to Zookeeper instances, since it would then be possible to install packages from untrusted sources (e.g. malicious repositories).
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr -c -Denable.packages=true
|
||||
----
|
||||
|
||||
=== Adding trusted repositories
|
||||
|
||||
In order to install packages into Solr, one has to add a repository hosting the packages. A repository is essentially a web service hosting package artifacts (jar files) and a public key (to validate the signatures of the jar files while installing). Note: Please do not add repositories that you don't trust or control. Also, only add repositories that are based on https and avoid repositories based on http to safeguard against MITM attacks.
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr package add-repo <name-of-repo> <repo-url>
|
||||
----
|
||||
|
||||
=== Listing and installing packages
|
||||
|
||||
To list installed packages:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr package list-installed
|
||||
----
|
||||
|
||||
|
||||
To list packages available for installation from added repositories:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr package list-available
|
||||
----
|
||||
|
||||
To install a package:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr package install <package-name>[:<version>]
|
||||
----
|
||||
|
||||
=== Deploying a package on to a collection
|
||||
|
||||
Once a package has been installed, the plugins contained in it can be used in a collection, using either of the two methods:
|
||||
|
||||
==== Deploying using `deploy` command
|
||||
This can be done using the package manager's `deploy` command, provided the package supports it (package author's documentation would usually mention that):
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
$ bin/solr package deploy <package-name>:[version] -collections <collection1>[,<collection2>,...]
|
||||
----
|
||||
|
||||
This may prompt you to execute a command to deploy the package. If you pass `-y` to the command, then this prompt can be skipped.
|
||||
|
||||
==== Deploying manually
|
||||
Alternatively, it is also possible manually edit a configset (solrconfig.xml, managedschema / schema.xml etc.) and using it by RELOADing a collection.
|
||||
|
||||
Example: Add a request handler from the package `mypackage` to a configset's solrconfig.xml:
|
||||
|
||||
[source, xml]
|
||||
----
|
||||
<requestHandler name="/myhandler" class="mypackage:full.path.to.MyClass"></requestHandler>
|
||||
----
|
||||
|
||||
After that, `RELOAD` your collection. Now, you should set the package version that this collection is using, as follows (say collection is called `collection1` and package name is `mypackage` and installed version is `1.0.0`):
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
curl "http://localhost:8983/api/collections/collection1/config/params" \
|
||||
-H 'Content-type:application/json' -d "{set: {PKG_VERSIONS: {mypackage: '1.0.0'}}}"
|
||||
----
|
||||
|
||||
==== Verifying the deployment
|
||||
After deploying, verify that the collection is using the package:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr package list-deployed -c <collection>
|
||||
----
|
||||
|
||||
=== Updating packages
|
||||
|
||||
In order to update a package, first step is make sure the updated version is available in the added repositories by running `list-available` command. Next, install the new version of the package from the repositories.
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr package install <package-name>:<version>
|
||||
----
|
||||
|
||||
Now, you can selectively update each of your collections using the old version (say, `1.0.0`) of the package (say, `mypackage`) to the newly added version (say `2.0.0`) as follows:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
$ bin/solr package deploy mypackage:2.0.0 --update -collections mycollection
|
||||
----
|
||||
|
||||
You can run the `list-deployed` command to verify that this collection is using the newly added version.
|
||||
|
||||
== Security
|
||||
Except the `add-repo` step, all other steps can be executed using a HTTP endpoint in Solr (see <<package-manager-internals.adoc#package-manager-internals,Package Manager internals>>). This step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to Zookeeper. Hence, as you can imagine, it is important to protect Zookeeper from unauthorized write access.
|
||||
|
||||
Also, keep in mind, that it is possible to install any package from a trusted and an already added repository. Hence, if you want to use some packages in production, then it is better to setup your own repository and add that to Solr, instead of adding a generic third-party repository that is beyond your administrative control.
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
= The Well-Configured Solr Instance
|
||||
:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api, solr-packages
|
||||
:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api, package-manager
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
|
|
@ -31,7 +31,7 @@ This section covers the following topics:
|
|||
|
||||
<<solr-plugins.adoc#solr-plugins,Solr Plugins>>: Introduces Solr plugins with pointers to more information.
|
||||
|
||||
<<solr-packages.adoc#solr-packages, Packages in Solr>>: Deploying, installing, updating packages with plugins into Solr Cluster
|
||||
<<package-manager.adoc#glossary-of-terms, Packages and Package Management>>: Installing, deploying and updating packages (containing plugins) into a Solr cluster
|
||||
|
||||
<<jvm-settings.adoc#jvm-settings,JVM Settings>>: Gives some guidance on best practices for working with Java Virtual Machines.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue