mirror of https://github.com/apache/lucene.git
SOLR-13822: Ref guide and error handling
SOLR-13822: Ref guide and error handling for package loader, package store
This commit is contained in:
Noble Paul
GitHub
parent
abb7087f6e
commit
40661489cd
|
|
@ -156,6 +156,10 @@ public class PackageStoreAPI {
|
|||
if (signatures != null) {
|
||||
vals.put("sig", signatures);
|
||||
}
|
||||
PackageStore.FileType type = packageStore.getType(path, true);
|
||||
if(type != PackageStore.FileType.NOFILE) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path);
|
||||
}
|
||||
packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), path));
|
||||
rsp.add(CommonParams.FILE, path);
|
||||
} catch (IOException e) {
|
||||
|
|
@ -177,7 +181,7 @@ public class PackageStoreAPI {
|
|||
}
|
||||
|
||||
private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf)
|
||||
throws SolrException {
|
||||
throws SolrException, IOException {
|
||||
String[] signatures = req.getParams().getParams("sig");
|
||||
if (signatures == null || signatures.length == 0) return null;
|
||||
List<String> sigs = Arrays.asList(signatures);
|
||||
|
|
@ -186,7 +190,7 @@ public class PackageStoreAPI {
|
|||
}
|
||||
|
||||
public void validate(List<String> sigs,
|
||||
ByteBuffer buf) throws SolrException {
|
||||
ByteBuffer buf) throws SolrException, IOException {
|
||||
Map<String, byte[]> keys = CloudUtil.getTrustedKeys(
|
||||
coreContainer.getZkController().getZkClient(), "exe");
|
||||
if (keys == null || keys.isEmpty()) {
|
||||
|
|
@ -202,7 +206,8 @@ public class PackageStoreAPI {
|
|||
}
|
||||
for (String sig : sigs) {
|
||||
if (cryptoKeys.verify(sig, buf) == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig);
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.limit()+ " content sha512: "+
|
||||
DigestUtils.sha512Hex(new ByteBufferInputStream(buf)));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -337,7 +342,7 @@ public class PackageStoreAPI {
|
|||
"Error parsing public keys in ZooKeeper");
|
||||
}
|
||||
for (String sig : sigs) {
|
||||
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig;
|
||||
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig + "sha256 "+ entry.getMetaData().sha512;
|
||||
if (entry.getBuffer() != null) {
|
||||
if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
|
||||
|
|
|
|||
|
|
@ -186,6 +186,7 @@ public class HttpSolrCall {
|
|||
this.response = response;
|
||||
this.retry = retry;
|
||||
this.requestType = RequestType.UNKNOWN;
|
||||
req.setAttribute(HttpSolrCall.class.getName(), this);
|
||||
queryParams = SolrRequestParsers.parseQueryString(req.getQueryString());
|
||||
// set a request timer which can be reused by requests if needed
|
||||
req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
|
||||
|
|
@ -196,7 +197,6 @@ public class HttpSolrCall {
|
|||
// this lets you handle /update/commit when /update is a servlet
|
||||
path += req.getPathInfo();
|
||||
}
|
||||
req.setAttribute(HttpSolrCall.class.getName(), this);
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
|
|
|
|||
|
|
@ -63,7 +63,6 @@ import org.apache.solr.util.RTimerTree;
|
|||
import org.apache.solr.util.SolrFileCleaningTracker;
|
||||
import org.apache.solr.util.tracing.GlobalTracer;
|
||||
|
||||
import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE;
|
||||
import static org.apache.solr.common.params.CommonParams.PATH;
|
||||
|
||||
|
||||
|
|
@ -270,7 +269,10 @@ public class SolrRequestParsers
|
|||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
|
||||
private static HttpSolrCall getHttpSolrCall(HttpServletRequest req) {
|
||||
return req == null ? null : (HttpSolrCall) req.getAttribute(HttpSolrCall.class.getName());
|
||||
}
|
||||
/**
|
||||
* Given a url-encoded query string (UTF-8), map it into solr params
|
||||
*/
|
||||
|
|
@ -733,7 +735,7 @@ public class SolrRequestParsers
|
|||
String contentType = req.getContentType();
|
||||
String method = req.getMethod(); // No need to uppercase... HTTP verbs are case sensitive
|
||||
String uri = req.getRequestURI();
|
||||
boolean isRawPut = "PUT".equals(method) && BINARY_CONTENT_TYPE.equals(contentType);
|
||||
boolean isV2 = getHttpSolrCall(req) instanceof V2HttpCall;
|
||||
boolean isPost = "POST".equals(method);
|
||||
|
||||
// SOLR-6787 changed the behavior of a POST without content type. Previously it would throw an exception,
|
||||
|
|
@ -749,7 +751,10 @@ public class SolrRequestParsers
|
|||
// POST was handled normally, but other methods (PUT/DELETE)
|
||||
// were handled by restlet if the URI contained /schema or /config
|
||||
// "handled by restlet" means that we don't attempt to handle any request body here.
|
||||
if (!isPost && !isRawPut) {
|
||||
if (!isPost) {
|
||||
if (isV2) {
|
||||
return raw.parseParamsAndFillStreams(req, streams);
|
||||
}
|
||||
if (contentType == null) {
|
||||
return parseQueryString(req.getQueryString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,296 @@
|
|||
= Packages in Solr
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
A Package in Solr may contain one or more .jar files and any no:of plugins.
|
||||
|
||||
|
||||
== A Plugin
|
||||
|
||||
A Plugin is an implementation of any of the standard Solr pluggable component. A Package has two components
|
||||
|
||||
* The jar file(s)
|
||||
* The configuration of the Package in Zookeeper in a file called `/packages.json`
|
||||
|
||||
=== Package loading security
|
||||
Packages are disabled by default. Start all your nodes with the system property `-Denable.package=true` to use this feature.
|
||||
|
||||
*Example*
|
||||
[source,bash]
|
||||
----
|
||||
|
||||
$ bin/solr -e cloud -noprompt -a "-Denable.packages=true"
|
||||
----
|
||||
|
||||
==== Upload your keys
|
||||
Package binaries must be signed with your private keys and ensure your public keys are published in Zookeeper
|
||||
|
||||
*Example*
|
||||
[source,bash]
|
||||
----
|
||||
$ openssl genrsa -out my_key.pem 512
|
||||
# create the public key in .der format
|
||||
$ openssl rsa -in my_key.pem -pubout -outform DER -out my_key.der
|
||||
# upload to Zookeeper
|
||||
$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd makepath /keys/exe/
|
||||
$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd putfile /keys/exe/my_key.der my_key.der
|
||||
----
|
||||
|
||||
== Package Store
|
||||
Package store is a distributed file store which can store arbitrary files in the file system. It is a peer to peer hierarchical storage system. If a new node comes up and it requires any of these files it can be downloaded from one of the nodes where it is available.
|
||||
|
||||
=== Package Store APIs
|
||||
|
||||
The end points are
|
||||
|
||||
* `PUT /api/cluster/files/{full/path/to/file}` in each node
|
||||
* `GET /api/node/files/{full/path/to/file}` to download the file
|
||||
* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the file
|
||||
* `GET /api/node/files/{full/path/to/}` get the list of files in `/full/path/to`
|
||||
|
||||
|
||||
|
||||
Use the following steps to upload a jar signed with your public key
|
||||
|
||||
1) If you don't have a jar file with plugins, download a sample from github
|
||||
|
||||
[source, bash]
|
||||
----
|
||||
$ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true
|
||||
|
||||
----
|
||||
2) Sign the jar with your private key
|
||||
[source, bash]
|
||||
----
|
||||
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
|
||||
----
|
||||
|
||||
3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) . Ensure that new lines and spaces are removed. Do not forget to do URL encoding of you signature( e.g escape `+` with `%2B`)
|
||||
[source, bash]
|
||||
----
|
||||
$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ==
|
||||
----
|
||||
|
||||
4) Verify your jar upload
|
||||
[source, bash]
|
||||
----
|
||||
$ curl http://localhost:7574/api/node/files/mypkg/1.0?omitHeader=true
|
||||
----
|
||||
[source, json]
|
||||
----
|
||||
{
|
||||
"files":{"/mypkg/1.0":[{
|
||||
"name":"myplugins.jar",
|
||||
"timestamp":"2019-11-11T07:36:17.354Z",
|
||||
"sha512":"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
|
||||
"sig":["elNjhmWIOgTgbAzeZ+OcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ=="]}]}}
|
||||
----
|
||||
|
||||
|
||||
== Packages
|
||||
A Package has the following attributes:
|
||||
|
||||
* A unique name
|
||||
* One or more versions with the following attributes:
|
||||
** `version` : The version string
|
||||
** `files` : An array of files from the package store
|
||||
|
||||
For every package/version in the packages definition, there is a unique `SolrResourceLoader` instance. This is a child of the `CoreContainer` resource loader.
|
||||
|
||||
=== API end points
|
||||
|
||||
* `GET /api/cluster/package` Get the list of packages
|
||||
* `POST /api/cluster/package` edit packages
|
||||
** `add` command: add a version of a package
|
||||
** `delete` command : delete a version of a package
|
||||
|
||||
==== Using them in Plugins
|
||||
Any class name can be prefixed with the packagename e.g : `mypkg:fully.qualified.ClassName` and Solr would use the latest version of the package to load the classes from.
|
||||
|
||||
example:
|
||||
|
||||
1) create a package
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d '
|
||||
{"add": {
|
||||
"package" : "mypkg",
|
||||
"version":"1.0",
|
||||
"files" :["/mypkg/1.0/myplugins.jar"]}}'
|
||||
----
|
||||
2) Verify the created package
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
curl http://localhost:8983/api/cluster/package?omitHeader=true
|
||||
----
|
||||
|
||||
[source,json]
|
||||
----
|
||||
{"result":{
|
||||
"znodeVersion":0,
|
||||
"packages":{"mypkg":[{
|
||||
"version":"1.0",
|
||||
"files":["/mypkg/1.0/myplugins.jar"]}]}}}
|
||||
----
|
||||
|
||||
3) The Package is ready to use now. Now, register a plugin in your collection from the package. Note the `"mypkg"` prefix applied to the `class` attribute. The same result can be achieved by editing your `solrconfig.xml` as well
|
||||
[source,bash]
|
||||
----
|
||||
curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
|
||||
"create-requesthandler" : { "name" : "/test",
|
||||
"class": "mypkg:org.apache.solr.core.RuntimeLibReqHandler" }}'
|
||||
----
|
||||
|
||||
4) Verify that the component is created and it is using the correct version of the package to load classes from
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
curl http://localhost:8983/solr/gettingstarted/config/requestHandler?componentName=/test&meta=true&omitHeader=true
|
||||
----
|
||||
[source,json]
|
||||
----
|
||||
{
|
||||
"config":{"requestHandler":{"/test":{
|
||||
"name":"/test",
|
||||
"class":"mypkg:org.apache.solr.core.RuntimeLibReqHandler",
|
||||
"_packageinfo_":{
|
||||
"package":"mypkg",
|
||||
"version":"1.0",
|
||||
"files":["/mypkg/1.0/myplugins.jar"]}}}}}
|
||||
----
|
||||
|
||||
5) Test the request handler
|
||||
[source,bash]
|
||||
----
|
||||
$ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true
|
||||
----
|
||||
[source,json]
|
||||
----
|
||||
{
|
||||
"params":{
|
||||
"omitHeader":"true"},
|
||||
"context":{
|
||||
"webapp":"/solr",
|
||||
"path":"/test",
|
||||
"httpMethod":"GET"},
|
||||
"class":"org.apache.solr.core.RuntimeLibReqHandler",
|
||||
"loader":"java.net.FactoryURLClassLoader"}
|
||||
----
|
||||
|
||||
6) Update the version of our component
|
||||
|
||||
Get a new version of the jar, sign and upload it
|
||||
|
||||
[source, bash]
|
||||
----
|
||||
$ curl -o runtimelibs3.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true
|
||||
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
|
||||
$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ==
|
||||
----
|
||||
|
||||
7) Verify it
|
||||
[source, bash]
|
||||
----
|
||||
$ curl http://localhost:8983/api/node/files/mypkg/2.0?omitHeader=true
|
||||
----
|
||||
|
||||
[source, json]
|
||||
----
|
||||
{
|
||||
"files":{"/mypkg/2.0":[{
|
||||
"name":"myplugins.jar",
|
||||
"timestamp":"2019-11-11T11:46:14.771Z",
|
||||
"sha512":"60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922",
|
||||
"sig":["ICkC+nGE+AqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ=="]}]}}
|
||||
----
|
||||
|
||||
8) Add a new version of the package
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
$ curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d '
|
||||
{"add": {
|
||||
"package" : "mypkg",
|
||||
"version":"2.0",
|
||||
"files" :["/mypkg/2.0/myplugins.jar"]}}'
|
||||
----
|
||||
|
||||
9) Verify the plugin to see if the correct version of the package is being used
|
||||
[source,bash]
|
||||
----
|
||||
$ curl http://localhost:8983/solr/gettingstarted/config/requestHandler?componentName=/test&meta=true&omitHeader=true
|
||||
----
|
||||
|
||||
[source,json]
|
||||
----
|
||||
{
|
||||
"config": {
|
||||
"requestHandler": {
|
||||
"/test": {
|
||||
"name": "/test",
|
||||
"class": "mypkg:org.apache.solr.core.RuntimeLibReqHandler",
|
||||
"_packageinfo_": {
|
||||
"package": "mypkg",
|
||||
"version": "2.0",
|
||||
"files": [
|
||||
"/mypkg/2.0/myplugins.jar"
|
||||
]
|
||||
}}}}}
|
||||
----
|
||||
|
||||
10) Test the plugin
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
$ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true
|
||||
----
|
||||
[source,json]
|
||||
----
|
||||
{
|
||||
"params": {
|
||||
"omitHeader": "true"
|
||||
},
|
||||
"context": {
|
||||
"webapp": "/solr",
|
||||
"path": "/test",
|
||||
"httpMethod": "GET"
|
||||
},
|
||||
"class": "org.apache.solr.core.RuntimeLibReqHandler",
|
||||
"loader": "java.net.FactoryURLClassLoader",
|
||||
"Version": "2"
|
||||
}
|
||||
----
|
||||
Note that the `Version` value is `"2"` . So the plugin is updated
|
||||
|
||||
==== How to avoid automatic upgrade?
|
||||
|
||||
The default version used in any collection is always the latest. However, setting a per-collection property in the `params.json` ensures that the versions are always fixed irrespective of the new versions added
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
$ curl http://localhost:8983/solr/gettingstarted/config/params -H 'Content-type:application/json' -d '{
|
||||
"set":{
|
||||
"PKG_VERSIONS":{
|
||||
"mypkg":"2.0"
|
||||
}
|
||||
}}'
|
||||
----
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
= The Well-Configured Solr Instance
|
||||
:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api
|
||||
:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api, solr-packages
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
|
|
@ -31,6 +31,8 @@ This section covers the following topics:
|
|||
|
||||
<<solr-plugins.adoc#solr-plugins,Solr Plugins>>: Introduces Solr plugins with pointers to more information.
|
||||
|
||||
<<solr-packages.adoc#solr-packages, Packages in Solr>>: Deploying, installing, updating packages with plugins into Solr Cluster
|
||||
|
||||
<<jvm-settings.adoc#jvm-settings,JVM Settings>>: Gives some guidance on best practices for working with Java Virtual Machines.
|
||||
|
||||
<<v2-api.adoc#v2-api,V2 API>>: Describes how to use the new V2 APIs, a redesigned API framework covering most Solr APIs.
|
||||
|
|
|
|||
Loading…
Reference in New Issue