SOLR-13822: Ref guide and error handling

SOLR-13822: Ref guide and error handling for package loader, package store
This commit is contained in:
Noble Paul 2019-11-12 16:45:04 +11:00 committed by GitHub
parent abb7087f6e
commit 40661489cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 318 additions and 10 deletions

View File

@ -156,6 +156,10 @@ public class PackageStoreAPI {
if (signatures != null) {
vals.put("sig", signatures);
}
PackageStore.FileType type = packageStore.getType(path, true);
if(type != PackageStore.FileType.NOFILE) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path);
}
packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), path));
rsp.add(CommonParams.FILE, path);
} catch (IOException e) {
@ -177,7 +181,7 @@ public class PackageStoreAPI {
}
private List<String> readSignatures(SolrQueryRequest req, ByteBuffer buf)
throws SolrException {
throws SolrException, IOException {
String[] signatures = req.getParams().getParams("sig");
if (signatures == null || signatures.length == 0) return null;
List<String> sigs = Arrays.asList(signatures);
@ -186,7 +190,7 @@ public class PackageStoreAPI {
}
public void validate(List<String> sigs,
ByteBuffer buf) throws SolrException {
ByteBuffer buf) throws SolrException, IOException {
Map<String, byte[]> keys = CloudUtil.getTrustedKeys(
coreContainer.getZkController().getZkClient(), "exe");
if (keys == null || keys.isEmpty()) {
@ -202,7 +206,8 @@ public class PackageStoreAPI {
}
for (String sig : sigs) {
if (cryptoKeys.verify(sig, buf) == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.limit()+ " content sha512: "+
DigestUtils.sha512Hex(new ByteBufferInputStream(buf)));
}
}
@ -337,7 +342,7 @@ public class PackageStoreAPI {
"Error parsing public keys in ZooKeeper");
}
for (String sig : sigs) {
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig;
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig + "sha256 "+ entry.getMetaData().sha512;
if (entry.getBuffer() != null) {
if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());

View File

@ -186,6 +186,7 @@ public class HttpSolrCall {
this.response = response;
this.retry = retry;
this.requestType = RequestType.UNKNOWN;
req.setAttribute(HttpSolrCall.class.getName(), this);
queryParams = SolrRequestParsers.parseQueryString(req.getQueryString());
// set a request timer which can be reused by requests if needed
req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
@ -196,7 +197,6 @@ public class HttpSolrCall {
// this lets you handle /update/commit when /update is a servlet
path += req.getPathInfo();
}
req.setAttribute(HttpSolrCall.class.getName(), this);
}
public String getPath() {

View File

@ -63,7 +63,6 @@ import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.SolrFileCleaningTracker;
import org.apache.solr.util.tracing.GlobalTracer;
import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE;
import static org.apache.solr.common.params.CommonParams.PATH;
@ -270,7 +269,10 @@ public class SolrRequestParsers
}
return q;
}
private static HttpSolrCall getHttpSolrCall(HttpServletRequest req) {
return req == null ? null : (HttpSolrCall) req.getAttribute(HttpSolrCall.class.getName());
}
/**
* Given a url-encoded query string (UTF-8), map it into solr params
*/
@ -733,7 +735,7 @@ public class SolrRequestParsers
String contentType = req.getContentType();
String method = req.getMethod(); // No need to uppercase... HTTP verbs are case sensitive
String uri = req.getRequestURI();
boolean isRawPut = "PUT".equals(method) && BINARY_CONTENT_TYPE.equals(contentType);
boolean isV2 = getHttpSolrCall(req) instanceof V2HttpCall;
boolean isPost = "POST".equals(method);
// SOLR-6787 changed the behavior of a POST without content type. Previously it would throw an exception,
@ -749,7 +751,10 @@ public class SolrRequestParsers
// POST was handled normally, but other methods (PUT/DELETE)
// were handled by restlet if the URI contained /schema or /config
// "handled by restlet" means that we don't attempt to handle any request body here.
if (!isPost && !isRawPut) {
if (!isPost) {
if (isV2) {
return raw.parseParamsAndFillStreams(req, streams);
}
if (contentType == null) {
return parseQueryString(req.getQueryString());
}

View File

@ -0,0 +1,296 @@
= Packages in Solr
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
A Package in Solr may contain one or more .jar files and any no:of plugins.
== A Plugin
A Plugin is an implementation of any of the standard Solr pluggable component. A Package has two components
* The jar file(s)
* The configuration of the Package in Zookeeper in a file called `/packages.json`
=== Package loading security
Packages are disabled by default. Start all your nodes with the system property `-Denable.package=true` to use this feature.
*Example*
[source,bash]
----
$ bin/solr -e cloud -noprompt -a "-Denable.packages=true"
----
==== Upload your keys
Package binaries must be signed with your private keys and ensure your public keys are published in Zookeeper
*Example*
[source,bash]
----
$ openssl genrsa -out my_key.pem 512
# create the public key in .der format
$ openssl rsa -in my_key.pem -pubout -outform DER -out my_key.der
# upload to Zookeeper
$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd makepath /keys/exe/
$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd putfile /keys/exe/my_key.der my_key.der
----
== Package Store
Package store is a distributed file store which can store arbitrary files in the file system. It is a peer to peer hierarchical storage system. If a new node comes up and it requires any of these files it can be downloaded from one of the nodes where it is available.
=== Package Store APIs
The end points are
* `PUT /api/cluster/files/{full/path/to/file}` in each node
* `GET /api/node/files/{full/path/to/file}` to download the file
* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the file
* `GET /api/node/files/{full/path/to/}` get the list of files in `/full/path/to`
Use the following steps to upload a jar signed with your public key
1) If you don't have a jar file with plugins, download a sample from github
[source, bash]
----
$ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true
----
2) Sign the jar with your private key
[source, bash]
----
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
----
3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) . Ensure that new lines and spaces are removed. Do not forget to do URL encoding of you signature( e.g escape `+` with `%2B`)
[source, bash]
----
$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ==
----
4) Verify your jar upload
[source, bash]
----
$ curl http://localhost:7574/api/node/files/mypkg/1.0?omitHeader=true
----
[source, json]
----
{
"files":{"/mypkg/1.0":[{
"name":"myplugins.jar",
"timestamp":"2019-11-11T07:36:17.354Z",
"sha512":"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
"sig":["elNjhmWIOgTgbAzeZ+OcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ=="]}]}}
----
== Packages
A Package has the following attributes:
* A unique name
* One or more versions with the following attributes:
** `version` : The version string
** `files` : An array of files from the package store
For every package/version in the packages definition, there is a unique `SolrResourceLoader` instance. This is a child of the `CoreContainer` resource loader.
=== API end points
* `GET /api/cluster/package` Get the list of packages
* `POST /api/cluster/package` edit packages
** `add` command: add a version of a package
** `delete` command : delete a version of a package
==== Using them in Plugins
Any class name can be prefixed with the packagename e.g : `mypkg:fully.qualified.ClassName` and Solr would use the latest version of the package to load the classes from.
example:
1) create a package
[source,bash]
----
curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d '
{"add": {
"package" : "mypkg",
"version":"1.0",
"files" :["/mypkg/1.0/myplugins.jar"]}}'
----
2) Verify the created package
[source,bash]
----
curl http://localhost:8983/api/cluster/package?omitHeader=true
----
[source,json]
----
{"result":{
"znodeVersion":0,
"packages":{"mypkg":[{
"version":"1.0",
"files":["/mypkg/1.0/myplugins.jar"]}]}}}
----
3) The Package is ready to use now. Now, register a plugin in your collection from the package. Note the `"mypkg"` prefix applied to the `class` attribute. The same result can be achieved by editing your `solrconfig.xml` as well
[source,bash]
----
curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
"create-requesthandler" : { "name" : "/test",
"class": "mypkg:org.apache.solr.core.RuntimeLibReqHandler" }}'
----
4) Verify that the component is created and it is using the correct version of the package to load classes from
[source,bash]
----
curl http://localhost:8983/solr/gettingstarted/config/requestHandler?componentName=/test&meta=true&omitHeader=true
----
[source,json]
----
{
"config":{"requestHandler":{"/test":{
"name":"/test",
"class":"mypkg:org.apache.solr.core.RuntimeLibReqHandler",
"_packageinfo_":{
"package":"mypkg",
"version":"1.0",
"files":["/mypkg/1.0/myplugins.jar"]}}}}}
----
5) Test the request handler
[source,bash]
----
$ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true
----
[source,json]
----
{
"params":{
"omitHeader":"true"},
"context":{
"webapp":"/solr",
"path":"/test",
"httpMethod":"GET"},
"class":"org.apache.solr.core.RuntimeLibReqHandler",
"loader":"java.net.FactoryURLClassLoader"}
----
6) Update the version of our component
Get a new version of the jar, sign and upload it
[source, bash]
----
$ curl -o runtimelibs3.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ==
----
7) Verify it
[source, bash]
----
$ curl http://localhost:8983/api/node/files/mypkg/2.0?omitHeader=true
----
[source, json]
----
{
"files":{"/mypkg/2.0":[{
"name":"myplugins.jar",
"timestamp":"2019-11-11T11:46:14.771Z",
"sha512":"60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922",
"sig":["ICkC+nGE+AqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ=="]}]}}
----
8) Add a new version of the package
[source,bash]
----
$ curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d '
{"add": {
"package" : "mypkg",
"version":"2.0",
"files" :["/mypkg/2.0/myplugins.jar"]}}'
----
9) Verify the plugin to see if the correct version of the package is being used
[source,bash]
----
$ curl http://localhost:8983/solr/gettingstarted/config/requestHandler?componentName=/test&meta=true&omitHeader=true
----
[source,json]
----
{
"config": {
"requestHandler": {
"/test": {
"name": "/test",
"class": "mypkg:org.apache.solr.core.RuntimeLibReqHandler",
"_packageinfo_": {
"package": "mypkg",
"version": "2.0",
"files": [
"/mypkg/2.0/myplugins.jar"
]
}}}}}
----
10) Test the plugin
[source,bash]
----
$ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true
----
[source,json]
----
{
"params": {
"omitHeader": "true"
},
"context": {
"webapp": "/solr",
"path": "/test",
"httpMethod": "GET"
},
"class": "org.apache.solr.core.RuntimeLibReqHandler",
"loader": "java.net.FactoryURLClassLoader",
"Version": "2"
}
----
Note that the `Version` value is `"2"` . So the plugin is updated
==== How to avoid automatic upgrade?
The default version used in any collection is always the latest. However, setting a per-collection property in the `params.json` ensures that the versions are always fixed irrespective of the new versions added
[source,bash]
----
$ curl http://localhost:8983/solr/gettingstarted/config/params -H 'Content-type:application/json' -d '{
"set":{
"PKG_VERSIONS":{
"mypkg":"2.0"
}
}}'
----

View File

@ -1,5 +1,5 @@
= The Well-Configured Solr Instance
:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api
:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api, solr-packages
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@ -31,6 +31,8 @@ This section covers the following topics:
<<solr-plugins.adoc#solr-plugins,Solr Plugins>>: Introduces Solr plugins with pointers to more information.
<<solr-packages.adoc#solr-packages, Packages in Solr>>: Deploying, installing, updating packages with plugins into Solr Cluster
<<jvm-settings.adoc#jvm-settings,JVM Settings>>: Gives some guidance on best practices for working with Java Virtual Machines.
<<v2-api.adoc#v2-api,V2 API>>: Describes how to use the new V2 APIs, a redesigned API framework covering most Solr APIs.