From 40661489cd590947f513e553a20707d0c82b82e5 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 12 Nov 2019 16:45:04 +1100 Subject: [PATCH] SOLR-13822: Ref guide and error handling SOLR-13822: Ref guide and error handling for package loader, package store --- .../solr/filestore/PackageStoreAPI.java | 13 +- .../org/apache/solr/servlet/HttpSolrCall.java | 2 +- .../solr/servlet/SolrRequestParsers.java | 13 +- solr/solr-ref-guide/src/solr-packages.adoc | 296 ++++++++++++++++++ .../the-well-configured-solr-instance.adoc | 4 +- 5 files changed, 318 insertions(+), 10 deletions(-) create mode 100644 solr/solr-ref-guide/src/solr-packages.adoc diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java index 7e80b9a4475..f1750eafe8c 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java @@ -156,6 +156,10 @@ public class PackageStoreAPI { if (signatures != null) { vals.put("sig", signatures); } + PackageStore.FileType type = packageStore.getType(path, true); + if(type != PackageStore.FileType.NOFILE) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path); + } packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), path)); rsp.add(CommonParams.FILE, path); } catch (IOException e) { @@ -177,7 +181,7 @@ public class PackageStoreAPI { } private List readSignatures(SolrQueryRequest req, ByteBuffer buf) - throws SolrException { + throws SolrException, IOException { String[] signatures = req.getParams().getParams("sig"); if (signatures == null || signatures.length == 0) return null; List sigs = Arrays.asList(signatures); @@ -186,7 +190,7 @@ public class PackageStoreAPI { } public void validate(List sigs, - ByteBuffer buf) throws SolrException { + ByteBuffer buf) throws SolrException, IOException { Map keys = CloudUtil.getTrustedKeys( coreContainer.getZkController().getZkClient(), "exe"); if (keys == null || keys.isEmpty()) { @@ -202,7 +206,8 @@ public class PackageStoreAPI { } for (String sig : sigs) { if (cryptoKeys.verify(sig, buf) == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Signature does not match any public key : " + sig +" len: "+buf.limit()+ " content sha512: "+ + DigestUtils.sha512Hex(new ByteBufferInputStream(buf))); } } @@ -337,7 +342,7 @@ public class PackageStoreAPI { "Error parsing public keys in ZooKeeper"); } for (String sig : sigs) { - Supplier errMsg = () -> "Signature does not match any public key : " + sig; + Supplier errMsg = () -> "Signature does not match any public key : " + sig + "sha256 "+ entry.getMetaData().sha512; if (entry.getBuffer() != null) { if (cryptoKeys.verify(sig, entry.getBuffer()) == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index e7002f3750e..72f6fe32c98 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -186,6 +186,7 @@ public class HttpSolrCall { this.response = response; this.retry = retry; this.requestType = RequestType.UNKNOWN; + req.setAttribute(HttpSolrCall.class.getName(), this); queryParams = SolrRequestParsers.parseQueryString(req.getQueryString()); // set a request timer which can be reused by requests if needed req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree()); @@ -196,7 +197,6 @@ public class HttpSolrCall { // this lets you handle /update/commit when /update is a servlet path += req.getPathInfo(); } - req.setAttribute(HttpSolrCall.class.getName(), this); } public String getPath() { diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index 7c21ad159f8..c8e886acf18 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -63,7 +63,6 @@ import org.apache.solr.util.RTimerTree; import org.apache.solr.util.SolrFileCleaningTracker; import org.apache.solr.util.tracing.GlobalTracer; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE; import static org.apache.solr.common.params.CommonParams.PATH; @@ -270,7 +269,10 @@ public class SolrRequestParsers } return q; } - + + private static HttpSolrCall getHttpSolrCall(HttpServletRequest req) { + return req == null ? null : (HttpSolrCall) req.getAttribute(HttpSolrCall.class.getName()); + } /** * Given a url-encoded query string (UTF-8), map it into solr params */ @@ -733,7 +735,7 @@ public class SolrRequestParsers String contentType = req.getContentType(); String method = req.getMethod(); // No need to uppercase... HTTP verbs are case sensitive String uri = req.getRequestURI(); - boolean isRawPut = "PUT".equals(method) && BINARY_CONTENT_TYPE.equals(contentType); + boolean isV2 = getHttpSolrCall(req) instanceof V2HttpCall; boolean isPost = "POST".equals(method); // SOLR-6787 changed the behavior of a POST without content type. Previously it would throw an exception, @@ -749,7 +751,10 @@ public class SolrRequestParsers // POST was handled normally, but other methods (PUT/DELETE) // were handled by restlet if the URI contained /schema or /config // "handled by restlet" means that we don't attempt to handle any request body here. - if (!isPost && !isRawPut) { + if (!isPost) { + if (isV2) { + return raw.parseParamsAndFillStreams(req, streams); + } if (contentType == null) { return parseQueryString(req.getQueryString()); } diff --git a/solr/solr-ref-guide/src/solr-packages.adoc b/solr/solr-ref-guide/src/solr-packages.adoc new file mode 100644 index 00000000000..a4144cb93da --- /dev/null +++ b/solr/solr-ref-guide/src/solr-packages.adoc @@ -0,0 +1,296 @@ += Packages in Solr +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +A Package in Solr may contain one or more .jar files and any no:of plugins. + + +== A Plugin + +A Plugin is an implementation of any of the standard Solr pluggable component. A Package has two components + + * The jar file(s) + * The configuration of the Package in Zookeeper in a file called `/packages.json` + +=== Package loading security +Packages are disabled by default. Start all your nodes with the system property `-Denable.package=true` to use this feature. + +*Example* +[source,bash] +---- + +$ bin/solr -e cloud -noprompt -a "-Denable.packages=true" +---- + +==== Upload your keys +Package binaries must be signed with your private keys and ensure your public keys are published in Zookeeper + +*Example* +[source,bash] +---- +$ openssl genrsa -out my_key.pem 512 +# create the public key in .der format +$ openssl rsa -in my_key.pem -pubout -outform DER -out my_key.der +# upload to Zookeeper +$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd makepath /keys/exe/ +$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd putfile /keys/exe/my_key.der my_key.der +---- + +== Package Store +Package store is a distributed file store which can store arbitrary files in the file system. It is a peer to peer hierarchical storage system. If a new node comes up and it requires any of these files it can be downloaded from one of the nodes where it is available. + +=== Package Store APIs + +The end points are + +* `PUT /api/cluster/files/{full/path/to/file}` in each node +* `GET /api/node/files/{full/path/to/file}` to download the file +* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the file +* `GET /api/node/files/{full/path/to/}` get the list of files in `/full/path/to` + + + +Use the following steps to upload a jar signed with your public key + +1) If you don't have a jar file with plugins, download a sample from github + +[source, bash] +---- +$ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true + +---- +2) Sign the jar with your private key +[source, bash] +---- +$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 +---- + +3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) . Ensure that new lines and spaces are removed. Do not forget to do URL encoding of you signature( e.g escape `+` with `%2B`) +[source, bash] +---- +$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ== +---- + +4) Verify your jar upload +[source, bash] +---- +$ curl http://localhost:7574/api/node/files/mypkg/1.0?omitHeader=true +---- +[source, json] +---- +{ + "files":{"/mypkg/1.0":[{ + "name":"myplugins.jar", + "timestamp":"2019-11-11T07:36:17.354Z", + "sha512":"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", + "sig":["elNjhmWIOgTgbAzeZ+OcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ=="]}]}} +---- + + +== Packages +A Package has the following attributes: + +* A unique name +* One or more versions with the following attributes: +** `version` : The version string +** `files` : An array of files from the package store + +For every package/version in the packages definition, there is a unique `SolrResourceLoader` instance. This is a child of the `CoreContainer` resource loader. + +=== API end points + +* `GET /api/cluster/package` Get the list of packages +* `POST /api/cluster/package` edit packages +** `add` command: add a version of a package +** `delete` command : delete a version of a package + +==== Using them in Plugins +Any class name can be prefixed with the packagename e.g : `mypkg:fully.qualified.ClassName` and Solr would use the latest version of the package to load the classes from. + +example: + +1) create a package + +[source,bash] +---- +curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' +{"add": { + "package" : "mypkg", + "version":"1.0", + "files" :["/mypkg/1.0/myplugins.jar"]}}' +---- +2) Verify the created package + +[source,bash] +---- +curl http://localhost:8983/api/cluster/package?omitHeader=true +---- + +[source,json] +---- + {"result":{ + "znodeVersion":0, + "packages":{"mypkg":[{ + "version":"1.0", + "files":["/mypkg/1.0/myplugins.jar"]}]}}} +---- + +3) The Package is ready to use now. Now, register a plugin in your collection from the package. Note the `"mypkg"` prefix applied to the `class` attribute. The same result can be achieved by editing your `solrconfig.xml` as well +[source,bash] +---- +curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{ + "create-requesthandler" : { "name" : "/test", + "class": "mypkg:org.apache.solr.core.RuntimeLibReqHandler" }}' +---- + +4) Verify that the component is created and it is using the correct version of the package to load classes from + +[source,bash] +---- +curl http://localhost:8983/solr/gettingstarted/config/requestHandler?componentName=/test&meta=true&omitHeader=true +---- +[source,json] +---- +{ + "config":{"requestHandler":{"/test":{ + "name":"/test", + "class":"mypkg:org.apache.solr.core.RuntimeLibReqHandler", + "_packageinfo_":{ + "package":"mypkg", + "version":"1.0", + "files":["/mypkg/1.0/myplugins.jar"]}}}}} +---- + +5) Test the request handler +[source,bash] +---- +$ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true +---- +[source,json] +---- +{ + "params":{ + "omitHeader":"true"}, + "context":{ + "webapp":"/solr", + "path":"/test", + "httpMethod":"GET"}, + "class":"org.apache.solr.core.RuntimeLibReqHandler", + "loader":"java.net.FactoryURLClassLoader"} +---- + +6) Update the version of our component + +Get a new version of the jar, sign and upload it + +[source, bash] +---- +$ curl -o runtimelibs3.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true +$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 +$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ== +---- + +7) Verify it +[source, bash] +---- +$ curl http://localhost:8983/api/node/files/mypkg/2.0?omitHeader=true +---- + +[source, json] +---- +{ + "files":{"/mypkg/2.0":[{ + "name":"myplugins.jar", + "timestamp":"2019-11-11T11:46:14.771Z", + "sha512":"60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922", + "sig":["ICkC+nGE+AqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ=="]}]}} +---- + +8) Add a new version of the package + +[source,bash] +---- +$ curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' +{"add": { + "package" : "mypkg", + "version":"2.0", + "files" :["/mypkg/2.0/myplugins.jar"]}}' +---- + +9) Verify the plugin to see if the correct version of the package is being used +[source,bash] +---- +$ curl http://localhost:8983/solr/gettingstarted/config/requestHandler?componentName=/test&meta=true&omitHeader=true +---- + +[source,json] +---- +{ + "config": { + "requestHandler": { + "/test": { + "name": "/test", + "class": "mypkg:org.apache.solr.core.RuntimeLibReqHandler", + "_packageinfo_": { + "package": "mypkg", + "version": "2.0", + "files": [ + "/mypkg/2.0/myplugins.jar" + ] + }}}}} +---- + +10) Test the plugin + +[source,bash] +---- +$ curl http://localhost:8983/solr/gettingstarted/test?omitHeader=true +---- +[source,json] +---- +{ + "params": { + "omitHeader": "true" + }, + "context": { + "webapp": "/solr", + "path": "/test", + "httpMethod": "GET" + }, + "class": "org.apache.solr.core.RuntimeLibReqHandler", + "loader": "java.net.FactoryURLClassLoader", + "Version": "2" +} +---- +Note that the `Version` value is `"2"` . So the plugin is updated + +==== How to avoid automatic upgrade? + +The default version used in any collection is always the latest. However, setting a per-collection property in the `params.json` ensures that the versions are always fixed irrespective of the new versions added + +[source,bash] +---- +$ curl http://localhost:8983/solr/gettingstarted/config/params -H 'Content-type:application/json' -d '{ + "set":{ + "PKG_VERSIONS":{ + "mypkg":"2.0" + } + }}' +---- + + + diff --git a/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc b/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc index 22e9be31e6c..14b28c64480 100644 --- a/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc +++ b/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc @@ -1,5 +1,5 @@ = The Well-Configured Solr Instance -:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api +:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api, solr-packages // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -31,6 +31,8 @@ This section covers the following topics: <>: Introduces Solr plugins with pointers to more information. +<>: Deploying, installing, updating packages with plugins into Solr Cluster + <>: Gives some guidance on best practices for working with Java Virtual Machines. <>: Describes how to use the new V2 APIs, a redesigned API framework covering most Solr APIs.