diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index f547d10ded4..b26b987021d 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -47,7 +47,6 @@ import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.pkg.PackagePluginHolder; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.update.processor.UpdateRequestProcessorChain; -import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder; import org.apache.solr.update.processor.UpdateRequestProcessorFactory; import org.apache.solr.util.CryptoKeys; import org.apache.solr.util.SimplePostTool; @@ -143,9 +142,7 @@ public class PluginBag implements AutoCloseable { } else { if (info.pkgName != null) { PackagePluginHolder holder = new PackagePluginHolder<>(info, core, meta); - return meta.clazz == UpdateRequestProcessorFactory.class ? - new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) : - holder; + return holder; } else { T inst = core.createInstance(info.className, (Class) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName)); initInstance(inst, info); diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java index 3e3b5834eaf..8d9af8f7a07 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java @@ -459,8 +459,8 @@ public class DistribPackageStore implements PackageStore { if (fetch(path, null)) { file = getRealpath(path).toFile(); } - if (!file.exists()) return FileType.NOFILE; } + if (!file.exists()) return FileType.NOFILE; if (file.isDirectory()) return FileType.DIRECTORY; return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE; } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index d80e8bd917b..1b93d894428 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -271,8 +271,18 @@ public class PackageAPI { log.error("Error deserializing packages.json", e); packages = new Packages(); } - packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() ; + List list = packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN); + for (Object o : list) { + if (o instanceof PkgVersion) { + PkgVersion version = (PkgVersion) o; + if (Objects.equals(version.version, add.version)) { + payload.addError("Version '" + add.version + "' exists already"); + return null; + } + } + } + list.add(new PkgVersion(add)); + packages.znodeVersion = stat.getVersion() + 1; finalState[0] = packages; return Utils.toJSON(packages); }); diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java index 5ddaf8b1013..a912e2ab0f0 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java @@ -296,7 +296,13 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized for (String s : names) { s = s.trim(); if (s.isEmpty()) continue; - UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s); + UpdateRequestProcessorFactory p = null; + PluginBag.PluginHolder holder = core.getUpdateProcessors().getRegistry().get(s); + if (holder instanceof PackagePluginHolder) { + p = new LazyUpdateRequestProcessorFactory(holder); + } else { + p = core.getUpdateProcessors().get(s); + } if (p == null) { Class factoryClass = implicits.get(s); if(factoryClass != null) { diff --git a/solr/core/src/test-files/runtimecode/TestVersionedURP.java b/solr/core/src/test-files/runtimecode/TestVersionedURP.java new file mode 100644 index 00000000000..ae91b808bd0 --- /dev/null +++ b/solr/core/src/test-files/runtimecode/TestVersionedURP.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtimecode; + +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.processor.SimpleUpdateProcessorFactory; + +public class TestVersionedURP extends SimpleUpdateProcessorFactory { + @Override + protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) { + cmd.solrDoc.addField("TestVersionedURP.Ver_s", "Version 2"); + } +} diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt index 4ef8e9cd3c2..f354b5f8933 100644 --- a/solr/core/src/test-files/runtimecode/sig.txt +++ b/solr/core/src/test-files/runtimecode/sig.txt @@ -57,6 +57,14 @@ openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem cache_v2.jar.bin | openss SOrekHt+uup+z2z+nZU5indk2huRRfmbM+W+vQ0variHrcZEG9EXt5LuPFl8Ki9A hr6klMHdVP8nj4wuQhu/Hg== +openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v1.jar.bin | openssl enc -base64 | tr -d \\n | sed + +h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ== + +openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v2.jar.bin | openssl enc -base64 | openssl enc -base64 | tr -d \\n | sed + +P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw== + ====================sha512==================== openssl dgst -sha512 runtimelibs.jar.bin @@ -80,6 +88,13 @@ openssl dgst -sha512 cache_v2.jar.bin 873337e67b90b7ff99df012b2e9093c63079c37a564643d34861a88c4cbaf0698ebb096905929d69cdbde3b4d29d55e31db24ee05c01b39c0b75a16e54eb4335 +openssl dgst -sha512 testurp_v1.jar.bin + +7b9df184202e62cc6f73e69a7f369a6f469ad9e1508a2d61eeb7744b204f2489a2c617808f28a496fcbca3a318e16dca238c111e60a903ace877a79900bb8729 + +openssl dgst -sha512 testurp_v2.jar.bin + +5c4c0c454a032916e48a1c14a0fecbd6658658a66aedec5168b7222f2e3c0c63fbe09637238a9325ce2e95a2c8521834397a97701ead46c681aa20c9fccb6654 =============sha256============================ openssl dgst -sha256 runtimelibs.jar.bin diff --git a/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin b/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin new file mode 100644 index 00000000000..7c9044c9d2f Binary files /dev/null and b/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin differ diff --git a/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin b/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin new file mode 100644 index 00000000000..0c8041ec583 Binary files /dev/null and b/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin differ diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 560cdea72d6..baa8a99b02d 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -26,6 +26,7 @@ import java.util.concurrent.Callable; import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; @@ -34,14 +35,17 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.request.beans.Package; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.ConfigRequest; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.MapWriterMap; import org.apache.solr.common.NavigableObject; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; @@ -86,6 +90,8 @@ public class TestPackages extends SolrCloudTestCase { String FILE1 = "/mypkg/runtimelibs.jar"; String FILE2 = "/mypkg/runtimelibs_v2.jar"; String FILE3 = "/mypkg/runtimelibs_v3.jar"; + String URP1 = "/mypkg/testurpv1.jar"; + String URP2 = "/mypkg/testurpv2.jar"; String COLLECTION_NAME = "testPluginLoadingColl"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); cluster.getZkClient().makePath("/keys/exe", true); @@ -93,10 +99,13 @@ public class TestPackages extends SolrCloudTestCase { postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + postFileAndWait(cluster, "runtimecode/testurp_v1.jar.bin", URP1, + "h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ=="); + Package.AddVersion add = new Package.AddVersion(); add.version = "1.0"; add.pkg = "mypkg"; - add.files = Arrays.asList(new String[]{FILE1}); + add.files = Arrays.asList(new String[]{FILE1, URP1}); V2Request req = new V2Request.Builder("/cluster/package") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) @@ -125,6 +134,7 @@ public class TestPackages extends SolrCloudTestCase { "'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," + "'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," + "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" + + "'create-updateProcessor' : { 'name' : 'myurp', 'class': 'mypkg:org.apache.solr.update.TestVersionedURP' }" + "}"; cluster.getSolrClient().request(new ConfigRequest(payload) { @Override @@ -145,7 +155,20 @@ public class TestPackages extends SolrCloudTestCase { COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "1.0" ); + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "updateProcessor", "myurp", + "mypkg", "1.0" ); + UpdateRequest ur = new UpdateRequest(); + ur.add(new SolrInputDocument("id", "1")); + ur.setParam("processor", "myurp"); + ur.process(cluster.getSolrClient(), COLLECTION_NAME); + cluster.getSolrClient().commit(COLLECTION_NAME, true, true); + + QueryResponse result = cluster.getSolrClient() + .query(COLLECTION_NAME, new SolrQuery( "id:1")); + + assertEquals("Version 1", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()), Utils.JAVABINCONSUMER, @@ -165,9 +188,11 @@ public class TestPackages extends SolrCloudTestCase { postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2, "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); + postFileAndWait(cluster, "runtimecode/testurp_v2.jar.bin", URP2, + "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); //add the version using package API add.version = "1.1"; - add.files = Arrays.asList(new String[]{FILE2}); + add.files = Arrays.asList(new String[]{FILE2,URP2}); req.process(cluster.getSolrClient()); verifyCmponent(cluster.getSolrClient(), @@ -182,6 +207,11 @@ public class TestPackages extends SolrCloudTestCase { COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "1.1" ); + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "updateProcessor", "myurp", + "mypkg", "1.1" ); + + executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()), Utils.JSONCONSUMER, Utils.makeMap( "Version","2")); @@ -192,7 +222,7 @@ public class TestPackages extends SolrCloudTestCase { "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); add.version = "2.1"; - add.files = Arrays.asList(new String[]{FILE3}); + add.files = Arrays.asList(new String[]{FILE3, URP2}); req.process(cluster.getSolrClient()); //now let's verify that the classes are updated @@ -212,6 +242,18 @@ public class TestPackages extends SolrCloudTestCase { Utils.JSONCONSUMER, Utils.makeMap("Version","2")); + //insert a doc with urp + ur = new UpdateRequest(); + ur.add(new SolrInputDocument("id", "2")); + ur.setParam("processor", "myurp"); + ur.process(cluster.getSolrClient(), COLLECTION_NAME); + cluster.getSolrClient().commit(COLLECTION_NAME, true, true); + + result = cluster.getSolrClient() + .query(COLLECTION_NAME, new SolrQuery( "id:2")); + + assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); + Package.DelVersion delVersion = new Package.DelVersion(); delVersion.pkg = "mypkg"; @@ -262,7 +304,7 @@ public class TestPackages extends SolrCloudTestCase { }.process(cluster.getSolrClient()) ; add.version = "2.1"; - add.files = Arrays.asList(new String[]{FILE3}); + add.files = Arrays.asList(new String[]{FILE3, URP2}); req.process(cluster.getSolrClient()); //the collections mypkg is set to use version 1.1 diff --git a/solr/solr-ref-guide/src/package-manager-internals.adoc b/solr/solr-ref-guide/src/package-manager-internals.adoc index 90ed865571e..6cb1956575f 100644 --- a/solr/solr-ref-guide/src/package-manager-internals.adoc +++ b/solr/solr-ref-guide/src/package-manager-internals.adoc @@ -1,7 +1,7 @@ = Package Manager Internals The package manager (CLI) internally uses various Solr APIs to install, deploy and update packages. This document contains an overview of those APIs. -== Design Objectives +== Salient Features * Zero disruption deployment (hot deployment): Should be possible to install and update packages without node restarts or core reloads, and hence deployments should be quick and without failed requests or stale caches. * Easy packaging: @@ -81,13 +81,13 @@ $ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/maste 2) Sign the jar with your private key [source, bash] ---- -$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 +$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed ---- -3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) . Ensure that new lines and spaces are removed. Do not forget to do URL encoding of you signature( e.g escape `+` with `%2B`) +3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) [source, bash] ---- -$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ== +$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig= ---- 4) Verify your jar upload @@ -263,8 +263,8 @@ Get a new version of the jar, sign and upload it [source, bash] ---- $ curl -o runtimelibs3.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true -$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 -$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ== +$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed +$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig= ---- 7) Verify it