mirror of https://github.com/apache/lucene.git
SOLR-13822: Bug fixs and tests for URP loading
This commit is contained in:
noble
parent
400514026e
commit
f98555854c
|
|
@ -47,7 +47,6 @@ import org.apache.solr.handler.component.SearchComponent;
|
|||
import org.apache.solr.pkg.PackagePluginHolder;
|
||||
import org.apache.solr.request.SolrRequestHandler;
|
||||
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
|
||||
import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder;
|
||||
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
|
||||
import org.apache.solr.util.CryptoKeys;
|
||||
import org.apache.solr.util.SimplePostTool;
|
||||
|
|
@ -143,9 +142,7 @@ public class PluginBag<T> implements AutoCloseable {
|
|||
} else {
|
||||
if (info.pkgName != null) {
|
||||
PackagePluginHolder<T> holder = new PackagePluginHolder<>(info, core, meta);
|
||||
return meta.clazz == UpdateRequestProcessorFactory.class ?
|
||||
new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) :
|
||||
holder;
|
||||
return holder;
|
||||
} else {
|
||||
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName));
|
||||
initInstance(inst, info);
|
||||
|
|
|
|||
|
|
@ -459,8 +459,8 @@ public class DistribPackageStore implements PackageStore {
|
|||
if (fetch(path, null)) {
|
||||
file = getRealpath(path).toFile();
|
||||
}
|
||||
if (!file.exists()) return FileType.NOFILE;
|
||||
}
|
||||
if (!file.exists()) return FileType.NOFILE;
|
||||
if (file.isDirectory()) return FileType.DIRECTORY;
|
||||
return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -269,8 +269,18 @@ public class PackageAPI {
|
|||
log.error("Error deserializing packages.json", e);
|
||||
packages = new Packages();
|
||||
}
|
||||
packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
|
||||
packages.znodeVersion = stat.getVersion() ;
|
||||
List list = packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN);
|
||||
for (Object o : list) {
|
||||
if (o instanceof PkgVersion) {
|
||||
PkgVersion version = (PkgVersion) o;
|
||||
if (Objects.equals(version.version, add.version)) {
|
||||
payload.addError("Version '" + add.version + "' exists already");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
list.add(new PkgVersion(add));
|
||||
packages.znodeVersion = stat.getVersion() + 1;
|
||||
finalState[0] = packages;
|
||||
return Utils.toJSON(packages);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -296,7 +296,13 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
|
|||
for (String s : names) {
|
||||
s = s.trim();
|
||||
if (s.isEmpty()) continue;
|
||||
UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s);
|
||||
UpdateRequestProcessorFactory p = null;
|
||||
PluginBag.PluginHolder<UpdateRequestProcessorFactory> holder = core.getUpdateProcessors().getRegistry().get(s);
|
||||
if (holder instanceof PackagePluginHolder) {
|
||||
p = new LazyUpdateRequestProcessorFactory(holder);
|
||||
} else {
|
||||
p = core.getUpdateProcessors().get(s);
|
||||
}
|
||||
if (p == null) {
|
||||
Class<UpdateRequestProcessorFactory> factoryClass = implicits.get(s);
|
||||
if(factoryClass != null) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package runtimecode;
|
||||
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.update.processor.SimpleUpdateProcessorFactory;
|
||||
|
||||
public class TestVersionedURP extends SimpleUpdateProcessorFactory {
|
||||
@Override
|
||||
protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
cmd.solrDoc.addField("TestVersionedURP.Ver_s", "Version 2");
|
||||
}
|
||||
}
|
||||
|
|
@ -57,6 +57,14 @@ openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem cache_v2.jar.bin | openss
|
|||
SOrekHt+uup+z2z+nZU5indk2huRRfmbM+W+vQ0variHrcZEG9EXt5LuPFl8Ki9A
|
||||
hr6klMHdVP8nj4wuQhu/Hg==
|
||||
|
||||
openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v1.jar.bin | openssl enc -base64 | tr -d \\n | sed
|
||||
|
||||
h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ==
|
||||
|
||||
openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v2.jar.bin | openssl enc -base64 | openssl enc -base64 | tr -d \\n | sed
|
||||
|
||||
P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw==
|
||||
|
||||
====================sha512====================
|
||||
|
||||
openssl dgst -sha512 runtimelibs.jar.bin
|
||||
|
|
@ -80,6 +88,13 @@ openssl dgst -sha512 cache_v2.jar.bin
|
|||
|
||||
873337e67b90b7ff99df012b2e9093c63079c37a564643d34861a88c4cbaf0698ebb096905929d69cdbde3b4d29d55e31db24ee05c01b39c0b75a16e54eb4335
|
||||
|
||||
openssl dgst -sha512 testurp_v1.jar.bin
|
||||
|
||||
7b9df184202e62cc6f73e69a7f369a6f469ad9e1508a2d61eeb7744b204f2489a2c617808f28a496fcbca3a318e16dca238c111e60a903ace877a79900bb8729
|
||||
|
||||
openssl dgst -sha512 testurp_v2.jar.bin
|
||||
|
||||
5c4c0c454a032916e48a1c14a0fecbd6658658a66aedec5168b7222f2e3c0c63fbe09637238a9325ce2e95a2c8521834397a97701ead46c681aa20c9fccb6654
|
||||
=============sha256============================
|
||||
|
||||
openssl dgst -sha256 runtimelibs.jar.bin
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -26,6 +26,7 @@ import java.util.concurrent.Callable;
|
|||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
||||
|
|
@ -34,14 +35,17 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
|||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.request.GenericSolrRequest;
|
||||
import org.apache.solr.client.solrj.request.RequestWriter;
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.V2Request;
|
||||
import org.apache.solr.client.solrj.request.beans.Package;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||
import org.apache.solr.cloud.ConfigRequest;
|
||||
import org.apache.solr.cloud.MiniSolrCloudCluster;
|
||||
import org.apache.solr.cloud.SolrCloudTestCase;
|
||||
import org.apache.solr.common.MapWriterMap;
|
||||
import org.apache.solr.common.NavigableObject;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.params.MapSolrParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
|
|
@ -75,6 +79,8 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
String FILE1 = "/mypkg/runtimelibs.jar";
|
||||
String FILE2 = "/mypkg/runtimelibs_v2.jar";
|
||||
String FILE3 = "/mypkg/runtimelibs_v3.jar";
|
||||
String URP1 = "/mypkg/testurpv1.jar";
|
||||
String URP2 = "/mypkg/testurpv2.jar";
|
||||
String COLLECTION_NAME = "testPluginLoadingColl";
|
||||
byte[] derFile = readFile("cryptokeys/pub_key512.der");
|
||||
cluster.getZkClient().makePath("/keys/exe", true);
|
||||
|
|
@ -82,10 +88,13 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
|
||||
"L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
|
||||
|
||||
postFileAndWait(cluster, "runtimecode/testurp_v1.jar.bin", URP1,
|
||||
"h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ==");
|
||||
|
||||
Package.AddVersion add = new Package.AddVersion();
|
||||
add.version = "1.0";
|
||||
add.pkg = "mypkg";
|
||||
add.files = Arrays.asList(new String[]{FILE1});
|
||||
add.files = Arrays.asList(new String[]{FILE1, URP1});
|
||||
V2Request req = new V2Request.Builder("/cluster/package")
|
||||
.forceV2(true)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
|
|
@ -114,6 +123,7 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," +
|
||||
"'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," +
|
||||
"'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" +
|
||||
"'create-updateProcessor' : { 'name' : 'myurp', 'class': 'mypkg:org.apache.solr.update.TestVersionedURP' }" +
|
||||
"}";
|
||||
cluster.getSolrClient().request(new ConfigRequest(payload) {
|
||||
@Override
|
||||
|
|
@ -134,7 +144,20 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
COLLECTION_NAME, "requestHandler", "/runtime",
|
||||
"mypkg", "1.0" );
|
||||
|
||||
verifyCmponent(cluster.getSolrClient(),
|
||||
COLLECTION_NAME, "updateProcessor", "myurp",
|
||||
"mypkg", "1.0" );
|
||||
|
||||
UpdateRequest ur = new UpdateRequest();
|
||||
ur.add(new SolrInputDocument("id", "1"));
|
||||
ur.setParam("processor", "myurp");
|
||||
ur.process(cluster.getSolrClient(), COLLECTION_NAME);
|
||||
cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
|
||||
|
||||
QueryResponse result = cluster.getSolrClient()
|
||||
.query(COLLECTION_NAME, new SolrQuery( "id:1"));
|
||||
|
||||
assertEquals("Version 1", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s"));
|
||||
|
||||
executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
|
||||
Utils.JAVABINCONSUMER,
|
||||
|
|
@ -154,9 +177,11 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
|
||||
"j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
|
||||
|
||||
postFileAndWait(cluster, "runtimecode/testurp_v2.jar.bin", URP2,
|
||||
"P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw==");
|
||||
//add the version using package API
|
||||
add.version = "1.1";
|
||||
add.files = Arrays.asList(new String[]{FILE2});
|
||||
add.files = Arrays.asList(new String[]{FILE2,URP2});
|
||||
req.process(cluster.getSolrClient());
|
||||
|
||||
verifyCmponent(cluster.getSolrClient(),
|
||||
|
|
@ -171,6 +196,11 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
COLLECTION_NAME, "requestHandler", "/runtime",
|
||||
"mypkg", "1.1" );
|
||||
|
||||
verifyCmponent(cluster.getSolrClient(),
|
||||
COLLECTION_NAME, "updateProcessor", "myurp",
|
||||
"mypkg", "1.1" );
|
||||
|
||||
|
||||
executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
|
||||
Utils.JSONCONSUMER,
|
||||
Utils.makeMap( "Version","2"));
|
||||
|
|
@ -181,7 +211,7 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
"a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==");
|
||||
|
||||
add.version = "2.1";
|
||||
add.files = Arrays.asList(new String[]{FILE3});
|
||||
add.files = Arrays.asList(new String[]{FILE3, URP2});
|
||||
req.process(cluster.getSolrClient());
|
||||
|
||||
//now let's verify that the classes are updated
|
||||
|
|
@ -201,6 +231,18 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
Utils.JSONCONSUMER,
|
||||
Utils.makeMap("Version","2"));
|
||||
|
||||
//insert a doc with urp
|
||||
ur = new UpdateRequest();
|
||||
ur.add(new SolrInputDocument("id", "2"));
|
||||
ur.setParam("processor", "myurp");
|
||||
ur.process(cluster.getSolrClient(), COLLECTION_NAME);
|
||||
cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
|
||||
|
||||
result = cluster.getSolrClient()
|
||||
.query(COLLECTION_NAME, new SolrQuery( "id:2"));
|
||||
|
||||
assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s"));
|
||||
|
||||
|
||||
Package.DelVersion delVersion = new Package.DelVersion();
|
||||
delVersion.pkg = "mypkg";
|
||||
|
|
@ -251,7 +293,7 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
}.process(cluster.getSolrClient()) ;
|
||||
|
||||
add.version = "2.1";
|
||||
add.files = Arrays.asList(new String[]{FILE3});
|
||||
add.files = Arrays.asList(new String[]{FILE3, URP2});
|
||||
req.process(cluster.getSolrClient());
|
||||
|
||||
//the collections mypkg is set to use version 1.1
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
= Package Manager Internals
|
||||
The package manager (CLI) internally uses various Solr APIs to install, deploy and update packages. This document contains an overview of those APIs.
|
||||
|
||||
== Design Objectives
|
||||
== Salient Features
|
||||
|
||||
* Zero disruption deployment (hot deployment): Should be possible to install and update packages without node restarts or core reloads, and hence deployments should be quick and without failed requests or stale caches.
|
||||
* Easy packaging:
|
||||
|
|
@ -81,13 +81,13 @@ $ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/maste
|
|||
2) Sign the jar with your private key
|
||||
[source, bash]
|
||||
----
|
||||
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
|
||||
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed
|
||||
----
|
||||
|
||||
3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) . Ensure that new lines and spaces are removed. Do not forget to do URL encoding of you signature( e.g escape `+` with `%2B`)
|
||||
3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command)
|
||||
[source, bash]
|
||||
----
|
||||
$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ==
|
||||
$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
|
||||
----
|
||||
|
||||
4) Verify your jar upload
|
||||
|
|
@ -261,8 +261,8 @@ Get a new version of the jar, sign and upload it
|
|||
[source, bash]
|
||||
----
|
||||
$ curl -o runtimelibs3.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true
|
||||
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
|
||||
$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ==
|
||||
$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed
|
||||
$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=<signature>
|
||||
----
|
||||
|
||||
7) Verify it
|
||||
|
|
|
|||
Loading…
Reference in New Issue