Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14404: support for openResource() in PackageResourceLoader & path-prefix for container plugins

This commit is contained in:
Noble Paul 2020-06-28 14:49:06 +10:00 committed by GitHub
parent fb3c5d2353
commit 1590ed56bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 124 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java
View File

@ -23,15 +23,16 @@ import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.beans.PluginMeta;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.ClusterPropertiesListener;
import org.apache.solr.common.util.Pair;
@ -49,6 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.lucene.util.IOUtils.closeWhileHandlingException;
import static org.apache.solr.common.util.Utils.makeMap;
public class CustomContainerPlugins implements ClusterPropertiesListener {
private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
@ -122,7 +124,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener {
}
if (e.getValue() == Diff.ADDED) {
for (ApiHolder holder : apiInfo.holders) {
containerApiBag.register(holder, Collections.singletonMap("plugin-name", e.getKey()));
containerApiBag.register(holder, getTemplateVars(apiInfo.info));
}
currentPlugins.put(e.getKey(), apiInfo);
} else {
@ -134,7 +136,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener {
if (oldApi instanceof ApiHolder) {
replaced.add((ApiHolder) oldApi);
}
containerApiBag.register(holder, Collections.singletonMap("plugin-name", e.getKey()));
containerApiBag.register(holder, getTemplateVars(apiInfo.info));
}
if (old != null) {
for (ApiHolder holder : old.holders) {
@ -151,6 +153,12 @@ public class CustomContainerPlugins implements ClusterPropertiesListener {
}
}
@SuppressWarnings({"rawtypes", "unchecked"})
private static Map<String, String> getTemplateVars(PluginMeta pluginMeta) {
Map result = makeMap("plugin-name", pluginMeta.name, "path-prefix", pluginMeta.pathPrefix);
return result;
}
private static class ApiHolder extends Api {
final AnnotatedApi api;
@ -236,7 +244,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener {
errs.add("The @EndPint must have exactly one method and path attributes");
}
List<String> pathSegments = StrUtils.splitSmart(endPoint.path()[0], '/', true);
PathTrie.replaceTemplates(pathSegments, Collections.singletonMap("plugin-name", info.name));
PathTrie.replaceTemplates(pathSegments, getTemplateVars(info));
if (V2HttpCall.knownPrefixes.contains(pathSegments.get(0))) {
errs.add("path must not have a prefix: "+pathSegments.get(0));
}
@ -270,6 +278,13 @@ public class CustomContainerPlugins implements ClusterPropertiesListener {
} else {
throw new RuntimeException("Must have a no-arg constructor or CoreContainer constructor ");
}
if (instance instanceof ResourceLoaderAware) {
try {
((ResourceLoaderAware) instance).inform(pkgVersion.getLoader());
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
this.holders = new ArrayList<>();
for (Api api : AnnotatedApi.getApis(instance)) {
holders.add(new ApiHolder((AnnotatedApi) api));

54
solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
View File

@ -18,18 +18,27 @@
package org.apache.solr.pkg;
import java.io.Closeable;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Supplier;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.SimplePostTool;
import org.apache.zookeeper.server.ByteBufferInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -303,9 +312,12 @@ public class PackageLoader implements Closeable {
}
}
static class PackageResourceLoader extends SolrResourceLoader {
List<Path> paths;
PackageResourceLoader(String name, List<Path> classpath, Path instanceDir, ClassLoader parent) {
super(name, classpath, instanceDir, parent);
this.paths = classpath;
}
@Override
@ -329,6 +341,48 @@ public class PackageLoader implements Closeable {
public <T> void addToInfoBeans(T obj) {
//do not do anything. It should be handled externally
}
@Override
public InputStream openResource(String resource) throws IOException {
for (Path path : paths) {
try(FileInputStream in = new FileInputStream(path.toFile())) {
ZipInputStream zis = new ZipInputStream(in);
try {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
if (resource == null || resource.equals(entry.getName())) {
SimplePostTool.BAOS out = new SimplePostTool.BAOS();
byte[] buffer = new byte[2048];
int size;
while ((size = zis.read(buffer, 0, buffer.length)) != -1) {
out.write(buffer, 0, size);
}
out.close();
return new ByteBufferStream(out.getByteBuffer());
}
}
} finally {
zis.closeEntry();
}
}
}
return null;
}
}
private static class ByteBufferStream extends ByteBufferInputStream implements Supplier<ByteBuffer> {
private final ByteBuffer buf ;
public ByteBufferStream(ByteBuffer buf) {
super(buf);
this.buf = buf;
}
@Override
public ByteBuffer get() {
return buf;
}
}
private static String findBiggest(String lessThan, List<String> sortedList) {

52
solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java
View File

@ -18,11 +18,16 @@
package org.apache.solr.handler;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.function.Supplier;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.api.Command;
import org.apache.solr.api.EndPoint;
import org.apache.solr.client.solrj.SolrClient;
@ -37,6 +42,7 @@ import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.filestore.PackageStoreAPI;
import org.apache.solr.filestore.TestDistribPackageStore;
import org.apache.solr.filestore.TestDistribPackageStore.Fetcher;
@ -124,9 +130,12 @@ public class TestContainerPlugin extends SolrCloudTestCase {
//test with a class @EndPoint methods. This also uses a template in the path name
plugin.klass = C4.class.getName();
plugin.name = "collections";
plugin.pathPrefix = "collections";
expectError(req, cluster.getSolrClient(), errPath, "path must not have a prefix: collections");
plugin.name = "my-random-name";
plugin.pathPrefix = "my-random-prefix";
req.process(cluster.getSolrClient());
//let's test the plugin
@ -138,7 +147,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
ImmutableMap.of("/method.name", "m1"));
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/my-random-name/their/plugin")
() -> new V2Request.Builder("/my-random-prefix/their/plugin")
.forceV2(true)
.withMethod(GET)
.build().process(cluster.getSolrClient()),
@ -188,7 +197,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
plugin.name = "myplugin";
plugin.klass = "mypkg:org.apache.solr.handler.MyPlugin";
plugin.version = add.version;
V2Request req1 = new V2Request.Builder("/cluster/plugin")
final V2Request req1 = new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.withMethod(POST)
.withPayload(singletonMap("add", plugin))
@ -235,15 +244,50 @@ public class TestContainerPlugin extends SolrCloudTestCase {
"/plugin/myplugin/class", plugin.klass,
"/plugin/myplugin/version", "2.0"
));
// invoke the plugin and test thye output
// invoke the plugin and test the output
TestDistribPackageStore.assertResponseValues(10,
invokePlugin,
ImmutableMap.of("/myplugin.version", "2.0"));
plugin.name = "plugin2";
plugin.klass = "mypkg:"+ C5.class.getName();
plugin.version = "2.0";
req1.process(cluster.getSolrClient());
assertNotNull(C5.classData);
assertEquals( 1452, C5.classData.limit());
} finally {
cluster.shutdown();
}
}
public static class C5 implements ResourceLoaderAware {
static ByteBuffer classData;
private SolrResourceLoader resourceLoader;
@Override
@SuppressWarnings("unchecked")
public void inform(ResourceLoader loader) throws IOException {
this.resourceLoader = (SolrResourceLoader) loader;
try {
InputStream is = resourceLoader.openResource("org/apache/solr/handler/MyPlugin.class");
if (is instanceof Supplier) {
classData = ((Supplier<ByteBuffer>) is).get();
}
} catch (IOException e) {
//do not do anything
}
}
@EndPoint(method = GET,
path = "/$plugin-name/m2",
permission = PermissionNameProvider.Name.COLL_READ_PERM)
public void m2() {
}
}
public static class C1 {
}
@ -279,7 +323,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
}
@EndPoint(method = GET,
path = "$plugin-name/their/plugin",
path = "$path-prefix/their/plugin",
permission = PermissionNameProvider.Name.READ_PERM)
public void m2(SolrQueryRequest req, SolrQueryResponse rsp) {
rsp.add("method.name", "m2");

3
solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PluginMeta.java
View File

@ -35,6 +35,9 @@ public class PluginMeta implements ReflectMapWriter {
@JsonProperty
public String version;
@JsonProperty("path-prefix")
public String pathPrefix;
public PluginMeta copy() {
PluginMeta result = new PluginMeta();