Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14977 : ContainerPlugins should be configurable (#2065)

This commit is contained in:
Noble Paul 2020-11-17 00:19:50 +11:00 committed by GitHub
parent ebc87a8a27
commit 73d5e7ae77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 251 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -176,6 +176,8 @@ Improvements
* SOLR-14683: Metrics API should ensure consistent placeholders for missing values. (ab)
* SOLR-14977 : ContainerPlugins should be configurable with custom objects (noble, ab)
Optimizations
---------------------
* SOLR-14975: Optimize CoreContainer.getAllCoreNames, getLoadedCoreNames and getCoreDescriptors. (Bruno Roustant)

5
solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
View File

@ -33,6 +33,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.SolrException;
@ -62,6 +63,8 @@ import org.slf4j.LoggerFactory;
public class AnnotatedApi extends Api implements PermissionNameProvider , Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
public static final String ERR = "Error executing commands :";
private EndPoint endPoint;
@ -222,7 +225,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider , Closea
final String command;
final MethodHandle method;
final Object obj;
ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
int paramsCount;
@SuppressWarnings({"rawtypes"})
Class parameterClass;

33
solr/core/src/java/org/apache/solr/api/ConfigurablePlugin.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.api;
import org.apache.solr.common.MapWriter;
/**Implement this interface if your plugin needs to accept some configuration
*
* @param <T> the configuration Object type
*/
public interface ConfigurablePlugin<T extends MapWriter> {
/**This is invoked soon after the Object is initialized
*
* @param cfg value deserialized from JSON
*/
void configure(T cfg);
}

88
solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java
View File

@ -22,6 +22,8 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -31,6 +33,7 @@ import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Supplier;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.lucene.util.ResourceLoaderAware;
import org.apache.solr.client.solrj.SolrRequest;
@ -69,7 +72,8 @@ import static org.apache.solr.common.util.Utils.makeMap;
public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapWriter, Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
private static final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
private final List<PluginRegistryListener> listeners = new CopyOnWriteArrayList<>();
@ -114,6 +118,30 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
return currentPlugins.get(name);
}
static class PluginMetaHolder {
private final Map<String, Object> original;
private final PluginMeta meta;
PluginMetaHolder(Map<String, Object> original) throws IOException {
this.original = original;
meta = mapper.readValue(Utils.toJSON(original), PluginMeta.class);
}
@Override
public boolean equals(Object obj) {
if (obj instanceof PluginMetaHolder) {
PluginMetaHolder that = (PluginMetaHolder) obj;
return Objects.equals(this.original,that.original);
}
return false;
}
@Override
public int hashCode() {
return original.hashCode();
}
}
@SuppressWarnings("unchecked")
public synchronized void refresh() {
Map<String, Object> pluginInfos = null;
try {
@ -122,19 +150,18 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
log.error("Could not read plugins data", e);
return;
}
Map<String,PluginMeta> newState = new HashMap<>(pluginInfos.size());
Map<String,PluginMetaHolder> newState = new HashMap<>(pluginInfos.size());
for (Map.Entry<String, Object> e : pluginInfos.entrySet()) {
try {
newState.put(e.getKey(),
mapper.readValue(Utils.toJSON(e.getValue()), PluginMeta.class));
newState.put(e.getKey(),new PluginMetaHolder((Map<String, Object>) e.getValue()));
} catch (Exception exp) {
log.error("Invalid apiInfo configuration :", exp);
}
}
Map<String, PluginMeta> currentState = new HashMap<>();
Map<String, PluginMetaHolder> currentState = new HashMap<>();
for (Map.Entry<String, ApiInfo> e : currentPlugins.entrySet()) {
currentState.put(e.getKey(), e.getValue().info);
currentState.put(e.getKey(), e.getValue().holder);
}
Map<String, Diff> diff = compareMaps(currentState, newState);
if (diff == null) return;//nothing has changed
@ -153,10 +180,10 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
}
} else {
//ADDED or UPDATED
PluginMeta info = newState.get(e.getKey());
PluginMetaHolder info = newState.get(e.getKey());
ApiInfo apiInfo = null;
List<String> errs = new ArrayList<>();
apiInfo = new ApiInfo(info, errs);
apiInfo = new ApiInfo(info,errs);
if (!errs.isEmpty()) {
log.error(StrUtils.join(errs, ','));
continue;
@ -243,8 +270,10 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
public class ApiInfo implements ReflectMapWriter {
List<ApiHolder> holders;
private final PluginMetaHolder holder;
@JsonProperty
private final PluginMeta info;
private PluginMeta info;
@JsonProperty(value = "package")
public final String pkg;
@ -272,8 +301,9 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
return info.copy();
}
@SuppressWarnings({"unchecked","rawtypes"})
public ApiInfo(PluginMeta info, List<String> errs) {
this.info = info;
public ApiInfo(PluginMetaHolder infoHolder, List<String> errs) {
this.holder = infoHolder;
this.info = infoHolder.meta;
PluginInfo.ClassName klassInfo = new PluginInfo.ClassName(info.klass);
pkg = klassInfo.pkg;
if (pkg != null) {
@ -349,7 +379,7 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
}
}
@SuppressWarnings({"rawtypes"})
@SuppressWarnings({"rawtypes", "unchecked"})
public void init() throws Exception {
if (this.holders != null) return;
Constructor constructor = klas.getConstructors()[0];
@ -360,6 +390,13 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
} else {
throw new RuntimeException("Must have a no-arg constructor or CoreContainer constructor ");
}
if (instance instanceof ConfigurablePlugin) {
Class<? extends MapWriter> c = getConfigClass((ConfigurablePlugin<? extends MapWriter>) instance);
if (c != null) {
MapWriter initVal = mapper.readValue(Utils.toJSON(holder.original), c);
((ConfigurablePlugin) instance).configure(initVal);
}
}
if (instance instanceof ResourceLoaderAware) {
try {
((ResourceLoaderAware) instance).inform(pkgVersion.getLoader());
@ -372,9 +409,34 @@ public class ContainerPluginsRegistry implements ClusterPropertiesListener, MapW
holders.add(new ApiHolder((AnnotatedApi) api));
}
}
}
public ApiInfo createInfo(PluginMeta info, List<String> errs) {
/**Get the generic type of a {@link ConfigurablePlugin}
*/
@SuppressWarnings("rawtypes")
public static Class getConfigClass(ConfigurablePlugin<?> o) {
Class klas = o.getClass();
do {
Type[] interfaces = klas.getGenericInterfaces();
for (Type type : interfaces) {
if (type instanceof ParameterizedType) {
ParameterizedType parameterizedType = (ParameterizedType) type;
if (parameterizedType.getRawType() == ConfigurablePlugin.class) {
return (Class) parameterizedType.getActualTypeArguments()[0];
}
}
}
klas = klas.getSuperclass();
} while (klas != null && klas != Object.class);
return null;
}
public ApiInfo createInfo(Map<String,Object> info, List<String> errs) throws IOException {
return new ApiInfo(new PluginMetaHolder(info), errs);
}
public ApiInfo createInfo(PluginMetaHolder info, List<String> errs) {
return new ApiInfo(info, errs);
}

2
solr/core/src/java/org/apache/solr/api/V2HttpCall.java
View File

@ -136,7 +136,7 @@ public class V2HttpCall extends HttpSolrCall {
initAdminRequest(path);
return;
} else {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no core retrieved for " + origCorename);
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no core retrieved for core name: " + origCorename + ". Path : "+ path);
}
}

18
solr/core/src/java/org/apache/solr/handler/admin/ContainerPluginsApi.java
View File

@ -19,10 +19,7 @@ package org.apache.solr.handler.admin;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.function.Function;
import java.util.function.Supplier;
@ -94,7 +91,7 @@ public class ContainerPluginsApi {
payload.addError(info.name + " already exists");
return null;
}
map.put(info.name, info);
map.put(info.name, payload.getDataMap());
return map;
});
}
@ -122,14 +119,19 @@ public class ContainerPluginsApi {
payload.addError("No such plugin: " + info.name);
return null;
} else {
map.put(info.name, info);
Map<String, Object> jsonObj = payload.getDataMap();
if(Objects.equals(jsonObj, existing)) {
//no need to change anything
return null;
}
map.put(info.name, jsonObj);
return map;
}
});
}
}
private void validateConfig(PayloadObj<PluginMeta> payload, PluginMeta info) {
private void validateConfig(PayloadObj<PluginMeta> payload, PluginMeta info) throws IOException {
if (info.klass.indexOf(':') > 0) {
if (info.version == null) {
payload.addError("Using package. must provide a packageVersion");
@ -137,7 +139,7 @@ public class ContainerPluginsApi {
}
}
List<String> errs = new ArrayList<>();
ContainerPluginsRegistry.ApiInfo apiInfo = coreContainer.getContainerPluginsRegistry().createInfo(info, errs);
ContainerPluginsRegistry.ApiInfo apiInfo = coreContainer.getContainerPluginsRegistry().createInfo( payload.getDataMap(), errs);
if (!errs.isEmpty()) {
for (String err : errs) payload.addError(err);
return;

124
solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java
View File

@ -28,6 +28,8 @@ import org.apache.commons.io.IOUtils;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
import org.apache.solr.api.Command;
import org.apache.solr.api.ConfigurablePlugin;
import org.apache.solr.api.ContainerPluginsRegistry;
import org.apache.solr.api.EndPoint;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
@ -41,6 +43,8 @@ import org.apache.solr.cloud.ClusterSingleton;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.util.ReflectMapWriter;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrResourceLoader;
@ -58,7 +62,6 @@ import org.junit.Test;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
@ -88,7 +91,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
//test with an invalid class
V2Request req = new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.withMethod(POST)
.POST()
.withPayload(singletonMap("add", plugin))
.build();
expectError(req, cluster.getSolrClient(), errPath, "No method with @Command in class");
@ -100,7 +103,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
//just check if the plugin is indeed registered
V2Request readPluginState = new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build();
V2Response rsp = readPluginState.process(cluster.getSolrClient());
assertEquals(C3.class.getName(), rsp._getStr("/plugin/testplugin/class", null));
@ -109,13 +112,13 @@ public class TestContainerPlugin extends SolrCloudTestCase {
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/plugin/my/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of("/testkey", "testval"));
//now remove the plugin
new V2Request.Builder("/cluster/plugin")
.withMethod(POST)
.POST()
.forceV2(true)
.withPayload("{remove : testplugin}")
.build()
@ -140,19 +143,19 @@ public class TestContainerPlugin extends SolrCloudTestCase {
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/my-random-name/my/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of("/method.name", "m1"));
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/my-random-prefix/their/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of("/method.name", "m2"));
//now remove the plugin
new V2Request.Builder("/cluster/plugin")
.withMethod(POST)
.POST()
.forceV2(true)
.withPayload("{remove : my-random-name}")
.build()
@ -160,12 +163,12 @@ public class TestContainerPlugin extends SolrCloudTestCase {
expectFail( () -> new V2Request.Builder("/my-random-prefix/their/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build()
.process(cluster.getSolrClient()));
expectFail(() -> new V2Request.Builder("/my-random-prefix/their/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build()
.process(cluster.getSolrClient()));
@ -177,7 +180,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
//just check if the plugin is indeed registered
readPluginState = new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.withMethod(GET)
.GET()
.build();
rsp = readPluginState.process(cluster.getSolrClient());
assertEquals(C6.class.getName(), rsp._getStr("/plugin/clusterSingleton/class", null));
@ -185,7 +188,47 @@ public class TestContainerPlugin extends SolrCloudTestCase {
assertTrue("ccProvided", C6.ccProvided);
assertTrue("startCalled", C6.startCalled);
assertFalse("stopCalled", C6.stopCalled);
// kill the Overseer leader
assertEquals( CConfig.class, ContainerPluginsRegistry.getConfigClass(new CC()));
assertEquals( CConfig.class, ContainerPluginsRegistry.getConfigClass(new CC1()));
assertEquals( CConfig.class, ContainerPluginsRegistry.getConfigClass(new CC2()));
CConfig p = new CConfig();
p.boolVal = Boolean.TRUE;
p.strVal = "Something";
p.longVal = 1234L;
p.name = "hello";
p.klass = CC.class.getName();
new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.POST()
.withPayload(singletonMap("add", p))
.build()
.process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("hello/plugin")
.forceV2(true)
.GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of("/config/boolVal", "true", "/config/strVal", "Something","/config/longVal", "1234" ));
p.strVal = "Something else";
new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.POST()
.withPayload(singletonMap("update", p))
.build()
.process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("hello/plugin")
.forceV2(true)
.GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of("/config/boolVal", "true", "/config/strVal", p.strVal,"/config/longVal", "1234" ));
// kill the Overseer leader
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
if (!jetty.getCoreContainer().getZkController().getOverseer().isClosed()) {
cluster.stopJettySolrRunner(jetty);
@ -234,7 +277,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
add.files = singletonList(FILE1);
V2Request addPkgVersionReq = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(POST)
.POST()
.withPayload(singletonMap("add", add))
.build();
addPkgVersionReq.process(cluster.getSolrClient());
@ -251,14 +294,14 @@ public class TestContainerPlugin extends SolrCloudTestCase {
plugin.version = add.version;
final V2Request req1 = new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.withMethod(POST)
.POST()
.withPayload(singletonMap("add", plugin))
.build();
req1.process(cluster.getSolrClient());
//verify the plugin creation
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/cluster/plugin").
withMethod(GET)
GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of(
"/plugin/myplugin/class", plugin.klass,
@ -267,7 +310,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
//let's test this now
Callable<NavigableObject> invokePlugin = () -> new V2Request.Builder("/plugin/my/path")
.forceV2(true)
.withMethod(GET)
.GET()
.build().process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
invokePlugin,
@ -282,7 +325,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
plugin.version = add.version;
new V2Request.Builder("/cluster/plugin")
.forceV2(true)
.withMethod(POST)
.POST()
.withPayload(singletonMap("update", plugin))
.build()
.process(cluster.getSolrClient());
@ -290,7 +333,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {
//now verify if it is indeed updated
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/cluster/plugin").
withMethod(GET)
GET()
.build().process(cluster.getSolrClient()),
ImmutableMap.of(
"/plugin/myplugin/class", plugin.klass,
@ -312,6 +355,50 @@ public class TestContainerPlugin extends SolrCloudTestCase {
}
}
public static class CC1 extends CC {
}
public static class CC2 extends CC1 {
}
public static class CC implements ConfigurablePlugin<CConfig> {
private CConfig cfg;
@Override
public void configure(CConfig cfg) {
this.cfg = cfg;
}
@EndPoint(method = GET,
path = "/hello/plugin",
permission = PermissionNameProvider.Name.READ_PERM)
public void m2(SolrQueryRequest req, SolrQueryResponse rsp) {
rsp.add("config", cfg);
}
}
public static class CConfig implements ReflectMapWriter {
@JsonProperty
public String strVal;
@JsonProperty
public Long longVal;
@JsonProperty
public Boolean boolVal;
@JsonProperty
public String name;
@JsonProperty(value = "class", required = true)
public String klass;
}
public static class C6 implements ClusterSingleton {
static boolean startCalled = false;
static boolean stopCalled = false;
@ -356,7 +443,6 @@ public class TestContainerPlugin extends SolrCloudTestCase {
private SolrResourceLoader resourceLoader;
@Override
@SuppressWarnings("unchecked")
public void inform(ResourceLoader loader) throws IOException {
this.resourceLoader = (SolrResourceLoader) loader;
try {

20
solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
View File

@ -165,6 +165,26 @@ public class V2Request extends SolrRequest<V2Response> implements MapWriter {
return this;
}
public Builder POST() {
this.method = METHOD.POST;
return this;
}
public Builder GET() {
this.method = METHOD.GET;
return this;
}
public Builder PUT() {
this.method = METHOD.PUT;
return this;
}
public Builder DELETE() {
this.method = METHOD.DELETE;
return this;
}
/**
* Only for testing. It's always true otherwise
*/

1
solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaCreator.java
View File

@ -87,6 +87,7 @@ public class JsonSchemaCreator {
if(p.required()) required.add(name);
}
if(!required.isEmpty()) map.put("required", new ArrayList<>(required));
map.put("additionalProperties", true);
}
}