Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory

This commit is contained in:
Noble Paul 2017-06-14 18:07:40 +09:30
parent f470bbcbdc
commit 92b17838a3
9 changed files with 106 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -159,6 +159,8 @@ Bug Fixes
* SOLR-10830: Solr now correctly enforces that the '_root_' field has the same fieldType as the
uniqueKey field. With out this enforcement, child document updating was unreliable. (hossman)
* SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory (noble)
Optimizations
----------------------

4
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
View File

@ -143,7 +143,7 @@ public class VariableResolver {
* @return the string with the placeholders replaced with their values
*/
public String replaceTokens(String template) {
return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun);
return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
}
public void addNamespace(String name, Map<String,Object> newMap) {
if (newMap != null) {
@ -164,7 +164,7 @@ public class VariableResolver {
}
public List<String> getVariables(String expr) {
return TemplateUpdateProcessorFactory.getVariables(expr, cache);
return TemplateUpdateProcessorFactory.getVariables(expr, cache, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
}
static class CurrentLevel {

32
solr/core/src/java/org/apache/solr/core/PluginBag.java
View File

@ -35,6 +35,9 @@ import java.util.zip.ZipInputStream;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
@ -46,15 +49,12 @@ import org.apache.solr.util.SimplePostTool;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonMap;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.api.ApiBag.HANDLER_NAME;
import static org.apache.solr.common.params.CommonParams.NAME;
/**
* This manages the lifecycle of a set of plugin of the same type .
@ -125,10 +125,12 @@ public class PluginBag<T> implements AutoCloseable {
public PluginHolder<T> createPlugin(PluginInfo info) {
if ("true".equals(String.valueOf(info.attributes.get("runtimeLib")))) {
log.debug(" {} : '{}' created with runtimeLib=true ", meta.getCleanTag(), info.name);
return new LazyPluginHolder<>(meta, info, core, core.getMemClassLoader());
return new LazyPluginHolder<T>(meta, info, core, "true".equals(System.getProperty("enable.runtime.lib")) ?
core.getMemClassLoader() :
core.getResourceLoader(), true);
} else if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader());
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
} else {
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
initInstance(inst, info);
@ -371,11 +373,13 @@ public class PluginBag<T> implements AutoCloseable {
protected SolrException solrException;
private final SolrCore core;
protected ResourceLoader resourceLoader;
private final boolean isRuntimeLib;
LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader) {
LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader, boolean isRuntimeLib) {
super(pluginInfo);
this.pluginMeta = pluginMeta;
this.isRuntimeLib = isRuntimeLib;
this.core = core;
this.resourceLoader = loader;
if (loader instanceof MemClassLoader) {
@ -413,7 +417,19 @@ public class PluginBag<T> implements AutoCloseable {
loader.loadJars();
}
Class<T> clazz = (Class<T>) pluginMeta.clazz;
T localInst = core.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader);
T localInst = null;
try {
localInst = core.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader);
} catch (SolrException e) {
if (isRuntimeLib && !(resourceLoader instanceof MemClassLoader)) {
throw new SolrException(SolrException.ErrorCode.getErrorCode(e.code()),
e.getMessage() + ". runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true",
e.getCause());
}
throw e;
}
initInstance(localInst, pluginInfo);
if (localInst instanceof SolrCoreAware) {
SolrResourceLoader.assertAwareCompatibility(SolrCoreAware.class, localInst);

18
solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
View File

@ -57,18 +57,18 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
doc.addField(fName, replaceTokens(template, templateCache, s -> {
Object v = doc.getFieldValue(s);
return v == null ? "" : v;
}));
}, BRACES_PLACEHOLDER_PATTERN));
}
}
}
public static Resolved getResolved(String template, Cache<String, Resolved> cache) {
public static Resolved getResolved(String template, Cache<String, Resolved> cache, Pattern pattern) {
Resolved r = cache == null ? null : cache.get(template);
if (r == null) {
r = new Resolved();
Matcher m = PLACEHOLDER_PATTERN.matcher(template);
Matcher m = pattern.matcher(template);
while (m.find()) {
String variable = m.group(1);
r.startIndexes.add(m.start(0));
@ -83,19 +83,19 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
/**
* Get a list of variables embedded in the template string.
*/
public static List<String> getVariables(String template, Cache<String, Resolved> cache) {
Resolved r = getResolved(template, cache);
public static List<String> getVariables(String template, Cache<String, Resolved> cache, Pattern pattern) {
Resolved r = getResolved(template, cache, pattern );
if (r == null) {
return Collections.emptyList();
}
return new ArrayList<>(r.variables);
}
public static String replaceTokens(String template, Cache<String, Resolved> cache, Function<String, Object> fun) {
public static String replaceTokens(String template, Cache<String, Resolved> cache, Function<String, Object> fun, Pattern pattern) {
if (template == null) {
return null;
}
Resolved r = getResolved(template, cache);
Resolved r = getResolved(template, cache, pattern);
if (r.startIndexes != null) {
StringBuilder sb = new StringBuilder(template);
for (int i = r.startIndexes.size() - 1; i >= 0; i--) {
@ -115,6 +115,8 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
public List<String> variables = new ArrayList<>(2);
}
public static final Pattern PLACEHOLDER_PATTERN = Pattern
public static final Pattern DOLLAR_BRACES_PLACEHOLDER_PATTERN = Pattern
.compile("[$][{](.*?)[}]");
public static final Pattern BRACES_PLACEHOLDER_PATTERN = Pattern
.compile("[{](.*?)[}]");
}

15
solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
View File

@ -29,7 +29,6 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
@ -272,16 +271,12 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
if (s.isEmpty()) continue;
UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s);
if (p == null) {
try {
PluginInfo pluginInfo = new PluginInfo("updateProcessor",
Utils.makeMap("name", s,
"class", s + "UpdateProcessorFactory",
"runtimeLib", "true"));
PluginInfo pluginInfo = new PluginInfo("updateProcessor",
Utils.makeMap("name", s,
"class", s + "UpdateProcessorFactory",
"runtimeLib", "true"));
PluginBag.PluginHolder<UpdateRequestProcessorFactory> pluginHolder = core.getUpdateProcessors().createPlugin(pluginInfo);
core.getUpdateProcessors().put(s, p = pluginHolder.get());
} catch (SolrException e) {
}
core.getUpdateProcessors().put(s, p = core.getUpdateProcessors().createPlugin(pluginInfo).get());
if (p == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such processor " + s);
}

1
solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml
View File

@ -24,5 +24,6 @@
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="id" type="string" indexed="true" stored="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true" />
<uniqueKey>id</uniqueKey>
</schema>

67
solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java
View File

@ -17,22 +17,56 @@
package org.apache.solr.update.processor;
import org.apache.solr.SolrTestCaseJ4;
import java.lang.invoke.MethodHandles;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TemplateUpdateProcessorTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(5)
.addConfig("conf1", configset("cloud-minimal"))
.configure();
}
@After
public void after() throws Exception {
cluster.deleteAllCollections();
cluster.shutdown();
}
@org.junit.Rule
public ExpectedException expectedException = ExpectedException.none();
public class TemplateUpdateProcessorTest extends SolrTestCaseJ4 {
public void testSimple() throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams()
.add("processor", "Template")
.add("Template.field", "id:{firstName}_{lastName}")
.add("Template.field", "another:{lastName}_{firstName}")
.add("Template.field", "missing:{lastName}_{unKnown}");
AddUpdateCommand cmd = new AddUpdateCommand(new LocalSolrQueryRequest(null,
new ModifiableSolrParams()
.add("processor", "Template")
.add("Template.field", "id:${firstName}_${lastName}")
.add("Template.field", "another:${lastName}_${firstName}")
.add("Template.field", "missing:${lastName}_${unKnown}")
params
));
cmd.solrDoc = new SolrInputDocument();
@ -44,5 +78,24 @@ public class TemplateUpdateProcessorTest extends SolrTestCaseJ4 {
assertEquals("Cruise_Tom", cmd.solrDoc.getFieldValue("another"));
assertEquals("Cruise_", cmd.solrDoc.getFieldValue("missing"));
SolrInputDocument solrDoc = new SolrInputDocument();
solrDoc.addField("id", "1");
params = new ModifiableSolrParams()
.add("processor", "Template")
.add("commit", "true")
.add("Template.field", "x_s:key_{id}");
params.add("commit", "true");
UpdateRequest add = new UpdateRequest().add(solrDoc);
add.setParams(params);
NamedList<Object> result = cluster.getSolrClient().request(CollectionAdminRequest.createCollection("c", "conf1", 1, 1));
Utils.toJSONString(result.asMap(4));
AbstractFullDistribZkTestBase.waitForCollection(cluster.getSolrClient().getZkStateReader(), "c",1);
cluster.getSolrClient().request(add, "c");
QueryResponse rsp = cluster.getSolrClient().query("c",
new ModifiableSolrParams().add("q","id:1"));
assertEquals( "key_1", rsp.getResults().get(0).getFieldValue("x_s"));
}
}

2
solr/solr-ref-guide/src/update-request-processors.adoc
View File

@ -392,7 +392,7 @@ For example:
[source,bash]
----
processor=Template&Template.field=fullName:Mr. ${firstName} ${lastName}
processor=Template&Template.field=fullName:Mr. {firstName} {lastName}
----
The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used.

2
solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
View File

@ -353,7 +353,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
}
protected static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception {
public static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception {
// wait until shards have started registering...
int cnt = 30;
while (!reader.getClusterState().hasCollection(collection)) {