Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-996 -- Expose Context to Evaluators 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@745734 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2009-02-19 05:28:48 +00:00
parent 22da31152c
commit a2b4aa53fc
7 changed files with 81 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
contrib/dataimporthandler/CHANGES.txt
View File

@ -13,6 +13,9 @@ $Id$
Upgrading from Solr 1.3
-----------------------
Evaluator API has been changed in a non back-compatible way. Users who have developed custom Evaluators will need
to change their code according to the new API for it to work. See SOLR-996 for details.
Detailed Change List
----------------------
@ -69,6 +72,9 @@ New Features
16.SOLR-989: Expose running statistics from the Context API.
(Noble Paul, shalin)
17.SOLR-996: Expose Context to Evaluators.
(Noble Paul, shalin)
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used

20
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -62,6 +62,7 @@ public class DocBuilder {
private Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
Map<String,Object> functionsNamespace;
public DocBuilder(DataImporter context, SolrWriter writer, DataImporter.RequestParams reqParams) {
INSTANCE.set(this);
@ -70,6 +71,7 @@ public class DocBuilder {
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
requestParameters = reqParams;
verboseDebug = requestParameters.debug && requestParameters.verbose;
functionsNamespace = EvaluatorBag.getFunctionsNamespace(dataImporter.getConfig().functions, this);
}
public VariableResolverImpl getVariableResolver() {
@ -80,13 +82,10 @@ public class DocBuilder {
DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getLastIndexTime()));
indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
indexerNamespace.put("request", requestParameters.requestParams);
indexerNamespace.put("functions", EvaluatorBag.getFunctionsNamespace(resolver,
dataImporter.getConfig().functions, this));
indexerNamespace.put("functions", functionsNamespace);
if (dataImporter.getConfig().script != null) {
indexerNamespace
.put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
indexerNamespace.put(DataConfig.SCRIPT_LANG,
dataImporter.getConfig().script.language);
indexerNamespace.put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
indexerNamespace.put(DataConfig.SCRIPT_LANG, dataImporter.getConfig().script.language);
}
resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
return resolver;
@ -290,6 +289,7 @@ public class DocBuilder {
ContextImpl ctx = new ContextImpl(entity, vr, null,
pk == null ? Context.FULL_DUMP : Context.DELTA_DUMP,
session, parentCtx, this);
vr.context = ctx;
entityProcessor.init(ctx);
if (requestParameters.start > 0) {
@ -517,7 +517,9 @@ public class DocBuilder {
Set<Map<String, Object>> deltaSet = new HashSet<Map<String, Object>>();
resolver.addNamespace(null, (Map) entity.allAttributes);
EntityProcessor entityProcessor = getEntityProcessor(entity, context.getCore());
entityProcessor.init(new ContextImpl(entity, resolver, null, Context.FIND_DELTA, session, null, this));
ContextImpl context1 = new ContextImpl(entity, resolver, null, Context.FIND_DELTA, session, null, this);
resolver.context = context1;
entityProcessor.init(context1);
LOG.info("Running ModifiedRowKey() for Entity: " + entity.name);
//get the modified rows in this entity
while (true) {
@ -560,7 +562,9 @@ public class DocBuilder {
//so propogate up the changes in the chain
if (parentEntity != null && parentEntity.isDocRoot) {
EntityProcessor parentEntityProcessor = getEntityProcessor(parentEntity, context.getCore());
parentEntityProcessor.init(new ContextImpl(parentEntity, resolver, null, Context.FIND_DELTA, session, null, this));
ContextImpl context2 = new ContextImpl(parentEntity, resolver, null, Context.FIND_DELTA, session, null, this);
resolver.context = context2;
parentEntityProcessor.init(context2);
// identifying deleted rows with deltas
for (Map<String, Object> row : myModifiedPks)

4
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java
View File

@ -39,9 +39,9 @@ public abstract class Evaluator {
* Return a String after processing an expression and a VariableResolver
*
* @see org.apache.solr.handler.dataimport.VariableResolver
* @param resolver the VariableResolver instance to be used for evaluation
* @param expression string to be evaluated
* @param context instance
* @return the value of the given expression evaluated using the resolver
*/
public abstract String evaluate(VariableResolver resolver, String expression);
public abstract String evaluate(String expression, Context context);
}

20
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
View File

@ -71,8 +71,8 @@ public class EvaluatorBag {
*/
public static Evaluator getSqlEscapingEvaluator() {
return new Evaluator() {
public String evaluate(VariableResolver resolver, String expression) {
Object o = resolver.resolve(expression);
public String evaluate(String expression, Context context) {
Object o = context.getVariableResolver().resolve(expression);
if (o == null)
return null;
@ -94,10 +94,10 @@ public class EvaluatorBag {
*/
public static Evaluator getUrlEvaluator() {
return new Evaluator() {
public String evaluate(VariableResolver resolver, String expression) {
public String evaluate(String expression, Context context) {
Object value = null;
try {
value = resolver.resolve(expression);
value = context.getVariableResolver().resolve(expression);
if (value == null)
return null;
@ -130,7 +130,7 @@ public class EvaluatorBag {
*/
public static Evaluator getDateFormatEvaluator() {
return new Evaluator() {
public String evaluate(VariableResolver resolver, String expression) {
public String evaluate(String expression, Context context) {
CacheEntry e = getCachedData(expression);
String expr = e.key;
SimpleDateFormat fmt = e.format;
@ -146,7 +146,7 @@ public class EvaluatorBag {
"Invalid expression for date", exp);
}
} else {
Object o = resolver.resolve(expr);
Object o = context.getVariableResolver().resolve(expr);
if (o == null)
return "";
Date date = null;
@ -193,8 +193,7 @@ public class EvaluatorBag {
};
}
static Map<String, Object> getFunctionsNamespace(
final VariableResolver resolver, final List<Map<String, String>> fn, DocBuilder docBuilder) {
static Map<String, Object> getFunctionsNamespace(final List<Map<String, String>> fn, DocBuilder docBuilder) {
final Map<String, Evaluator> evaluators = new HashMap<String, Evaluator>();
evaluators.put(DATE_FORMAT_EVALUATOR, getDateFormatEvaluator());
evaluators.put(SQL_ESCAPE_EVALUATOR, getSqlEscapingEvaluator());
@ -222,8 +221,11 @@ public class EvaluatorBag {
Evaluator evaluator = evaluators.get(fname);
if (evaluator == null)
return null;
return evaluator.evaluate(resolver, m.group(2));
VariableResolverImpl vri = VariableResolverImpl.CURRENT_VARIABLE_RESOLVER.get();
Context ctx = vri == null ? null : vri.context;
return evaluator.evaluate(m.group(2), ctx);
}
};
}

41
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java
View File

@ -34,11 +34,21 @@ import java.util.regex.Pattern;
public class VariableResolverImpl extends VariableResolver {
private Map<String, Object> container = new HashMap<String, Object>();
/**
* Used for creating Evaluators
*/
ContextImpl context;
private static final TemplateString TEMPLATE_STRING = new TemplateString();
public VariableResolverImpl() {
}
/**
* The current resolver instance
*/
static final ThreadLocal<VariableResolverImpl> CURRENT_VARIABLE_RESOLVER = new ThreadLocal<VariableResolverImpl>();
@SuppressWarnings("unchecked")
public VariableResolverImpl addNamespace(String name, Map<String, Object> map) {
if (name != null) {
@ -84,22 +94,27 @@ public class VariableResolverImpl extends VariableResolver {
if ("".equals(name))
return null;
String[] parts = DOT_SPLIT.split(name, 0);
Map<String, Object> namespace = container;
for (int i = 0; i < parts.length; i++) {
String thePart = parts[i];
if (i == parts.length - 1) {
return namespace.get(thePart);
}
Object temp = namespace.get(thePart);
if (temp == null) {
return namespace.get(mergeAll(parts, i));
} else {
if (temp instanceof Map) {
namespace = (Map) temp;
CURRENT_VARIABLE_RESOLVER.set(this);
try {
Map<String, Object> namespace = container;
for (int i = 0; i < parts.length; i++) {
String thePart = parts[i];
if (i == parts.length - 1) {
return namespace.get(thePart);
}
Object temp = namespace.get(thePart);
if (temp == null) {
return namespace.get(mergeAll(parts, i));
} else {
return null;
if (temp instanceof Map) {
namespace = (Map) temp;
} else {
return null;
}
}
}
} finally {
CURRENT_VARIABLE_RESOLVER.set(null);
}
return null;
}

24
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
View File

@ -23,14 +23,13 @@ import org.junit.Test;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
* <p>
* Test for EvaluatorBag
* </p>
* <p> Test for EvaluatorBag </p>
*
* @version $Id$
* @since solr 1.3
@ -66,8 +65,7 @@ public class TestEvaluatorBag {
}
/**
* Test method for
* {@link EvaluatorBag#getSqlEscapingEvaluator()}.
* Test method for {@link EvaluatorBag#getSqlEscapingEvaluator()}.
*/
@Test
public void testGetSqlEscapingEvaluator() {
@ -76,8 +74,7 @@ public class TestEvaluatorBag {
}
/**
* Test method for
* {@link EvaluatorBag#getUrlEvaluator()}.
* Test method for {@link EvaluatorBag#getUrlEvaluator()}.
*/
@Test
public void testGetUrlEvaluator() throws Exception {
@ -86,32 +83,35 @@ public class TestEvaluatorBag {
}
/**
* Test method for
* {@link EvaluatorBag#getDateFormatEvaluator()}.
* Test method for {@link EvaluatorBag#getDateFormatEvaluator()}.
*/
@Test
@Ignore
public void testGetDateFormatEvaluator() {
Evaluator dateFormatEval = EvaluatorBag.getDateFormatEvaluator();
resolver.context = new ContextImpl(null, resolver, null, 0, Collections.EMPTY_MAP, null, null);
assertEquals(new SimpleDateFormat("yyyy-MM-dd").format(new Date()),
dateFormatEval.evaluate(resolver, "'NOW',yyyy-MM-dd HH:mm"));
dateFormatEval.evaluate("'NOW',yyyy-MM-dd HH:mm", resolver.context));
Map<String, Object> map = new HashMap<String, Object>();
map.put("key", new Date());
resolver.addNamespace("A", map);
assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm").format(new Date()),
dateFormatEval.evaluate(resolver, "A.key, yyyy-MM-dd HH:mm"));
dateFormatEval.evaluate("A.key, yyyy-MM-dd HH:mm", resolver.context));
}
private void runTests(Map<String, String> tests, Evaluator evaluator) {
ContextImpl ctx = new ContextImpl(null, resolver, null, 0, Collections.EMPTY_MAP, null, null);
resolver.context = ctx;
for (Map.Entry<String, String> entry : tests.entrySet()) {
Map<String, Object> values = new HashMap<String, Object>();
values.put("key", entry.getKey());
resolver.addNamespace("A", values);
String expected = (String) entry.getValue();
String actual = evaluator.evaluate(resolver, "A.key");
String actual = evaluator.evaluate("A.key", ctx);
assertEquals(expected, actual);
}
}

14
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
View File

@ -68,8 +68,10 @@ public class TestVariableResolver {
@Test
public void dateNamespaceWithValue() {
VariableResolverImpl vri = new VariableResolverImpl();
ContextImpl context = new ContextImpl(null,vri, null, 0,Collections.EMPTY_MAP, null,null);
vri.context = context;
vri.addNamespace("dataimporter.functions", EvaluatorBag
.getFunctionsNamespace(vri, Collections.EMPTY_LIST, null));
.getFunctionsNamespace(Collections.EMPTY_LIST, null));
Map<String, Object> ns = new HashMap<String, Object>();
Date d = new Date();
ns.put("dt", d);
@ -84,8 +86,10 @@ public class TestVariableResolver {
@Test
public void dateNamespaceWithExpr() {
VariableResolverImpl vri = new VariableResolverImpl();
ContextImpl context = new ContextImpl(null,vri, null, 0,Collections.EMPTY_MAP, null,null);
vri.context = context;
vri.addNamespace("dataimporter.functions", EvaluatorBag
.getFunctionsNamespace(vri, Collections.EMPTY_LIST,null));
.getFunctionsNamespace(Collections.EMPTY_LIST,null));
String s = vri
.replaceTokens("${dataimporter.functions.formatDate('NOW',yyyy-MM-dd HH:mm)}");
Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm")
@ -113,13 +117,15 @@ public class TestVariableResolver {
@Test
public void testFunctionNamespace1() {
final VariableResolverImpl resolver = new VariableResolverImpl();
ContextImpl context = new ContextImpl(null,resolver, null, 0,Collections.EMPTY_MAP, null,null);
resolver.context = context;
final List<Map<String ,String >> l = new ArrayList<Map<String, String>>();
Map<String ,String > m = new HashMap<String, String>();
m.put("name","test");
m.put("class",E.class.getName());
l.add(m);
resolver.addNamespace("dataimporter.functions", EvaluatorBag
.getFunctionsNamespace(resolver, l,null));
.getFunctionsNamespace(l,null));
String s = resolver
.replaceTokens("${dataimporter.functions.formatDate('NOW',yyyy-MM-dd HH:mm)}");
Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm")
@ -129,7 +135,7 @@ public class TestVariableResolver {
}
public static class E extends Evaluator{
public String evaluate(VariableResolver resolver, String expression) {
public String evaluate(String expression, Context context) {
return "Hello World";
}
}