Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2141 / SOLR-4047 / SOLR-3842 - fix problems with VariableResolver, better test coverage 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1414242 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
James Dyer 2012-11-27 16:12:26 +00:00
parent aa0958f735
commit 9c1d89fe1b
5 changed files with 172 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
View File

@ -67,6 +67,15 @@ public class DateFormatEvaluator extends Evaluator {
availableLocales.put(locale.toString(), locale);
}
}
private SimpleDateFormat getDateFormat(String pattern, Locale locale) {
DateFormatCacheKey dfck = new DateFormatCacheKey(locale, pattern);
SimpleDateFormat sdf = cache.get(dfck);
if(sdf == null) {
sdf = new SimpleDateFormat(pattern, locale);
cache.put(dfck, sdf);
}
return sdf;
}
@Override
@ -81,15 +90,13 @@ public class DateFormatEvaluator extends Evaluator {
VariableWrapper wrapper = (VariableWrapper) format;
o = wrapper.resolve();
format = o.toString();
}
}
Locale locale = Locale.ROOT;
if(l.size()==3) {
Object localeObj = l.get(2);
String localeStr = null;
if (localeObj instanceof VariableWrapper) {
VariableWrapper wrapper = (VariableWrapper) localeObj;
o = wrapper.resolve();
localeStr = o.toString();
localeStr = ((VariableWrapper) localeObj).resolve().toString();
} else {
localeStr = localeObj.toString();
}
@ -97,14 +104,9 @@ public class DateFormatEvaluator extends Evaluator {
if(locale==null) {
throw new DataImportHandlerException(SEVERE, "Unsupported locale: " + localeStr);
}
}
}
String dateFmt = format.toString();
DateFormatCacheKey dfck = new DateFormatCacheKey(locale, dateFmt);
SimpleDateFormat sdf = cache.get(dfck);
if(sdf==null) {
sdf = new SimpleDateFormat(dateFmt, locale);
cache.put(dfck, sdf);
}
SimpleDateFormat fmt = getDateFormat(dateFmt, locale);
Date date = null;
if (o instanceof VariableWrapper) {
VariableWrapper variableWrapper = (VariableWrapper) o;
@ -114,13 +116,7 @@ public class DateFormatEvaluator extends Evaluator {
} else {
String s = variableval.toString();
try {
dfck = new DateFormatCacheKey(locale, DEFAULT_DATE_FORMAT);
sdf = cache.get(dfck);
if(sdf==null) {
sdf = new SimpleDateFormat(dfck.dateFormat, dfck.locale);
cache.put(dfck, sdf);
}
date = new SimpleDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
date = getDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
} catch (ParseException exp) {
wrapAndThrow(SEVERE, exp, "Invalid expression for date");
}
@ -134,7 +130,7 @@ public class DateFormatEvaluator extends Evaluator {
wrapAndThrow(SEVERE, e, "Invalid expression for date");
}
}
return sdf.format(date);
return fmt.format(date);
}
static DateMathParser getDateMathParser(Locale l) {
return new DateMathParser(TimeZone.getDefault(), l) {

14
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -68,7 +68,6 @@ public class DocBuilder {
Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
//private Map<String, Object> functionsNamespace;
private Map<String, Object> persistedProperties;
private DIHProperties propWriter;
@ -640,11 +639,20 @@ public class DocBuilder {
if (field != null) {
for (EntityField f : field) {
String name = f.getName();
boolean multiValued = f.isMultiValued();
boolean toWrite = f.isToWrite();
if(f.isDynamicName()){
name = vr.replaceTokens(name);
SchemaField schemaField = dataImporter.getSchemaField(name);
if(schemaField == null) {
toWrite = false;
} else {
multiValued = schemaField.multiValued();
toWrite = true;
}
}
if (f.isToWrite()) {
addFieldToDoc(entry.getValue(), name, f.getBoost(), f.isMultiValued(), doc);
if (toWrite) {
addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
}
}
}

11
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
View File

@ -63,6 +63,7 @@ public class VariableResolver {
}
public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
public VariableResolver() {
rootNamespace = new HashMap<String,Object>();
@ -95,7 +96,11 @@ public class VariableResolver {
r = currentLevel.get(nameParts[nameParts.length - 1]);
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
&& name.length() > FUNCTIONS_NAMESPACE.length()) {
return resolveEvaluator(name);
return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
}
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
&& name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
}
if (r == null) {
r = System.getProperty(name);
@ -104,12 +109,12 @@ public class VariableResolver {
return r == null ? "" : r;
}
private Object resolveEvaluator(String name) {
private Object resolveEvaluator(String namespace, String name) {
if (evaluators == null) {
return "";
}
Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
.substring(FUNCTIONS_NAMESPACE.length()));
.substring(namespace.length()));
if (m.find()) {
String fname = m.group(1);
Evaluator evaluator = evaluators.get(fname);

2
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
View File

@ -24,7 +24,7 @@ import java.text.SimpleDateFormat;
import java.util.*;
/**
* <p> Test for EvaluatorBag </p>
* <p> Test for Evaluators </p>
*
*
* @since solr 1.3

137
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java Normal file
View File

@ -0,0 +1,137 @@
package org.apache.solr.handler.dataimport;
import java.sql.Connection;
import java.sql.Statement;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import junit.framework.Assert;
import org.apache.solr.request.SolrQueryRequest;
import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
@Test
public void test() throws Exception {
h.query("/dataimport", generateRequest());
SolrQueryRequest req = null;
try {
req = req("q", "*:*", "wt", "json", "indent", "true");
String response = h.query(req);
log.debug(response);
response = response.replaceAll("\\s","");
Assert.assertTrue(response.contains("\"numFound\":1"));
Pattern p = Pattern.compile("[\"]second1_s[\"][:][\"](.*?)[\"]");
Matcher m = p.matcher(response);
Assert.assertTrue(m.find());
String yearStr = m.group(1);
Assert.assertTrue(response.contains("\"second1_s\":\"" + yearStr + "\""));
Assert.assertTrue(response.contains("\"second2_s\":\"" + yearStr + "\""));
Assert.assertTrue(response.contains("\"second3_s\":\"" + yearStr + "\""));
Assert.assertTrue(response.contains("\"PORK_s\":\"GRILL\""));
Assert.assertTrue(response.contains("\"FISH_s\":\"FRY\""));
Assert.assertTrue(response.contains("\"BEEF_CUTS_mult_s\":[\"ROUND\",\"SIRLOIN\"]"));
} catch(Exception e) {
throw e;
} finally {
req.close();
}
}
@Override
protected String generateConfig() {
String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault() + "'");
StringBuilder sb = new StringBuilder();
sb.append("<dataConfig> \n");
sb.append("<dataSource name=\"hsqldb\" driver=\"org.hsqldb.jdbcDriver\" url=\"jdbc:hsqldb:mem:.\" /> \n");
sb.append("<document name=\"TestEvaluators\"> \n");
sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
sb.append(" query=\"" +
"select " +
" 1 as id, " +
" 'SELECT' as SELECT_KEYWORD, " +
" CURRENT_TIMESTAMP as FIRST_TS " +
"from DUAL \" >\n");
sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
sb.append(" query=\"" +
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
" 1 as SORT, " +
" CURRENT_TIMESTAMP as SECOND_TS, " +
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
" 'PORK' AS MEAT, " +
" 'GRILL' AS METHOD, " +
" 'ROUND' AS CUTS, " +
" 'BEEF_CUTS' AS WHATKIND " +
"from DUAL " +
"WHERE 1=${FIRST.ID} " +
"UNION " +
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
" 2 as SORT, " +
" CURRENT_TIMESTAMP as SECOND_TS, " +
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
" 'FISH' AS MEAT, " +
" 'FRY' AS METHOD, " +
" 'SIRLOIN' AS CUTS, " +
" 'BEEF_CUTS' AS WHATKIND " +
"from DUAL " +
"WHERE 1=${FIRST.ID} " +
"ORDER BY SORT \"" +
">\n");
sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
sb.append(" <field column=\"SECOND1_S\" name=\"second1_s\" /> \n");
sb.append(" <field column=\"second2_s\" template=\"${dataimporter.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
sb.append(" <field column=\"second3_s\" template=\"${dih.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
sb.append(" <field column=\"METHOD\" name=\"${SECOND.MEAT}_s\"/>\n");
sb.append(" <field column=\"CUTS\" name=\"${SECOND.WHATKIND}_mult_s\"/>\n");
sb.append(" </entity>\n");
sb.append("</entity>\n");
sb.append("</document> \n");
sb.append("</dataConfig> \n");
String config = sb.toString();
log.debug(config);
return config;
}
@Override
protected void populateData(Connection conn) throws Exception {
Statement s = null;
try {
s = conn.createStatement();
s.executeUpdate("create table dual(dual char(1) not null)");
s.executeUpdate("insert into dual values('Y')");
conn.commit();
} catch (Exception e) {
throw e;
} finally {
try {
s.close();
} catch (Exception ex) {}
try {
conn.close();
} catch (Exception ex) {}
}
}
@Override
protected Database setAllowedDatabases() {
return Database.HSQLDB;
}
}