mirror of https://github.com/apache/lucene.git
SOLR-2141 / SOLR-4047 / SOLR-3842 - fix problems with VariableResolver, better test coverage
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1414242 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
aa0958f735
commit
9c1d89fe1b
|
@ -67,6 +67,15 @@ public class DateFormatEvaluator extends Evaluator {
|
|||
availableLocales.put(locale.toString(), locale);
|
||||
}
|
||||
}
|
||||
private SimpleDateFormat getDateFormat(String pattern, Locale locale) {
|
||||
DateFormatCacheKey dfck = new DateFormatCacheKey(locale, pattern);
|
||||
SimpleDateFormat sdf = cache.get(dfck);
|
||||
if(sdf == null) {
|
||||
sdf = new SimpleDateFormat(pattern, locale);
|
||||
cache.put(dfck, sdf);
|
||||
}
|
||||
return sdf;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
|
@ -81,15 +90,13 @@ public class DateFormatEvaluator extends Evaluator {
|
|||
VariableWrapper wrapper = (VariableWrapper) format;
|
||||
o = wrapper.resolve();
|
||||
format = o.toString();
|
||||
}
|
||||
}
|
||||
Locale locale = Locale.ROOT;
|
||||
if(l.size()==3) {
|
||||
Object localeObj = l.get(2);
|
||||
String localeStr = null;
|
||||
if (localeObj instanceof VariableWrapper) {
|
||||
VariableWrapper wrapper = (VariableWrapper) localeObj;
|
||||
o = wrapper.resolve();
|
||||
localeStr = o.toString();
|
||||
localeStr = ((VariableWrapper) localeObj).resolve().toString();
|
||||
} else {
|
||||
localeStr = localeObj.toString();
|
||||
}
|
||||
|
@ -97,14 +104,9 @@ public class DateFormatEvaluator extends Evaluator {
|
|||
if(locale==null) {
|
||||
throw new DataImportHandlerException(SEVERE, "Unsupported locale: " + localeStr);
|
||||
}
|
||||
}
|
||||
}
|
||||
String dateFmt = format.toString();
|
||||
DateFormatCacheKey dfck = new DateFormatCacheKey(locale, dateFmt);
|
||||
SimpleDateFormat sdf = cache.get(dfck);
|
||||
if(sdf==null) {
|
||||
sdf = new SimpleDateFormat(dateFmt, locale);
|
||||
cache.put(dfck, sdf);
|
||||
}
|
||||
SimpleDateFormat fmt = getDateFormat(dateFmt, locale);
|
||||
Date date = null;
|
||||
if (o instanceof VariableWrapper) {
|
||||
VariableWrapper variableWrapper = (VariableWrapper) o;
|
||||
|
@ -114,13 +116,7 @@ public class DateFormatEvaluator extends Evaluator {
|
|||
} else {
|
||||
String s = variableval.toString();
|
||||
try {
|
||||
dfck = new DateFormatCacheKey(locale, DEFAULT_DATE_FORMAT);
|
||||
sdf = cache.get(dfck);
|
||||
if(sdf==null) {
|
||||
sdf = new SimpleDateFormat(dfck.dateFormat, dfck.locale);
|
||||
cache.put(dfck, sdf);
|
||||
}
|
||||
date = new SimpleDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
|
||||
date = getDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
|
||||
} catch (ParseException exp) {
|
||||
wrapAndThrow(SEVERE, exp, "Invalid expression for date");
|
||||
}
|
||||
|
@ -134,7 +130,7 @@ public class DateFormatEvaluator extends Evaluator {
|
|||
wrapAndThrow(SEVERE, e, "Invalid expression for date");
|
||||
}
|
||||
}
|
||||
return sdf.format(date);
|
||||
return fmt.format(date);
|
||||
}
|
||||
static DateMathParser getDateMathParser(Locale l) {
|
||||
return new DateMathParser(TimeZone.getDefault(), l) {
|
||||
|
|
|
@ -68,7 +68,6 @@ public class DocBuilder {
|
|||
Map<String, Object> session = new HashMap<String, Object>();
|
||||
|
||||
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
|
||||
//private Map<String, Object> functionsNamespace;
|
||||
private Map<String, Object> persistedProperties;
|
||||
|
||||
private DIHProperties propWriter;
|
||||
|
@ -640,11 +639,20 @@ public class DocBuilder {
|
|||
if (field != null) {
|
||||
for (EntityField f : field) {
|
||||
String name = f.getName();
|
||||
boolean multiValued = f.isMultiValued();
|
||||
boolean toWrite = f.isToWrite();
|
||||
if(f.isDynamicName()){
|
||||
name = vr.replaceTokens(name);
|
||||
SchemaField schemaField = dataImporter.getSchemaField(name);
|
||||
if(schemaField == null) {
|
||||
toWrite = false;
|
||||
} else {
|
||||
multiValued = schemaField.multiValued();
|
||||
toWrite = true;
|
||||
}
|
||||
}
|
||||
if (f.isToWrite()) {
|
||||
addFieldToDoc(entry.getValue(), name, f.getBoost(), f.isMultiValued(), doc);
|
||||
if (toWrite) {
|
||||
addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -63,6 +63,7 @@ public class VariableResolver {
|
|||
}
|
||||
|
||||
public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
|
||||
public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
|
||||
|
||||
public VariableResolver() {
|
||||
rootNamespace = new HashMap<String,Object>();
|
||||
|
@ -95,7 +96,11 @@ public class VariableResolver {
|
|||
r = currentLevel.get(nameParts[nameParts.length - 1]);
|
||||
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
|
||||
&& name.length() > FUNCTIONS_NAMESPACE.length()) {
|
||||
return resolveEvaluator(name);
|
||||
return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
|
||||
}
|
||||
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
|
||||
&& name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
|
||||
return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
|
||||
}
|
||||
if (r == null) {
|
||||
r = System.getProperty(name);
|
||||
|
@ -104,12 +109,12 @@ public class VariableResolver {
|
|||
return r == null ? "" : r;
|
||||
}
|
||||
|
||||
private Object resolveEvaluator(String name) {
|
||||
private Object resolveEvaluator(String namespace, String name) {
|
||||
if (evaluators == null) {
|
||||
return "";
|
||||
}
|
||||
Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
|
||||
.substring(FUNCTIONS_NAMESPACE.length()));
|
||||
.substring(namespace.length()));
|
||||
if (m.find()) {
|
||||
String fname = m.group(1);
|
||||
Evaluator evaluator = evaluators.get(fname);
|
||||
|
|
|
@ -24,7 +24,7 @@ import java.text.SimpleDateFormat;
|
|||
import java.util.*;
|
||||
|
||||
/**
|
||||
* <p> Test for EvaluatorBag </p>
|
||||
* <p> Test for Evaluators </p>
|
||||
*
|
||||
*
|
||||
* @since solr 1.3
|
||||
|
|
|
@ -0,0 +1,137 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Statement;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import junit.framework.Assert;
|
||||
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.junit.Test;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
h.query("/dataimport", generateRequest());
|
||||
SolrQueryRequest req = null;
|
||||
try {
|
||||
req = req("q", "*:*", "wt", "json", "indent", "true");
|
||||
String response = h.query(req);
|
||||
log.debug(response);
|
||||
response = response.replaceAll("\\s","");
|
||||
Assert.assertTrue(response.contains("\"numFound\":1"));
|
||||
Pattern p = Pattern.compile("[\"]second1_s[\"][:][\"](.*?)[\"]");
|
||||
Matcher m = p.matcher(response);
|
||||
Assert.assertTrue(m.find());
|
||||
String yearStr = m.group(1);
|
||||
Assert.assertTrue(response.contains("\"second1_s\":\"" + yearStr + "\""));
|
||||
Assert.assertTrue(response.contains("\"second2_s\":\"" + yearStr + "\""));
|
||||
Assert.assertTrue(response.contains("\"second3_s\":\"" + yearStr + "\""));
|
||||
Assert.assertTrue(response.contains("\"PORK_s\":\"GRILL\""));
|
||||
Assert.assertTrue(response.contains("\"FISH_s\":\"FRY\""));
|
||||
Assert.assertTrue(response.contains("\"BEEF_CUTS_mult_s\":[\"ROUND\",\"SIRLOIN\"]"));
|
||||
} catch(Exception e) {
|
||||
throw e;
|
||||
} finally {
|
||||
req.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String generateConfig() {
|
||||
String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault() + "'");
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("<dataConfig> \n");
|
||||
sb.append("<dataSource name=\"hsqldb\" driver=\"org.hsqldb.jdbcDriver\" url=\"jdbc:hsqldb:mem:.\" /> \n");
|
||||
sb.append("<document name=\"TestEvaluators\"> \n");
|
||||
sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
|
||||
sb.append(" query=\"" +
|
||||
"select " +
|
||||
" 1 as id, " +
|
||||
" 'SELECT' as SELECT_KEYWORD, " +
|
||||
" CURRENT_TIMESTAMP as FIRST_TS " +
|
||||
"from DUAL \" >\n");
|
||||
sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
|
||||
sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
|
||||
sb.append(" query=\"" +
|
||||
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
|
||||
" 1 as SORT, " +
|
||||
" CURRENT_TIMESTAMP as SECOND_TS, " +
|
||||
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
|
||||
" 'PORK' AS MEAT, " +
|
||||
" 'GRILL' AS METHOD, " +
|
||||
" 'ROUND' AS CUTS, " +
|
||||
" 'BEEF_CUTS' AS WHATKIND " +
|
||||
"from DUAL " +
|
||||
"WHERE 1=${FIRST.ID} " +
|
||||
"UNION " +
|
||||
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
|
||||
" 2 as SORT, " +
|
||||
" CURRENT_TIMESTAMP as SECOND_TS, " +
|
||||
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
|
||||
" 'FISH' AS MEAT, " +
|
||||
" 'FRY' AS METHOD, " +
|
||||
" 'SIRLOIN' AS CUTS, " +
|
||||
" 'BEEF_CUTS' AS WHATKIND " +
|
||||
"from DUAL " +
|
||||
"WHERE 1=${FIRST.ID} " +
|
||||
"ORDER BY SORT \"" +
|
||||
">\n");
|
||||
sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
|
||||
sb.append(" <field column=\"SECOND1_S\" name=\"second1_s\" /> \n");
|
||||
sb.append(" <field column=\"second2_s\" template=\"${dataimporter.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
|
||||
sb.append(" <field column=\"second3_s\" template=\"${dih.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
|
||||
sb.append(" <field column=\"METHOD\" name=\"${SECOND.MEAT}_s\"/>\n");
|
||||
sb.append(" <field column=\"CUTS\" name=\"${SECOND.WHATKIND}_mult_s\"/>\n");
|
||||
sb.append(" </entity>\n");
|
||||
sb.append("</entity>\n");
|
||||
sb.append("</document> \n");
|
||||
sb.append("</dataConfig> \n");
|
||||
String config = sb.toString();
|
||||
log.debug(config);
|
||||
return config;
|
||||
}
|
||||
@Override
|
||||
protected void populateData(Connection conn) throws Exception {
|
||||
Statement s = null;
|
||||
try {
|
||||
s = conn.createStatement();
|
||||
s.executeUpdate("create table dual(dual char(1) not null)");
|
||||
s.executeUpdate("insert into dual values('Y')");
|
||||
conn.commit();
|
||||
} catch (Exception e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try {
|
||||
s.close();
|
||||
} catch (Exception ex) {}
|
||||
try {
|
||||
conn.close();
|
||||
} catch (Exception ex) {}
|
||||
}
|
||||
}
|
||||
@Override
|
||||
protected Database setAllowedDatabases() {
|
||||
return Database.HSQLDB;
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue