SOLR-10486: Add Length Conversion Evaluators

This commit is contained in:
Joel Bernstein 2017-04-17 10:10:05 -04:00
parent 5169ecb976
commit 8417307c63
6 changed files with 431 additions and 50 deletions

View File

@ -33,41 +33,7 @@ import org.apache.solr.client.solrj.io.ModelCache;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.eval.AbsoluteValueEvaluator;
import org.apache.solr.client.solrj.io.eval.AddEvaluator;
import org.apache.solr.client.solrj.io.eval.AndEvaluator;
import org.apache.solr.client.solrj.io.eval.ArcCosineEvaluator;
import org.apache.solr.client.solrj.io.eval.ArcSineEvaluator;
import org.apache.solr.client.solrj.io.eval.ArcTangentEvaluator;
import org.apache.solr.client.solrj.io.eval.CeilingEvaluator;
import org.apache.solr.client.solrj.io.eval.CoalesceEvaluator;
import org.apache.solr.client.solrj.io.eval.CosineEvaluator;
import org.apache.solr.client.solrj.io.eval.CubedRootEvaluator;
import org.apache.solr.client.solrj.io.eval.DivideEvaluator;
import org.apache.solr.client.solrj.io.eval.EqualsEvaluator;
import org.apache.solr.client.solrj.io.eval.ExclusiveOrEvaluator;
import org.apache.solr.client.solrj.io.eval.FloorEvaluator;
import org.apache.solr.client.solrj.io.eval.GreaterThanEqualToEvaluator;
import org.apache.solr.client.solrj.io.eval.GreaterThanEvaluator;
import org.apache.solr.client.solrj.io.eval.HyperbolicCosineEvaluator;
import org.apache.solr.client.solrj.io.eval.HyperbolicSineEvaluator;
import org.apache.solr.client.solrj.io.eval.HyperbolicTangentEvaluator;
import org.apache.solr.client.solrj.io.eval.IfThenElseEvaluator;
import org.apache.solr.client.solrj.io.eval.LessThanEqualToEvaluator;
import org.apache.solr.client.solrj.io.eval.LessThanEvaluator;
import org.apache.solr.client.solrj.io.eval.ModuloEvaluator;
import org.apache.solr.client.solrj.io.eval.MultiplyEvaluator;
import org.apache.solr.client.solrj.io.eval.NaturalLogEvaluator;
import org.apache.solr.client.solrj.io.eval.NotEvaluator;
import org.apache.solr.client.solrj.io.eval.OrEvaluator;
import org.apache.solr.client.solrj.io.eval.PowerEvaluator;
import org.apache.solr.client.solrj.io.eval.RawValueEvaluator;
import org.apache.solr.client.solrj.io.eval.RoundEvaluator;
import org.apache.solr.client.solrj.io.eval.SineEvaluator;
import org.apache.solr.client.solrj.io.eval.SquareRootEvaluator;
import org.apache.solr.client.solrj.io.eval.SubtractEvaluator;
import org.apache.solr.client.solrj.io.eval.TangentEvaluator;
import org.apache.solr.client.solrj.io.eval.UuidEvaluator;
import org.apache.solr.client.solrj.io.eval.*;
import org.apache.solr.client.solrj.io.graph.GatherNodesStream;
import org.apache.solr.client.solrj.io.graph.ShortestPathStream;
import org.apache.solr.client.solrj.io.ops.ConcatOperation;
@ -85,17 +51,6 @@ import org.apache.solr.client.solrj.io.stream.metrics.MaxMetric;
import org.apache.solr.client.solrj.io.stream.metrics.MeanMetric;
import org.apache.solr.client.solrj.io.stream.metrics.MinMetric;
import org.apache.solr.client.solrj.io.stream.metrics.SumMetric;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDay;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfQuarter;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfYear;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorEpoch;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorHour;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMinute;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMonth;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorQuarter;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorSecond;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorWeek;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorYear;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -278,6 +233,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
// Conditional Stream Evaluators
.withFunctionName("if", IfThenElseEvaluator.class)
.withFunctionName("analyze", AnalyzeEvaluator.class)
.withFunctionName("convert", ConversionEvaluator.class)
;
// This pulls all the overrides and additions from the config

View File

@ -0,0 +1,166 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
import java.util.Locale;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
public class ConversionEvaluator extends ComplexEvaluator {
enum LENGTH_CONSTANT {MILES, YARDS, FEET, INCHES, MILLIMETERS, CENTIMETERS, METERS, KILOMETERS};
private LENGTH_CONSTANT from;
private LENGTH_CONSTANT to;
private Convert convert;
public ConversionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
super(expression, factory);
if (3 != subEvaluators.size()) {
throw new EvaluatorException(new IOException(String.format(Locale.ROOT, "Invalid expression %s - expecting 3 value but found %d", expression, subEvaluators.size())));
}
try {
from = LENGTH_CONSTANT.valueOf(subEvaluators.get(0).toExpression(factory).toString().toUpperCase(Locale.ROOT));
to = LENGTH_CONSTANT.valueOf(subEvaluators.get(1).toExpression(factory).toString().toUpperCase(Locale.ROOT));
this.convert = getConvert(from, to);
} catch (IllegalArgumentException e) {
throw new EvaluatorException(e);
}
}
private String listParams() {
StringBuffer buf = new StringBuffer();
for(LENGTH_CONSTANT lc : LENGTH_CONSTANT.values()) {
if(buf.length() > 0) {
buf.append(", ");
}
buf.append(lc.toString());
}
return buf.toString();
}
@Override
public Object evaluate(Tuple tuple) throws IOException {
StreamEvaluator streamEvaluator = subEvaluators.get(2);
Object tupleValue = streamEvaluator.evaluate(tuple);
if (tupleValue == null) return null;
Number number = (Number)tupleValue;
double d = number.doubleValue();
return convert.convert(d);
}
private Convert getConvert(LENGTH_CONSTANT from, LENGTH_CONSTANT to) throws IOException {
switch(from) {
case INCHES:
switch(to) {
case MILLIMETERS:
return (double d) -> d*25.4;
case CENTIMETERS:
return (double d) -> d*2.54;
case METERS:
return (double d) -> d*0.0254;
default:
throw new EvaluatorException("No conversion available from "+from+" to "+to);
}
case FEET:
switch(to) {
case METERS:
return (double d) -> d * .30;
}
case YARDS:
switch(to) {
case METERS:
return (double d) -> d * .91;
case KILOMETERS:
return (double d) -> d * 0.00091;
default:
throw new EvaluatorException("No conversion available from "+from+" to "+to);
}
case MILES:
switch(to) {
case KILOMETERS:
return (double d) -> d * 1.61;
default:
throw new EvaluatorException("No conversion available from "+from+" to "+to);
}
case MILLIMETERS:
switch (to) {
case INCHES:
return (double d) -> d * 0.039;
default:
throw new EvaluatorException("No conversion available from "+from+" to "+to);
}
case CENTIMETERS:
switch(to) {
case INCHES:
return (double d) -> d * 0.39;
default:
throw new EvaluatorException("No conversion available from "+from+" to "+to);
}
case METERS:
switch(to) {
case FEET:
return (double d) -> d * 3.28;
default:
throw new EvaluatorException("No conversion available from "+from+" to "+to);
}
case KILOMETERS:
switch(to) {
case MILES:
return (double d) -> d * 0.62;
case FEET:
return (double d) -> d * 3280.8;
}
default:
throw new EvaluatorException("No conversion available from "+from);
}
}
private interface Convert {
public double convert(double d);
}
@Override
public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
for (StreamEvaluator evaluator : subEvaluators) {
expression.addParameter(evaluator.toExpression(factory));
}
return expression;
}
@Override
public Explanation toExplanation(StreamFactory factory) throws IOException {
return new Explanation(nodeId.toString())
.withExpressionType(Explanation.ExpressionType.EVALUATOR)
.withImplementingClass(getClass().getName())
.withExpression(toExpression(factory).toString());
}
}

View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
public class EvaluatorException extends IOException {
public EvaluatorException(Throwable t) {
super(t);
}
public EvaluatorException(String message) {
super(message);
}
}

View File

@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.eval.EvaluatorException;
import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
import org.apache.solr.client.solrj.io.ops.StreamOperation;
import org.apache.solr.client.solrj.io.stream.expr.Explanation;
@ -125,8 +126,17 @@ public class SelectStream extends TupleStream implements Expressible {
selectedEvaluators.put(factory.constructEvaluator(asValueExpression), asName);
handled = true;
}
}
catch(Throwable e){
} catch(Throwable e) {
Throwable t = e;
while(true) {
if(t instanceof EvaluatorException) {
throw new IOException(t);
}
t = t.getCause();
if(t == null) {
break;
}
}
// it was not handled, so treat as a non-evaluator
}
}

View File

@ -4847,7 +4847,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(t.getString("test_t").equals("c"));
assertTrue(t.getString("id").equals("1"));
//Try with single param
expr = "cartesianProduct(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id, test_t\", sort=\"id desc\"), analyze(test_t) as test_t)";
paramsLoc = new ModifiableSolrParams();
@ -4883,7 +4882,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(t.getString("test_t").equals("c"));
assertTrue(t.getString("id").equals("1"));
//Try with null in the test_t field
expr = "cartesianProduct(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id\", sort=\"id desc\"), analyze(test_t, test_t) as test_t)";
paramsLoc = new ModifiableSolrParams();
@ -4920,6 +4918,73 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
@Test
public void testConvertEvaluator() throws Exception {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.add(id, "1", "miles_i", "50");
updateRequest.add(id, "2", "miles_i", "70");
updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
//Test annotating tuple
String expr = "select(eval(), convert(miles, kilometers, 10) as kilometers)";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", expr);
paramsLoc.set("qt", "/stream");
String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
TupleStream solrStream = new SolrStream(url, paramsLoc);
StreamContext context = new StreamContext();
solrStream.setStreamContext(context);
List<Tuple> tuples = getTuples(solrStream);
assertTrue(tuples.size() == 1);
double d = (double)tuples.get(0).get("kilometers");
assertTrue(d == (double)(10*1.61));
expr = "select(search("+COLLECTIONORALIAS+", q=\"*:*\", sort=\"miles_i asc\", fl=\"miles_i\"), convert(miles, kilometers, miles_i) as kilometers)";
paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", expr);
paramsLoc.set("qt", "/stream");
solrStream = new SolrStream(url, paramsLoc);
context = new StreamContext();
solrStream.setStreamContext(context);
tuples = getTuples(solrStream);
assertTrue(tuples.size() == 2);
d = (double)tuples.get(0).get("kilometers");
assertTrue(d == (double)(50*1.61));
d = (double)tuples.get(1).get("kilometers");
assertTrue(d == (double)(70*1.61));
expr = "parallel("+COLLECTIONORALIAS+", workers=2, sort=\"miles_i asc\", select(search("+COLLECTIONORALIAS+", q=\"*:*\", partitionKeys=miles_i, sort=\"miles_i asc\", fl=\"miles_i\"), convert(miles, kilometers, miles_i) as kilometers))";
paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", expr);
paramsLoc.set("qt", "/stream");
solrStream = new SolrStream(url, paramsLoc);
context = new StreamContext();
solrStream.setStreamContext(context);
tuples = getTuples(solrStream);
assertTrue(tuples.size() == 2);
d = (double)tuples.get(0).get("kilometers");
assertTrue(d == (double)(50*1.61));
d = (double)tuples.get(1).get("kilometers");
assertTrue(d == (double)(70*1.61));
expr = "select(stats("+COLLECTIONORALIAS+", q=\"*:*\", sum(miles_i)), convert(miles, kilometers, sum(miles_i)) as kilometers)";
paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", expr);
paramsLoc.set("qt", "/stream");
solrStream = new SolrStream(url, paramsLoc);
context = new StreamContext();
solrStream.setStreamContext(context);
tuples = getTuples(solrStream);
assertTrue(tuples.size() == 1);
d = (double)tuples.get(0).get("kilometers");
assertTrue(d == (double)(120*1.61));
}
@Test
public void testExecutorStream() throws Exception {

View File

@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.io.stream.eval;
import java.io.IOException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.MonthDay;
import java.time.YearMonth;
import java.time.ZoneOffset;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import org.apache.commons.collections.map.HashedMap;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.eval.ConversionEvaluator;
import org.apache.solr.client.solrj.io.eval.RawValueEvaluator;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDay;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfQuarter;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorDayOfYear;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorEpoch;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorHour;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMinute;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorMonth;
import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorQuarter;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorSecond;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorWeek;
import org.apache.solr.client.solrj.io.eval.TemporalEvaluatorYear;
import org.apache.solr.client.solrj.io.stream.StreamContext;
import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
import org.junit.Test;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
/**
* Test ConversionEvaluators
*/
public class ConversionEvaluatorsTest {
StreamFactory factory;
Map<String, Object> values;
public ConversionEvaluatorsTest() {
super();
factory = new StreamFactory();
factory.withFunctionName("convert", ConversionEvaluator.class).withFunctionName("raw", RawValueEvaluator.class);
values = new HashedMap();
}
@Test
public void testInvalidExpression() throws Exception {
StreamEvaluator evaluator;
try {
evaluator = factory.constructEvaluator("convert(inches)");
StreamContext streamContext = new StreamContext();
evaluator.setStreamContext(streamContext);
assertTrue(false);
} catch (IOException e) {
assertTrue(e.getCause().getCause().getMessage().contains("Invalid expression convert(inches) - expecting 3 value but found 1"));
}
try {
evaluator = factory.constructEvaluator("convert(inches, yards, 3)");
StreamContext streamContext = new StreamContext();
evaluator.setStreamContext(streamContext);
Tuple tuple = new Tuple(new HashMap());
evaluator.evaluate(tuple);
assertTrue(false);
} catch (IOException e) {
assertTrue(e.getCause().getCause().getMessage().contains("No conversion available from INCHES to YARDS"));
}
}
@Test
public void testInches() throws Exception {
testFunction("convert(inches, centimeters, 2)", (double)(2*2.54));
testFunction("convert(inches, meters, 2)", (double)(2*0.0254));
testFunction("convert(inches, millimeters, 2)", (double)(2*25.40));
}
@Test
public void testYards() throws Exception {
testFunction("convert(yards, meters, 2)", (double)(2*.91));
testFunction("convert(yards, kilometers, 2)", (double)(2*.00091));
}
@Test
public void testMiles() throws Exception {
testFunction("convert(miles, kilometers, 2)", (double)(2*1.61));
}
@Test
public void testMillimeters() throws Exception {
testFunction("convert(millimeters, inches, 2)", (double)(2*.039));
}
@Test
public void testCentimeters() throws Exception {
testFunction("convert(centimeters, inches, 2)", (double)(2*.39));
}
@Test
public void testMeters() throws Exception {
testFunction("convert(meters, feet, 2)", (double)(2*3.28));
}
@Test
public void testKiloMeters() throws Exception {
testFunction("convert(kilometers, feet, 2)", (double)(2*3280.8));
testFunction("convert(kilometers, miles, 2)", (double)(2*.62));
}
public void testFunction(String expression, Number expected) throws Exception {
StreamEvaluator evaluator = factory.constructEvaluator(expression);
StreamContext streamContext = new StreamContext();
evaluator.setStreamContext(streamContext);
Object result = evaluator.evaluate(new Tuple(values));
assertTrue(result instanceof Number);
assertEquals(expected, result);
}
}