From a0bb5017722ce698fc390f3990243697341d2b8d Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 11 Oct 2018 10:43:30 -0400 Subject: [PATCH] SOLR-12851: Improvements and fixes to let and select Streaming Expressions --- .../client/solrj/io/stream/LetStream.java | 11 ++++- .../client/solrj/io/stream/SelectStream.java | 11 +++-- .../solrj/io/stream/expr/StreamFactory.java | 5 +-- .../solrj/io/stream/MathExpressionTest.java | 41 ++++++++++++++++++- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java index e88eaf6e2b7..23881c3a446 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java @@ -36,6 +36,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation; import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; /** @@ -76,7 +77,11 @@ public class LetStream extends TupleStream implements Expressible { } StreamExpressionParameter param = ((StreamExpressionNamedParameter)np).getParameter(); - if(factory.isEvaluator((StreamExpression)param)) { + + if(param instanceof StreamExpressionValue) { + String paramValue = ((StreamExpressionValue) param).getValue(); + letParams.put(name, factory.constructPrimitiveObject(paramValue)); + } else if(factory.isEvaluator((StreamExpression)param)) { StreamEvaluator evaluator = factory.constructEvaluator((StreamExpression) param); letParams.put(name, evaluator); } else { @@ -182,7 +187,7 @@ public class LetStream extends TupleStream implements Expressible { } finally { tStream.close(); } - } else { + } else if(o instanceof StreamEvaluator) { //Add the data from the StreamContext to a tuple. //Let the evaluator works from this tuple. //This will allow columns to be created from tuples already in the StreamContext. @@ -196,6 +201,8 @@ public class LetStream extends TupleStream implements Expressible { } else { lets.put(name, eo); } + } else { + lets.put(name, o); } } stream.open(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java index d6664cd8457..d87a637e63e 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java @@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.stream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -63,14 +64,14 @@ public class SelectStream extends TupleStream implements Expressible { this.selectedFields.put(selectedField, selectedField); } operations = new ArrayList<>(); - selectedEvaluators = new HashMap<>(); + selectedEvaluators = new LinkedHashMap(); } public SelectStream(TupleStream stream, Map selectedFields) throws IOException { this.stream = stream; this.selectedFields = selectedFields; operations = new ArrayList<>(); - selectedEvaluators = new HashMap<>(); + selectedEvaluators = new LinkedHashMap(); } public SelectStream(StreamExpression expression,StreamFactory factory) throws IOException { @@ -100,7 +101,7 @@ public class SelectStream extends TupleStream implements Expressible { stream = factory.constructStream(streamExpressions.get(0)); selectedFields = new HashMap(); - selectedEvaluators = new HashMap(); + selectedEvaluators = new LinkedHashMap(); for(StreamExpressionParameter parameter : selectAsFieldsExpressions){ StreamExpressionValue selectField = (StreamExpressionValue)parameter; String value = selectField.getValue().trim(); @@ -281,7 +282,9 @@ public class SelectStream extends TupleStream implements Expressible { // Apply all evaluators for(Map.Entry selectedEvaluator : selectedEvaluators.entrySet()) { - workingToReturn.put(selectedEvaluator.getValue(), selectedEvaluator.getKey().evaluate(workingForEvaluators)); + Object o = selectedEvaluator.getKey().evaluate(workingForEvaluators); + workingForEvaluators.put(selectedEvaluator.getValue(), o); + workingToReturn.put(selectedEvaluator.getValue(), o); } return workingToReturn; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/StreamFactory.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/StreamFactory.java index 321545bd77a..0ba5d5c3e42 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/StreamFactory.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/StreamFactory.java @@ -439,9 +439,8 @@ public class StreamFactory implements Serializable { if("null".equals(lower)){ return null; } if("true".equals(lower) || "false".equals(lower)){ return Boolean.parseBoolean(lower); } try{ return Long.valueOf(original); } catch(Exception ignored){}; - try{ if (original.matches(".{1,8}")){ return Double.valueOf(original); }} catch(Exception ignored){}; - try{ if (original.matches(".{1,17}")){ return Double.valueOf(original); }} catch(Exception ignored){}; - + try{ return Double.valueOf(original); } catch(Exception ignored){}; + // is a string return original; } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java index a095dd89076..45633e3db4a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java @@ -1169,7 +1169,6 @@ public class MathExpressionTest extends SolrCloudTestCase { List tuples = getTuples(solrStream); assertTrue(tuples.size() == 1); List> out = (List>)tuples.get(0).get("c"); - System.out.println("###### out:"+out); assertEquals(out.size(), 2); List row1 = out.get(0); assertEquals(row1.get(0).doubleValue(), 2.1, 0); @@ -1721,6 +1720,46 @@ public class MathExpressionTest extends SolrCloudTestCase { assertEquals(sd, 3.5, 0.0); } + @Test + public void testSelectWithSequentialEvaluators() throws Exception { + String cexpr = "select(list(tuple(a=add(1,2)), tuple(a=add(2,2))), " + + " add(1, a) as blah, " + + " add(1, blah) as blah1)"; + ModifiableSolrParams paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", cexpr); + paramsLoc.set("qt", "/stream"); + String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS; + TupleStream solrStream = new SolrStream(url, paramsLoc); + StreamContext context = new StreamContext(); + solrStream.setStreamContext(context); + List tuples = getTuples(solrStream); + assertEquals(tuples.size(), 2); + Tuple tuple0 = tuples.get(0); + assertEquals(tuple0.getLong("blah").longValue(), 4L); + assertEquals(tuple0.getLong("blah1").longValue(), 5L); + + Tuple tuple1 = tuples.get(1); + assertEquals(tuple1.getLong("blah").longValue(), 5L); + assertEquals(tuple1.getLong("blah1").longValue(), 6L); + } + + @Test + public void testLetWithNumericVariables() throws Exception { + String cexpr = "let(echo=true, a=1.88888, b=8888888888.98)"; + ModifiableSolrParams paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", cexpr); + paramsLoc.set("qt", "/stream"); + String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS; + TupleStream solrStream = new SolrStream(url, paramsLoc); + StreamContext context = new StreamContext(); + solrStream.setStreamContext(context); + List tuples = getTuples(solrStream); + assertEquals(tuples.size(), 1); + Tuple tuple = tuples.get(0); + assertEquals(tuple.getDouble("a").doubleValue(), 1.88888, 0.0); + assertEquals(tuple.getDouble("b").doubleValue(), 8888888888.98, 0.0); + } + @Test public void testTermVectors() throws Exception { // Test termVectors with only documents and default termVector settings