SOLR-11943: Add machine learning functions for location data

solr/core/src/java/org/apache/solr/handler/HaversineMetersEvaluator.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler;
+
+import java.io.IOException;
+
+import org.apache.commons.math3.exception.DimensionMismatchException;
+import org.apache.commons.math3.ml.distance.DistanceMeasure;
+import org.apache.lucene.util.SloppyMath;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.eval.RecursiveEvaluator;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class HaversineMetersEvaluator extends RecursiveEvaluator {
+  protected static final long serialVersionUID = 1L;
+
+  public HaversineMetersEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+    super(expression, factory);
+  }
+
+
+  @Override
+  public Object evaluate(Tuple tuple) throws IOException {
+    return new HaversineDistance();
+  }
+
+  @Override
+  public Object doWork(Object... values) throws IOException {
+    // Nothing to do here
+    throw new IOException("This call should never occur");
+  }
+
+  public static class HaversineDistance implements DistanceMeasure {
+    private static final long serialVersionUID = -9108154600539125566L;
+
+    public HaversineDistance() {
+    }
+
+    public double compute(double[] a, double[] b) throws DimensionMismatchException {
+      return SloppyMath.haversinMeters(a[0], a[1], b[0], b[1]);
+    }
+  }
+
+}

solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java

@@ -35,6 +35,7 @@ public class SolrDefaultStreamFactory extends DefaultStreamFactory {
     super();
     this.withFunctionName("analyze",  AnalyzeEvaluator.class);
     this.withFunctionName("classify", ClassifyStream.class);
+    this.withFunctionName("haversineMeters", HaversineMetersEvaluator.class);
   }
 
   public SolrDefaultStreamFactory withSolrResourceLoader(SolrResourceLoader solrResourceLoader) {

solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java

@@ -255,6 +255,7 @@ public class Lang {
         .withFunctionName("removeCache", RemoveCacheEvaluator.class)
         .withFunctionName("listCache", ListCacheEvaluator.class)
         .withFunctionName("zscores", NormalizeEvaluator.class)
+        .withFunctionName("locationVectors", LocationVectorsEvaluator.class)
 
         // Boolean Stream Evaluators
 

solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class LocationVectorsEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+  protected static final long serialVersionUID = 1L;
+
+  private String field;
+
+  public LocationVectorsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+    super(expression, factory);
+
+    List<StreamExpressionNamedParameter> namedParams = factory.getNamedOperands(expression);
+
+    for (StreamExpressionNamedParameter namedParam : namedParams) {
+      if(namedParam.getName().equals("field")) {
+        this.field = namedParam.getParameter().toString();
+      } else {
+        throw new IOException("Unexpected named parameter:" + namedParam.getName());
+      }
+    }
+
+    if(field == null) {
+      throw new IOException("The named parameter \"field\" must be set for the locationVectors function.");
+    }
+  }
+
+  @Override
+  public Object doWork(Object... objects) throws IOException {
+
+    if (objects.length == 1) {
+      //Just docs
+      if(!(objects[0] instanceof List)) {
+        throw new IOException("The locationVectors function expects a list of Tuples as a parameter.");
+      } else {
+        List list = (List)objects[0];
+        if(list.size() > 0) {
+          Object o = list.get(0);
+          if(!(o instanceof Tuple)) {
+            throw new IOException("The locationVectors function expects a list of Tuples as a parameter.");
+          }
+        } else {
+          throw new IOException("Empty list was passed as a parameter to termVectors function.");
+        }
+      }
+
+      List<Tuple> tuples = (List<Tuple>) objects[0];
+
+      double[][] locationVectors = new double[tuples.size()][2];
+      List<String> features = new ArrayList();
+      features.add("lat");
+      features.add("long");
+
+      List<String> rowLabels = new ArrayList();
+
+      for(int i=0; i< tuples.size(); i++) {
+        Tuple tuple = tuples.get(i);
+        String value = tuple.getString(field);
+        String[] latLong = null;
+        if(value.contains(",")) {
+          latLong = value.split(",");
+        } else {
+          latLong = value.split(" ");
+        }
+
+        locationVectors[i][0] = Double.parseDouble(latLong[0].trim());
+        locationVectors[i][1] = Double.parseDouble(latLong[1].trim());
+        if(tuple.get("id") != null) {
+          rowLabels.add(tuple.get("id").toString());
+        }
+      }
+
+      Matrix matrix = new Matrix(locationVectors);
+      matrix.setColumnLabels(features);
+      matrix.setRowLabels(rowLabels);
+      return matrix;
+    } else {
+      throw new IOException("The termVectors function takes a single positional parameter.");
+    }
+  }
+}

solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java

@@ -70,7 +70,7 @@ public class TestLang extends LuceneTestCase {
       "mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
       "cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
       "earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress", "gaussfit",
-      "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores"};
+      "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "locationVectors"};
 
   @Test
   public void testLang() {

solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java

@@ -310,6 +310,52 @@ public class MathExpressionTest extends SolrCloudTestCase {
     assertEquals(array.get(2).intValue(), 50);
     assertEquals(array.get(3).intValue(), 50);
   }
+
+  @Test
+  public void testLocationFunctions() throws Exception {
+    UpdateRequest updateRequest = new UpdateRequest();
+
+    int i=0;
+    while(i<5) {
+      updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"),
+          "price_i",  Integer.toString(i), "loc_p", (42.906797030808235+i)+","+(76.69455762489834+i));
+    }
+
+
+    updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+    String expr = "let(echo=true," +
+        "              a=search("+COLLECTIONORALIAS+", q=*:*, fl=\"id, loc_p, price_i\",rows=100, sort=\"price_i asc\"),"+
+        "              b=locationVectors(a, field=loc_p)," +
+        "              c=distance(array(40.7128, 74.0060), array(45.7128, 74.0060), haversineMeters()))";
+
+
+    ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+    paramsLoc.set("expr", expr);
+    paramsLoc.set("qt", "/stream");
+
+    String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+    TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+    StreamContext context = new StreamContext();
+    solrStream.setStreamContext(context);
+    List<Tuple> tuples = getTuples(solrStream);
+    assertTrue(tuples.size() == 1);
+    List<List<Number>>locVectors = (List<List<Number>>)tuples.get(0).get("b");
+    System.out.println(locVectors);
+    int v=1;
+    for(List<Number> row : locVectors) {
+     double lat = row.get(0).doubleValue();
+     double lon = row.get(1).doubleValue();
+     assertEquals(lat, 42.906797030808235+v, 0);
+     assertEquals(lon, 76.69455762489834+v, 0);
+     ++v;
+    }
+
+    double distance = tuples.get(0).getDouble("c");
+    assertEquals(distance, 555975.3986718428, 1.0);
+
+  }
   
   @Test
   public void testHist() throws Exception {