SQL: Add remaining matrix aggregations (elastic/x-pack-elasticsearch#3330)

* Add remaining matrix aggregations

This adds the remaining [matrix aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-matrix-stats-aggregation.html).

These aggregations aren't currently implemented due to the inter-plugin
communication not being set up, so they return "innerkey/matrix stats not
handled (yet)".

For matrix aggs that share a name with an existing aggregation (like 'count'),
they have be prefixed with "matrix_", so, "matrix_count", "matrix_mean", and
"matrix_variance".

Relates to elastic/x-pack-elasticsearch#2885

* Return HTTP 400 for innerkey/matrix stats aggs

* Add integration test for unimplemented matrix aggs

Original commit: elastic/x-pack-elasticsearch@34459c59aa

qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java

@@ -335,6 +335,16 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
         assertEquals(0, getNumberOfSearchContexts("test"));
     }
 
+    public void testSelectUnimplementedMatrixAggs() throws IOException {
+        StringBuilder bulk = new StringBuilder();
+        bulk.append("{\"index\":{\"_id\":\"1\"}}\n");
+        bulk.append("{\"foo\":1}\n");
+        client().performRequest("POST", "/test/doc/_bulk", singletonMap("refresh", "true"),
+                new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON));
+        expectBadRequest(() -> runSql("SELECT covariance(foo) FROM test"), containsString("innerkey/matrix stats not handled (yet)"));
+    }
+
+
     private Tuple<String, String> runSqlAsText(String sql) throws IOException {
         return runSqlAsText("", new StringEntity("{\"query\":\"" + sql + "\"}", ContentType.APPLICATION_JSON));
     }

qa/sql/src/main/resources/command.csv-spec

@@ -15,11 +15,16 @@ SUM             |AGGREGATE
 MEAN            |AGGREGATE
 STDDEV_POP      |AGGREGATE
 VAR_POP         |AGGREGATE
-SUM_OF_SQUARES  |AGGREGATE
-SKEWNESS        |AGGREGATE
-KURTOSIS        |AGGREGATE
 PERCENTILE      |AGGREGATE
 PERCENTILE_RANK |AGGREGATE
+SUM_OF_SQUARES  |AGGREGATE
+MATRIX_COUNT    |AGGREGATE
+MATRIX_MEAN     |AGGREGATE
+MATRIX_VARIANCE |AGGREGATE
+SKEWNESS        |AGGREGATE
+KURTOSIS        |AGGREGATE
+COVARIANCE      |AGGREGATE
+CORRELATION     |AGGREGATE
 DAY_OF_MONTH    |SCALAR
 DAY             |SCALAR
 DOM             |SCALAR

sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/DefaultFunctionRegistry.java

@@ -7,8 +7,13 @@ package org.elasticsearch.xpack.sql.expression.function;
 
 import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Avg;
+import org.elasticsearch.xpack.sql.expression.function.aggregate.Correlation;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Count;
+import org.elasticsearch.xpack.sql.expression.function.aggregate.Covariance;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Kurtosis;
+import org.elasticsearch.xpack.sql.expression.function.aggregate.MatrixCount;
+import org.elasticsearch.xpack.sql.expression.function.aggregate.MatrixMean;
+import org.elasticsearch.xpack.sql.expression.function.aggregate.MatrixVariance;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Max;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Mean;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Min;
@@ -99,13 +104,17 @@ public class DefaultFunctionRegistry extends AbstractFunctionRegistry {
                 Mean.class,
                 StddevPop.class,
                 VarPop.class,
+                Percentile.class,
+                PercentileRank.class,
                 SumOfSquares.class,
+                // Matrix aggs
+                MatrixCount.class,
+                MatrixMean.class,
+                MatrixVariance.class,
                 Skewness.class,
                 Kurtosis.class,
-                Percentile.class,
+                Covariance.class,
-                PercentileRank.class
+                Correlation.class
-                // TODO: add multi arg functions like Covariance, Correlate
-
                 );
     }
     

sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Correlation.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.aggregate;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+public class Correlation extends NumericAggregate implements MatrixStatsEnclosed {
+
+    public Correlation(Location location, Expression field) {
+        super(location, field);
+    }
+
+    @Override
+    public String innerName() {
+        return "correlation";
+    }
+}

sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Covariance.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.aggregate;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+public class Covariance extends NumericAggregate implements MatrixStatsEnclosed {
+
+    public Covariance(Location location, Expression field) {
+        super(location, field);
+    }
+
+    @Override
+    public String innerName() {
+        return "covariance";
+    }
+}

sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/MatrixCount.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.aggregate;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+public class MatrixCount extends NumericAggregate implements MatrixStatsEnclosed {
+
+    public MatrixCount(Location location, Expression field) {
+        super(location, field);
+    }
+
+    @Override
+    public String innerName() {
+        return "matrix_count";
+    }
+}

sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/MatrixMean.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.aggregate;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+public class MatrixMean extends NumericAggregate implements MatrixStatsEnclosed {
+
+    public MatrixMean(Location location, Expression field) {
+        super(location, field);
+    }
+
+    @Override
+    public String innerName() {
+        return "matrix_mean";
+    }
+}

sql/server/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/MatrixVariance.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.function.aggregate;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+public class MatrixVariance extends NumericAggregate implements MatrixStatsEnclosed {
+
+    public MatrixVariance(Location location, Expression field) {
+        super(location, field);
+    }
+
+    @Override
+    public String innerName() {
+        return "matrix_variance";
+    }
+}

sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/PlanningException.java

@@ -5,24 +5,35 @@
  */
 package org.elasticsearch.xpack.sql.planner;
 
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.sql.ClientSqlException;
 import org.elasticsearch.xpack.sql.planner.Verifier.Failure;
 import org.elasticsearch.xpack.sql.util.StringUtils;
 
 import java.util.Collection;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static java.lang.String.format;
 
 public class PlanningException extends ClientSqlException {
 
+    private final Optional<RestStatus> status;
+
     public PlanningException(String message, Object... args) {
         super(message, args);
+        status = Optional.empty();
+    }
+
+    public PlanningException(String message, RestStatus restStatus, Object... args) {
+        super(message, args);
+        status = Optional.of(restStatus);
     }
 
     public PlanningException(Collection<Failure> sources) {
         super(extractMessage(sources));
+        status = Optional.empty();
     }
 
     private static String extractMessage(Collection<Failure> failures) {
@@ -30,4 +41,9 @@ public class PlanningException extends ClientSqlException {
                 .map(f -> format(Locale.ROOT, "line %s:%s: %s", f.source().location().getLineNumber(), f.source().location().getColumnNumber(), f.message()))
                 .collect(Collectors.joining(StringUtils.NEW_LINE, "Found " + failures.size() + " problem(s)\n", StringUtils.EMPTY));
     }
+
+    @Override
+    public RestStatus status() {
+        return status.orElse(super.status());
+    }
 }

sql/server/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.planner;
 
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.sql.expression.Alias;
 import org.elasticsearch.xpack.sql.expression.Attribute;
 import org.elasticsearch.xpack.sql.expression.Expression;
@@ -354,7 +355,7 @@ class QueryFolder extends RuleExecutor<PhysicalPlan> {
                                 //FIXME: what about inner key
                                 queryC = withAgg.v1().addAggColumn(withAgg.v2().context());
                                 if (withAgg.v2().innerKey() != null) {
-                                    throw new PlanningException("innerkey/matrix stats not handled (yet)");
+                                    throw new PlanningException("innerkey/matrix stats not handled (yet)", RestStatus.BAD_REQUEST);
                                 }
                             }
                         }