Cleanup sql build

Make the projects more like the rest of Elasticsearch:
* Move integration tests from `itest` to `test`
* Make a build file per sql subproject
* Make sql work properly with elasticsearch-extra

Original commit: elastic/x-pack-elasticsearch@6fbb794f68

plugin/build.gradle

@@ -83,11 +83,11 @@ dependencies {
   compile 'net.sf.supercsv:super-csv:2.4.0'
   nativeBundle "org.elasticsearch.ml:ml-cpp:${project.version}@zip"
   testCompile 'org.ini4j:ini4j:0.5.2'
-  
+
   // sql deps
   antlr "org.antlr:antlr4:4.5.3"
-  compile project(":jdbc-proto")
-  compile project(":cli-proto")
+  compile project(':x-pack-elasticsearch:sql-clients:jdbc-proto')
+  compile project(':x-pack-elasticsearch:sql-clients:cli-proto')
 
   // common test deps
   testCompile 'org.elasticsearch:securemock:1.2'

plugin/src/main/java/org/elasticsearch/xpack/sql/plugin/cli/http/CliUtils.java

@@ -9,7 +9,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.elasticsearch.xpack.sql.session.RowSetCursor;
 
-abstract class CliUtils {
+public abstract class CliUtils { // TODO made public so it could be shared with tests
 
     // this toString is a bit convoluted since it tries to be smart and pad the columns according to their values
     // as such it will look inside the row, find the max for each column and pad all the values accordingly
@@ -17,7 +17,7 @@ abstract class CliUtils {
     // a row needs to be iterated upon to fill up the values that don't take extra lines
 
     // Warning: this method _consumes_ a rowset
-    static String toString(RowSetCursor cursor) {
+    public static String toString(RowSetCursor cursor) {
         if (cursor.rowSize() == 1 && cursor.size() == 1 && cursor.column(0).toString().startsWith("digraph ")) {
             return cursor.column(0).toString();
         }

plugin/src/test/java/org/elasticsearch/xpack/sql/TestUtils.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.xpack.sql.analysis.catalog.EsCatalog;
+import org.elasticsearch.xpack.sql.execution.PlanExecutor;
+import org.elasticsearch.xpack.sql.plugin.cli.http.CliUtils;
+import org.elasticsearch.xpack.sql.session.RowSetCursor;
+
+public class TestUtils {
+    public static PlanExecutor planExecutor(Client client) {
+        PlanExecutor executor = new PlanExecutor(client,
+                () -> client.admin().cluster().prepareState().get(TimeValue.timeValueMinutes(1)).getState());
+        ((EsCatalog) executor.catalog()).setIndexNameExpressionResolver(new IndexNameExpressionResolver(client.settings()));
+        return executor;
+    }
+
+    public static void sqlOut(PlanExecutor executor, String sql) {
+        executor.sql(sql, new ActionListener<RowSetCursor>() {
+
+            @Override
+            public void onResponse(RowSetCursor cursor) {
+                System.out.println(CliUtils.toString(cursor));
+            }
+
+            @Override
+            public void onFailure(Exception ex) {
+                throw ex instanceof RuntimeException ? (RuntimeException) ex : new SqlException(ex);
+            }
+        });
+    }
+}

settings.gradle

@@ -1,20 +1,3 @@
-include "net-client"
-project(":net-client").projectDir = file("sql-clients/net-client")
-
-include "jdbc-proto"
-project(":jdbc-proto").projectDir = file("sql-clients/jdbc-proto")
-
-include "jdbc"
-project(":jdbc").projectDir = file("sql-clients/jdbc")
-
-// CLI
-include "cli-proto"
-project(":cli-proto").projectDir = file("sql-clients/cli-proto")
-
-include "cli"
-project(":cli").projectDir = file("sql-clients/cli")
-
-// NOTE: we ensure the elasticsearch dir is present in buildSrc
 File extrasDir = new File(settingsDir, '..').getCanonicalFile()
 if (extrasDir.name.endsWith('-extra') == false) {
   throw new GradleException("x-pack-elasticsearch must be checked out under an elasticsearch-extra directory, found ${extrasDir.name}")
@@ -22,4 +5,3 @@ if (extrasDir.name.endsWith('-extra') == false) {
 File elasticsearchDir = new File(extrasDir.parentFile, extrasDir.name[0..-7])
 project(':').projectDir = elasticsearchDir
 apply from: "${elasticsearchDir}/settings.gradle"
-

sql-clients/build.gradle

@@ -1,201 +1,35 @@
 description = 'Elasticsearch SQL Clients'
 
-def revHash = 123123123123123
-
-
 import org.gradle.plugins.ide.eclipse.model.*;
 
-def template() { { project ->
-    apply plugin: 'elasticsearch.build'
-    apply plugin: "java"
-    apply plugin: 'eclipse'
+subprojects {
+  apply plugin: 'elasticsearch.build'
+  apply plugin: "java"
+  apply plugin: 'eclipse'
 
-    sourceSets.test.resources.srcDirs = ["src/test/resources", "src/test/java"]
-    
-    // Massaging Eclipse
-    eclipse {
-        classpath.file {
-            whenMerged { cp ->
-                
-                def con = entries.find { e ->
-                    e.kind == "con" && e.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER")
-                }
-                ((AbstractClasspathEntry) con).accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
-                
-                entries.unique { a, b ->
-                    return a.path.compareTo(b.path)
-                }
-                entries.removeAll { it.path.endsWith('.pom') }
-            }
+  sourceSets.test.resources.srcDirs = ["src/test/resources", "src/test/java"]
+
+  // Massaging Eclipse
+  eclipse {
+    classpath.file {
+      whenMerged { cp ->
+        def con = entries.find { e ->
+          e.kind == "con" && e.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER")
         }
-        jdt {
-            javaRuntimeName = "JavaSE-1.8"
-            // specify again the compatibility to override the default JavaRE settings
-            sourceCompatibility = 1.8
-            targetCompatibility = 1.8
+        ((AbstractClasspathEntry) con).accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
+
+        entries.unique { a, b ->
+          return a.path.compareTo(b.path)
         }
+        entries.removeAll { it.path.endsWith('.pom') }
+      }
     }
-    
-    dependencies {
-        testCompile("junit:junit:${versions.junit}") {
-			exclude group:'org.hamcrest', module:'hamcrest-core'
-		}
-        testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
-    }
+  }
+
+  dependencies {
+    testCompile("junit:junit:${versions.junit}") {
+      exclude group:'org.hamcrest', module:'hamcrest-core'
     }
+    testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+  }
 }
-
-// Shared HTTP Client
-project(":net-client") {
-    apply plugin: 'elasticsearch.build'
-    apply plugin: "java"
-
-    // add some utilities for doing proto testing as this project is common to all protos
-
-    sourceSets {
-        itest {}
-    }
-    
-    dependencies {
-        itestCompile "org.elasticsearch.client:transport:${version}"
-    }
-}  
-
-// JDBC
-project(":jdbc-proto") {
-    apply plugin: 'elasticsearch.build'
-    apply plugin: "java"
-
-    
-    dependencies {
-        compile project(":net-client")
-    }
-}
-
-project(":jdbc") {
-    apply plugin: 'elasticsearch.build'
-    apply plugin: "java"
-
-    def generatedResources = "$buildDir/generated-resources/main"
-    
-    sourceSets {
-        main {
-            output.dir(generatedResources, builtBy: "generateGitHash")
-        }
-    }
-    
-    task generateGitHash {
-        doLast {
-            Properties props = new Properties()
-            props.put("version", version)
-            props.put("hash", revHash)
-            props.put("version.major", jdbcMajorVer)
-            props.put("version.minor", jdbcMinorVer)
-            File output = new File(generatedResources, "jdbc-build.properties")
-            new File(generatedResources).mkdirs()
-            output.createNewFile()
-            props.store(output.newWriter(), null) 
-        }
-    }
-
-    sourceSets {
-        itest {}
-    }
-    
-    dependencies {
-        compile project(":net-client")
-        compile project(":jdbc-proto")
-
-        itestCompile project(":plugins")
-        itestCompile configurations.compile
-        itestCompile configurations.runtime
-        itestCompile configurations.testCompile
-        itestRuntime configurations.testRuntime
-        
-        itestRuntime "com.h2database:h2:1.4.194"
-        itestRuntime "net.sourceforge.csvjdbc:csvjdbc:1.0.31"
-    }
-
-    eclipse.classpath {
-        plusConfigurations << configurations.itestRuntime
-    }
-    
-    jar {
-        from(zipTree(project(":net-client").jar.archivePath))
-        from(zipTree(project(":jdbc-proto").jar.archivePath))
-    }
-    
-    task integrationTest(type: Test) {
-        group 'Verification'
-        description 'Runs the integration tests.'
-        
-        testClassesDir = sourceSets.itest.output.classesDir
-        classpath = sourceSets.itest.runtimeClasspath
-        excludes = ["**/Abstract*.class"]
-        
-        mustRunAfter tasks.test
-    }
-}
-
-// CLI
-project(":cli-proto") {
-    apply plugin: 'elasticsearch.build'
-    apply plugin: "java"
-
-    dependencies {
-        compile project(":net-client")
-    }
-}
-
-project(":cli") {
-    apply plugin: 'elasticsearch.build'
-    apply plugin: "java"
-
-    apply plugin: 'application'
-
-    sourceSets {
-        itest {}
-    }
-    
-    dependencies {
-        compile "org.jline:jline:3.3.0"
-
-        compile project(":net-client")
-        compile project(":cli-proto")
-        
-        itestCompile project(":plugins")
-        itestCompile "org.elasticsearch.client:transport:$version"
-        itestCompile configurations.compile
-        itestCompile configurations.runtime
-        itestCompile configurations.testCompile
-        itestRuntime configurations.testRuntime
-    }
-    
-    eclipse.classpath {
-        plusConfigurations << configurations.itestRuntime
-    }
-
-    task integrationTest(type: Test) {
-        group 'Verification'
-        description 'Runs the integration tests.'
-        
-        testClassesDir = sourceSets.itest.output.classesDir
-        classpath = sourceSets.itest.runtimeClasspath
-        excludes = ["**/Abstract*.class"]
-        
-        mustRunAfter tasks.test
-    }
-
-    jar {
-        from(zipTree(project(":net-client").jar.archivePath))
-        from(zipTree(project(":cli-proto").jar.archivePath))
-    }
-    
-    mainClassName = "org.elasticsearch.sql.console.SqlConsole"
-
-    run {
-      classpath = sourceSets.test.runtimeClasspath
-    }
-    
-}
-defaultTasks 'build'

sql-clients/cli-proto/build.gradle

@@ -0,0 +1,5 @@
+apply plugin: 'elasticsearch.build'
+
+dependencies {
+  compile project(':x-pack-elasticsearch:sql-clients:net-client')
+}

sql-clients/cli/build.gradle

@@ -0,0 +1,24 @@
+apply plugin: 'elasticsearch.build'
+
+apply plugin: 'application'
+
+dependencies {
+  compile "org.jline:jline:3.3.0"
+
+  compile project(':x-pack-elasticsearch:sql-clients:net-client')
+  compile project(':x-pack-elasticsearch:sql-clients:cli-proto')
+
+  testCompile project(":x-pack-elasticsearch:transport-client")
+}
+
+// TODO seems like we should use the jars....
+jar {
+  from(zipTree(project(':x-pack-elasticsearch:sql-clients:net-client').jar.archivePath))
+  from(zipTree(project(':x-pack-elasticsearch:sql-clients:cli-proto').jar.archivePath))
+}
+
+mainClassName = "org.elasticsearch.sql.console.SqlConsole"
+
+run {
+  classpath = sourceSets.test.runtimeClasspath
+}

sql-clients/cli/src/test/java/org/elasticsearch/xpack/sql/cli/integration/net/protocol/ProtoTest.java

@@ -5,10 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.cli.integration.net.protocol;
 
-import java.util.Properties;
-
 import org.elasticsearch.client.Client;
-import org.elasticsearch.sql.TestUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.elasticsearch.xpack.sql.cli.CliConfiguration;
 import org.elasticsearch.xpack.sql.cli.integration.server.CliHttpServer;
 import org.elasticsearch.xpack.sql.cli.net.client.HttpCliClient;
@@ -18,13 +17,14 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import static org.junit.Assert.assertThat;
+import java.util.Properties;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isEmptyOrNullString;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertThat;
 
 public class ProtoTest {
 
@@ -35,7 +35,7 @@ public class ProtoTest {
     @BeforeClass
     public static void setUp() throws Exception {
         if (esClient == null) {
-            esClient = TestUtils.newClient();
+            esClient = new PreBuiltTransportClient(Settings.EMPTY);
         }
         if (server == null) {
             server = new CliHttpServer(esClient);

sql-clients/cli/src/test/java/org/elasticsearch/xpack/sql/cli/integration/server/CliProtoHandler.java

@@ -5,19 +5,19 @@
  */
 package org.elasticsearch.xpack.sql.cli.integration.server;
 
-import java.io.DataInput;
-import java.io.IOException;
+import com.sun.net.httpserver.HttpExchange;
 
 import org.elasticsearch.client.Client;
-import org.elasticsearch.sql.TestUtils;
-import org.elasticsearch.sql.plugin.cli.http.CliServerProtoUtils;
-import org.elasticsearch.sql.plugin.cli.server.CliServer;
+import org.elasticsearch.xpack.sql.TestUtils;
 import org.elasticsearch.xpack.sql.cli.net.protocol.ProtoUtils;
 import org.elasticsearch.xpack.sql.cli.net.protocol.Request;
 import org.elasticsearch.xpack.sql.cli.net.protocol.Response;
 import org.elasticsearch.xpack.sql.net.client.integration.server.ProtoHandler;
+import org.elasticsearch.xpack.sql.plugin.cli.http.CliServerProtoUtils;
+import org.elasticsearch.xpack.sql.plugin.cli.server.CliServer;
 
-import com.sun.net.httpserver.HttpExchange;
+import java.io.DataInput;
+import java.io.IOException;
 
 import static org.elasticsearch.action.ActionListener.wrap;
 

sql-clients/jdbc-proto/build.gradle

@@ -0,0 +1,5 @@
+apply plugin: 'elasticsearch.build'
+
+dependencies {
+  compile project(":x-pack-elasticsearch:sql-clients:net-client")
+}

sql-clients/jdbc/build.gradle

@@ -0,0 +1,49 @@
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.build'
+
+def generatedResources = "$buildDir/generated-resources/main"
+
+sourceSets {
+  main {
+    output.dir(generatedResources, builtBy: "generateGitHash")
+  }
+}
+
+task generateGitHash {
+  // TODO use the manifest file automatically built by elasticsearch.build
+  doLast {
+    Version current = Version.fromString(versions.elasticsearch)
+    String revHash = '123123123123123'
+    Properties props = new Properties()
+    props.put("version", versions.elasticsearch)
+    props.put("hash", revHash)
+    props.put("version.major", current.major as String)
+    props.put("version.minor", current.minor as String)
+    File output = new File(generatedResources, "jdbc-build.properties")
+    new File(generatedResources).mkdirs()
+    output.createNewFile()
+    def writer = output.newWriter("UTF-8")
+    try {
+      props.store(writer, null)
+    } finally {
+      writer.close()
+    }
+  }
+}
+
+dependencies {
+  compile project(':x-pack-elasticsearch:sql-clients:net-client')
+  compile project(':x-pack-elasticsearch:sql-clients:jdbc-proto')
+
+  testCompile project(":x-pack-elasticsearch:transport-client")
+
+  testRuntime "com.h2database:h2:1.4.194"
+  testRuntime "net.sourceforge.csvjdbc:csvjdbc:1.0.31"
+}
+
+// TODO seems like we should use the jars....
+jar {
+  from(zipTree(project(':x-pack-elasticsearch:sql-clients:net-client').jar.archivePath))
+  from(zipTree(project(':x-pack-elasticsearch:sql-clients:jdbc-proto').jar.archivePath))
+}

sql-clients/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/integration/net/protocol/ProtoTest.java

@@ -5,16 +5,10 @@
  */
 package org.elasticsearch.xpack.sql.jdbc.integration.net.protocol;
 
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
 import org.elasticsearch.client.Client;
-import org.elasticsearch.xpack.sql.TestUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.elasticsearch.xpack.sql.jdbc.integration.server.JdbcHttpServer;
 import org.elasticsearch.xpack.sql.jdbc.integration.util.JdbcTemplate;
 import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
@@ -26,7 +20,14 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import static org.junit.Assert.assertThat;
+import java.net.InetAddress;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -36,6 +37,7 @@ import static org.hamcrest.Matchers.isEmptyOrNullString;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertThat;
 
 
 public class ProtoTest {
@@ -50,7 +52,8 @@ public class ProtoTest {
     @BeforeClass
     public static void setUp() throws Exception {
         if (esClient == null) {
-            esClient = TestUtils.newClient();
+            esClient = new PreBuiltTransportClient(Settings.EMPTY)
+                    .addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(), 9300));
         }
         if (server == null) {
             server = new JdbcHttpServer(esClient);

sql-clients/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/integration/util/EsDataLoader.java

@@ -5,6 +5,23 @@
  */
 package org.elasticsearch.xpack.sql.jdbc.integration.util;
 
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.URL;
@@ -14,28 +31,9 @@ import java.util.List;
 import java.util.Random;
 import java.util.stream.Stream;
 
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.IndicesAdminClient;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import org.junit.BeforeClass;
-import org.junit.FixMethodOrder;
-import org.junit.Test;
-import org.junit.runners.MethodSorters;
-
 import static java.util.stream.Collectors.toList;
-
-import static org.junit.Assert.fail;
-
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.junit.Assert.fail;
 
 // used rarely just to load the data (hence why it's marked as abstract)
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
@@ -50,7 +48,7 @@ public abstract class EsDataLoader {
     private static void initClient() {
         if (client == null) {
             client = new PreBuiltTransportClient(Settings.EMPTY)
-                       .addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), 9300));
+                       .addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(), 9300));
         }
     }
     private static Client client()  {

sql-clients/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/integration/util/EsJdbcServer.java

@@ -5,10 +5,6 @@
  */
 package org.elasticsearch.xpack.sql.jdbc.integration.util;
 
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.util.Properties;
-
 import org.elasticsearch.client.Client;
 import org.elasticsearch.xpack.sql.integration.es.LocalEs;
 import org.elasticsearch.xpack.sql.jdbc.integration.server.JdbcHttpServer;
@@ -16,6 +12,10 @@ import org.elasticsearch.xpack.sql.jdbc.integration.util.JdbcTemplate.JdbcSuppli
 import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcDriver;
 import org.junit.rules.ExternalResource;
 
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Properties;
+
 import static org.junit.Assert.assertNotNull;
 
 public class EsJdbcServer extends ExternalResource implements JdbcSupplier<Connection> {

sql-clients/net-client/build.gradle

@@ -0,0 +1,8 @@
+apply plugin: 'elasticsearch.build'
+
+description = 'Common base for protos'
+// Tests have some testing utilities
+
+dependencies {
+  testCompile "org.elasticsearch.client:transport:${version}"
+}