Add support for username and password in SQL CLI (elastic/x-pack-elasticsearch#2718)

Add support for username and password in SQL CLI and adds tests that CLI works with security.

Original commit: elastic/x-pack-elasticsearch@39c8dbfc97

qa/sql/security/build.gradle

@@ -40,10 +40,11 @@ run {
   setting 'xpack.security.audit.outputs', '[logfile, index]'
   // Only log the events we need so we don't have as much to sort through
   setting 'xpack.security.audit.index.events.include', '[access_denied, access_granted]'
-  // Try and speed up the logging process without overwelming it
+  // Try and speed up audit logging without overwelming it
   setting 'xpack.security.audit.index.flush_interval', '250ms'
-  setting 'xpack.security.audit.index.settings.index.number_of_shards', '1'
+  // NOCOMMIT reenable after https://github.com/elastic/x-pack-elasticsearch/issues/2705 for lower overhead tests
-  setting 'xpack.security.audit.index.settings.index.refresh_interval', '250ms'
+  // setting 'xpack.security.audit.index.settings.index.number_of_shards', '1'
+  // setting 'xpack.security.audit.index.settings.index.refresh_interval', '250ms'
   // Setup roles used by tests
   extraConfigFile 'x-pack/roles.yml', 'roles.yml'
   /* Setup the one admin user that we run the tests as.

qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliFetchSizeIT.java

@@ -5,15 +5,20 @@
  */
 package org.elasticsearch.xpack.qa.sql.security;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.qa.sql.cli.FetchSizeTestCase;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/2074")
 public class CliFetchSizeIT extends FetchSizeTestCase {
-    // NOCOMMIT get this working with security....
+    static String securityEsUrlPrefix() {
+        return "test_admin:x-pack-test-password@";
+    }
     @Override
     protected Settings restClientSettings() {
         return RestSqlIT.securitySettings();
     }
+
+    @Override
+    protected String esUrlPrefix() {
+        return securityEsUrlPrefix();
+    }
 }

qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSelectIT.java

@@ -5,15 +5,17 @@
  */
 package org.elasticsearch.xpack.qa.sql.security;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.qa.sql.cli.SelectTestCase;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/2074")
 public class CliSelectIT extends SelectTestCase {
-    // NOCOMMIT get this working with security....
     @Override
     protected Settings restClientSettings() {
         return RestSqlIT.securitySettings();
     }
+
+    @Override
+    protected String esUrlPrefix() {
+        return CliFetchSizeIT.securityEsUrlPrefix();
+    }
 }

qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliShowIT.java

@@ -5,15 +5,17 @@
  */
 package org.elasticsearch.xpack.qa.sql.security;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.qa.sql.cli.ShowTestCase;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/2074")
 public class CliShowIT extends ShowTestCase {
-    // NOCOMMIT get this working with security....
     @Override
     protected Settings restClientSettings() {
         return RestSqlIT.securitySettings();
     }
+
+    @Override
+    protected String esUrlPrefix() {
+        return CliFetchSizeIT.securityEsUrlPrefix();
+    }
 }

qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java

@@ -110,7 +110,7 @@ public abstract class CliIntegrationTestCase extends ESRestTestCase {
         cliSocket.setSoTimeout(10000);
 
         out = new PrintWriter(new OutputStreamWriter(cliSocket.getOutputStream(), StandardCharsets.UTF_8), true);
-        out.println(ES.get());
+        out.println(esUrlPrefix() + ES.get());
         in = new BufferedReader(new InputStreamReader(cliSocket.getInputStream(), StandardCharsets.UTF_8));
         // Throw out the logo and warnings about making a dumb terminal
         while (false == readLine().contains("SQL"));
@@ -146,6 +146,14 @@ public abstract class CliIntegrationTestCase extends ESRestTestCase {
         }
     }
 
+    /**
+     * Prefix to the Elasticsearch URL. Override to add
+     * authentication support.
+     */
+    protected String esUrlPrefix() {
+        return "";
+    }
+
     protected void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
         XContentBuilder builder = JsonXContent.contentBuilder().startObject();
         body.accept(builder);

sql/cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java

@@ -6,7 +6,9 @@
 package org.elasticsearch.xpack.sql.cli;
 
 import org.elasticsearch.xpack.sql.cli.net.protocol.QueryResponse;
+import org.elasticsearch.xpack.sql.net.client.SuppressForbidden;
 import org.elasticsearch.xpack.sql.net.client.util.IOUtils;
+import org.elasticsearch.xpack.sql.net.client.util.StringUtils;
 import org.elasticsearch.xpack.sql.protocol.shared.AbstractQueryInitRequest;
 import org.jline.reader.EndOfFileException;
 import org.jline.reader.LineReader;
@@ -18,9 +20,12 @@ import org.jline.utils.AttributedString;
 import org.jline.utils.AttributedStringBuilder;
 import org.jline.utils.InfoCmp.Capability;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Locale;
 import java.util.Properties;
 import java.util.logging.LogManager;
@@ -36,15 +41,56 @@ import static org.jline.utils.AttributedStyle.YELLOW;
 public class Cli {
     public static void main(String... args) throws Exception {
         /* Initialize the logger from the a properties file we bundle. This makes sure
-         * we get useful error messages. */
+         * we get useful error messages from jLine. */
         LogManager.getLogManager().readConfiguration(Cli.class.getResourceAsStream("/logging.properties"));
-        String url = "localhost:9200/_sql/cli";
+        String hostAndPort = "localhost:9200";
+        Properties properties = new Properties();
+        String user = null;
+        String password = null;
         if (args.length > 0) {
-            url = args[0] + "/_sql/cli";
+            hostAndPort = args[0];
+            if (false == hostAndPort.contains("://")) {
+                // Default to http
+                hostAndPort = "http://" + hostAndPort;
+            }
+            URI parsed;
+            try {
+                parsed = new URI(hostAndPort);
+            } catch (URISyntaxException e) {
+                exit("Invalid connection configuration [" + hostAndPort + "]: " + e.getMessage(), 1);
+                return;
+            }
+            if (false == "".equals(parsed.getPath())) {
+                exit("Invalid connection configuration [" + hostAndPort + "]: Path not allowed", 1);
+                return;
+            }
+            user = parsed.getUserInfo();
+            if (user != null) {
+                // NOCOMMIT just use a URI the whole time
+                hostAndPort = parsed.getScheme() + "://" + parsed.getHost() + ":" + parsed.getPort();
+                int colonIndex = user.indexOf(':');
+                if (colonIndex >= 0) {
+                    password = user.substring(colonIndex + 1);
+                    user = user.substring(0, colonIndex);
+                }
+            }
         }
         try (Terminal term = TerminalBuilder.builder().build()) {
             try {
-                Cli console = new Cli(new CliConfiguration(url, new Properties()), term);
+                if (user != null) {
+                    if (password == null) {
+                        term.writer().print("password: ");
+                        term.writer().flush();
+                        term.echo(false);
+                        password = new BufferedReader(term.reader()).readLine();
+                        term.echo(true);
+                    }
+                    properties.setProperty("user", user);
+                    properties.setProperty("pass", password);
+                }
+
+                boolean debug = StringUtils.parseBoolean(System.getProperty("cli.debug", "false"));
+                Cli console = new Cli(debug, new CliConfiguration(hostAndPort + "/_sql/cli", properties), term);
                 console.run();
             } catch (FatalException e) {
                 term.writer().println(e.getMessage());
@@ -52,12 +98,14 @@ public class Cli {
         }
     }
 
+    private final boolean debug;
     private final Terminal term;
     private final CliHttpClient cliClient;
     private int fetchSize = AbstractQueryInitRequest.DEFAULT_FETCH_SIZE;
     private String fetchSeparator = "";
 
-    Cli(CliConfiguration cfg, Terminal terminal) {
+    Cli(boolean debug, CliConfiguration cfg, Terminal terminal) {
+        this.debug = debug;
         term = terminal;
         cliClient = new CliHttpClient(cfg);
     }
@@ -145,6 +193,9 @@ public class Cli {
         asb.append(e.getMessage(), DEFAULT.boldOff().italic().foreground(YELLOW));
         asb.append("]", BOLD.underlineOff().foreground(RED));
         term.writer().println(asb.toAnsi(term));
+        if (debug) {
+            e.printStackTrace(term.writer());
+        }
     }
 
     private static String logo() {
@@ -259,4 +310,10 @@ public class Cli {
             super(message);
         }
     }
+
+    @SuppressForbidden(reason = "CLI application")
+    private static void exit(String message, int code) {
+        System.err.println(message);
+        System.exit(code);
+    }
 }

test/sql-cli-fixture/src/main/java/org/elasticsearch/xpack/sql/cli/fixture/CliFixture.java

@@ -88,6 +88,7 @@ public class CliFixture {
                     }
                     List<String> command = new ArrayList<>();
                     command.add(javaExecutable.toString());
+                    command.add("-Dcli.debug=true");
                     //                    command.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000");
                     // Force a specific terminal type so we have consistent responses for testing.
                     command.add("-Dorg.jline.terminal.type=xterm-256color");