Plugins: Add status bar on download (#18695)

As some plugins are becoming big now, it is hard for the user to know, if the plugin
is being downloaded or just nothing happens.

This commit adds a progress bar during download, which can be disabled by using the `-q`
parameter.

In addition this updates to jimfs 1.1, which allows us to test the batch mode, as adding
security policies are now supported due to having jimfs:// protocol support in URL stream
handlers.

core/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cli;
 
+import org.elasticsearch.common.SuppressForbidden;
+
 import java.io.BufferedReader;
 import java.io.Console;
 import java.io.IOException;
@@ -26,8 +28,6 @@ import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
 
-import org.elasticsearch.common.SuppressForbidden;
-
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
  *
@@ -81,8 +81,13 @@ public abstract class Terminal {
 
     /** Prints a line to the terminal at {@code verbosity} level. */
     public final void println(Verbosity verbosity, String msg) {
+        print(verbosity, msg + lineSeparator);
+    }
+
+    /** Prints message to the terminal at {@code verbosity} level, without a newline. */
+    public final void print(Verbosity verbosity, String msg) {
         if (this.verbosity.ordinal() >= verbosity.ordinal()) {
-            getWriter().print(msg + lineSeparator);
+            getWriter().print(msg);
             getWriter().flush();
         }
     }

core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -19,12 +19,31 @@
 
 package org.elasticsearch.plugins;
 
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.JarHell;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.URL;
+import java.net.URLConnection;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
@@ -49,24 +68,6 @@ import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.search.spell.LevensteinDistance;
-import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 
 /**
@@ -107,7 +108,7 @@ class InstallPluginCommand extends SettingCommand {
     static final Set<String> MODULES;
     static {
         try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
-             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
             Set<String> modules = new HashSet<>();
             String line = reader.readLine();
             while (line != null) {
@@ -124,7 +125,7 @@ class InstallPluginCommand extends SettingCommand {
     static final Set<String> OFFICIAL_PLUGINS;
     static {
         try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
-             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
             Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
             String line = reader.readLine();
             while (line != null) {
@@ -141,6 +142,7 @@ class InstallPluginCommand extends SettingCommand {
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;
 
+
     public static final Set<PosixFilePermission> DIR_AND_EXECUTABLE_PERMS;
     public static final Set<PosixFilePermission> FILE_PERMS;
 
@@ -273,13 +275,49 @@ class InstallPluginCommand extends SettingCommand {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
         URL url = new URL(urlString);
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
-        try (InputStream in = url.openStream()) {
+        URLConnection urlConnection = url.openConnection();
+        int contentLength = urlConnection.getContentLength();
+        try (InputStream in = new TerminalProgressInputStream(urlConnection.getInputStream(), contentLength, terminal)) {
             // must overwrite since creating the temp file above actually created the file
             Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING);
         }
         return zip;
     }
 
+    /**
+     * content length might be -1 for unknown and progress only makes sense if the content length is greater than 0
+     */
+    private class TerminalProgressInputStream extends ProgressInputStream {
+
+        private final Terminal terminal;
+        private int width = 50;
+        private final boolean enabled;
+
+        public TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) {
+            super(is, expectedTotalSize);
+            this.terminal = terminal;
+            this.enabled = expectedTotalSize > 0;
+        }
+
+        @Override
+        public void onProgress(int percent) {
+            if (enabled) {
+                int currentPosition = percent * width / 100;
+                StringBuilder sb = new StringBuilder("\r[");
+                sb.append(String.join("=", Collections.nCopies(currentPosition, "")));
+                if (currentPosition > 0 && percent < 100) {
+                    sb.append(">");
+                }
+                sb.append(String.join(" ", Collections.nCopies(width - currentPosition, "")));
+                sb.append("] %s   ");
+                if (percent == 100) {
+                    sb.append("\n");
+                }
+                terminal.print(Terminal.Verbosity.NORMAL, String.format(Locale.ROOT, sb.toString(), percent + "%"));
+            }
+        }
+    }
+
     /** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */
     private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
         Path zip = downloadZip(terminal, urlString, tmpDir);

core/src/main/java/org/elasticsearch/plugins/PluginCli.java

@@ -26,8 +26,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
-import java.util.Collections;
-
 /**
  * A cli tool for adding, removing and listing plugins for elasticsearch.
  */

core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An input stream that allows to add a listener to monitor progress
+ * The listener is triggered whenever a full percent is increased
+ * The listener is never triggered twice on the same percentage
+ * The listener will always return 99 percent, if the expectedTotalSize is exceeded, until it is finished
+ *
+ * Only used by the InstallPluginCommand, thus package private here
+ */
+abstract class ProgressInputStream extends FilterInputStream {
+
+    private final int expectedTotalSize;
+    private int currentPercent;
+    private int count = 0;
+
+    public ProgressInputStream(InputStream is, int expectedTotalSize) {
+        super(is);
+        this.expectedTotalSize = expectedTotalSize;
+        this.currentPercent = 0;
+    }
+
+    @Override
+    public int read() throws IOException {
+        int read = in.read();
+        checkProgress(read == -1 ? -1 : 1);
+        return read;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int byteCount = super.read(b, off, len);
+        checkProgress(byteCount);
+        return byteCount;
+    }
+
+    @Override
+    public int read(byte b[]) throws IOException {
+        return read(b, 0, b.length);
+    }
+
+    void checkProgress(int byteCount) {
+        // are we done?
+        if (byteCount == -1) {
+            currentPercent = 100;
+            onProgress(currentPercent);
+        } else {
+            count += byteCount;
+            // rounding up to 100% would mean we say we are done, before we are...
+            // this also catches issues, when expectedTotalSize was guessed wrong
+            int percent = Math.min(99, (int) Math.floor(100.0*count/expectedTotalSize));
+            if (percent > currentPercent) {
+                currentPercent = percent;
+                onProgress(percent);
+            }
+        }
+    }
+
+    public void onProgress(int percent) {}
+}

core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+
+public class ProgressInputStreamTests extends ESTestCase {
+
+    private List<Integer> progresses = new ArrayList<>();
+
+    public void testThatProgressListenerIsCalled() throws Exception {
+        ProgressInputStream is = newProgressInputStream(0);
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(1));
+        assertThat(progresses, hasItems(100));
+    }
+
+    public void testThatProgressListenerIsCalledOnUnexpectedCompletion() throws Exception {
+        ProgressInputStream is = newProgressInputStream(2);
+        is.checkProgress(-1);
+        assertThat(progresses, hasItems(100));
+    }
+
+    public void testThatProgressListenerReturnsMaxValueOnWrongExpectedSize() throws Exception {
+        ProgressInputStream is = newProgressInputStream(2);
+
+        is.checkProgress(1);
+        assertThat(progresses, hasItems(50));
+
+        is.checkProgress(3);
+        assertThat(progresses, hasItems(50, 99));
+
+        is.checkProgress(-1);
+        assertThat(progresses, hasItems(50, 99, 100));
+    }
+
+    public void testOneByte() throws Exception {
+        ProgressInputStream is = newProgressInputStream(1);
+        is.checkProgress(1);
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasItems(99, 100));
+
+    }
+
+    public void testOddBytes() throws Exception {
+        int odd = (randomIntBetween(100, 200) / 2) + 1;
+        ProgressInputStream is = newProgressInputStream(odd);
+        for (int i = 0; i < odd; i++) {
+            is.checkProgress(1);
+        }
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(odd+1));
+        assertThat(progresses, hasItem(100));
+    }
+
+    public void testEvenBytes() throws Exception {
+        int even = (randomIntBetween(100, 200) / 2);
+        ProgressInputStream is = newProgressInputStream(even);
+
+        for (int i = 0; i < even; i++) {
+            is.checkProgress(1);
+        }
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(even+1));
+        assertThat(progresses, hasItem(100));
+    }
+
+    public void testOnProgressCannotBeCalledMoreThanOncePerPercent() throws Exception {
+        int count = randomIntBetween(150, 300);
+        ProgressInputStream is = newProgressInputStream(count);
+
+        for (int i = 0; i < count; i++) {
+            is.checkProgress(1);
+        }
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(100));
+    }
+
+    private ProgressInputStream newProgressInputStream(int expectedSize) {
+        return new ProgressInputStream(null, expectedSize) {
+            @Override
+            public void onProgress(int percent) {
+                progresses.add(percent);
+            }
+        };
+    }
+}

docs/plugins/plugin-script.asciidoc

@@ -51,7 +51,7 @@ sudo bin/elasticsearch-plugin install analysis-icu
 -----------------------------------
 
 This command will install the version of the plugin that matches your
-Elasticsearch version.
+Elasticsearch version and also show a progress bar while downloading.
 
 [float]
 === Custom URL or file system
@@ -117,8 +117,8 @@ The `plugin` scripts supports a number of other command line parameters:
 === Silent/Verbose mode
 
 The `--verbose` parameter outputs more debug information, while the `--silent`
-parameter turns off all output.  The script may return the following exit
-codes:
+parameter turns off all output including the progress bar. The script may
+return the following exit codes:
 
 [horizontal]
 `0`:: everything was OK

qa/evil-tests/build.gradle

@@ -26,7 +26,7 @@
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-  testCompile 'com.google.jimfs:jimfs:1.0'
+  testCompile 'com.google.jimfs:jimfs:1.1'
 }
 
 // TODO: give each evil test its own fresh JVM for more isolation.

qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -25,6 +25,7 @@ import com.google.common.jimfs.Jimfs;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
@@ -70,6 +71,7 @@ import java.util.zip.ZipOutputStream;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.not;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class InstallPluginCommandTests extends ESTestCase {
@@ -179,6 +181,10 @@ public class InstallPluginCommandTests extends ESTestCase {
 
     /** creates a plugin .zip and returns the url for testing */
     static String createPlugin(String name, Path structure) throws IOException {
+        return createPlugin(name, structure, false);
+    }
+
+    static String createPlugin(String name, Path structure, boolean createSecurityPolicyFile) throws IOException {
         PluginTestUtil.writeProperties(structure,
             "description", "fake desc",
             "name", name,
@@ -186,6 +192,10 @@ public class InstallPluginCommandTests extends ESTestCase {
             "elasticsearch.version", Version.CURRENT.toString(),
             "java.version", System.getProperty("java.specification.version"),
             "classname", "FakePlugin");
+        if (createSecurityPolicyFile) {
+            String securityPolicyContent = "grant {\n  permission java.lang.RuntimePermission \"setFactory\";\n};\n";
+            Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8));
+        }
         writeJar(structure.resolve("plugin.jar"), "FakePlugin");
         return writeZip(structure, "elasticsearch");
     }
@@ -583,7 +593,41 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
     }
 
-    // TODO: test batch flag?
+    public void testBatchFlag() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        installPlugin(terminal, true);
+        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
+    }
+
+    public void testQuietFlagDisabled() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(randomFrom(Terminal.Verbosity.NORMAL, Terminal.Verbosity.VERBOSE));
+        installPlugin(terminal, false);
+        assertThat(terminal.getOutput(), containsString("100%"));
+    }
+
+    public void testQuietFlagEnabled() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(Terminal.Verbosity.SILENT);
+        installPlugin(terminal, false);
+        assertThat(terminal.getOutput(), not(containsString("100%")));
+    }
+
+    private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        // if batch is enabled, we also want to add a security policy
+        String pluginZip = createPlugin("fake", pluginDir, isBatch);
+
+        Map<String, String> settings = new HashMap<>();
+        settings.put("path.home", env.v1().toString());
+        new InstallPluginCommand() {
+            @Override
+            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+            }
+        }.execute(terminal, pluginZip, isBatch, settings);
+    }
+
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...
 }