From 56fa751928c2d9f89063573de5fc1b97c471a669 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Wed, 29 Jun 2016 16:44:12 +0200 Subject: [PATCH] Plugins: Add status bar on download (#18695) As some plugins are becoming big now, it is hard for the user to know, if the plugin is being downloaded or just nothing happens. This commit adds a progress bar during download, which can be disabled by using the `-q` parameter. In addition this updates to jimfs 1.1, which allows us to test the batch mode, as adding security policies are now supported due to having jimfs:// protocol support in URL stream handlers. --- .../java/org/elasticsearch/cli/Terminal.java | 11 +- .../plugins/InstallPluginCommand.java | 80 ++++++++---- .../org/elasticsearch/plugins/PluginCli.java | 2 - .../plugins/ProgressInputStream.java | 83 +++++++++++++ .../plugins/ProgressInputStreamTests.java | 116 ++++++++++++++++++ docs/plugins/plugin-script.asciidoc | 6 +- qa/evil-tests/build.gradle | 2 +- .../plugins/InstallPluginCommandTests.java | 46 ++++++- 8 files changed, 315 insertions(+), 31 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java create mode 100644 core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java diff --git a/core/src/main/java/org/elasticsearch/cli/Terminal.java b/core/src/main/java/org/elasticsearch/cli/Terminal.java index d2dc57263dc..58eb5012d07 100644 --- a/core/src/main/java/org/elasticsearch/cli/Terminal.java +++ b/core/src/main/java/org/elasticsearch/cli/Terminal.java @@ -19,6 +19,8 @@ package org.elasticsearch.cli; +import org.elasticsearch.common.SuppressForbidden; + import java.io.BufferedReader; import java.io.Console; import java.io.IOException; @@ -26,8 +28,6 @@ import java.io.InputStreamReader; import java.io.PrintWriter; import java.nio.charset.Charset; -import org.elasticsearch.common.SuppressForbidden; - /** * A Terminal wraps access to reading input and writing output for a cli. * @@ -81,8 +81,13 @@ public abstract class Terminal { /** Prints a line to the terminal at {@code verbosity} level. */ public final void println(Verbosity verbosity, String msg) { + print(verbosity, msg + lineSeparator); + } + + /** Prints message to the terminal at {@code verbosity} level, without a newline. */ + public final void print(Verbosity verbosity, String msg) { if (this.verbosity.ordinal() >= verbosity.ordinal()) { - getWriter().print(msg + lineSeparator); + getWriter().print(msg); getWriter().flush(); } } diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 51bb3b6b82f..e9ea2d11e37 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -19,12 +19,31 @@ package org.elasticsearch.plugins; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.apache.lucene.search.spell.LevensteinDistance; +import org.apache.lucene.util.CollectionUtil; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.Version; +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.SettingCommand; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.internal.InternalSettingsPreparer; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.URL; +import java.net.URLConnection; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; @@ -49,24 +68,6 @@ import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import joptsimple.OptionSet; -import joptsimple.OptionSpec; -import org.apache.lucene.search.spell.LevensteinDistance; -import org.apache.lucene.util.CollectionUtil; -import org.apache.lucene.util.IOUtils; -import org.elasticsearch.Version; -import org.elasticsearch.bootstrap.JarHell; -import org.elasticsearch.cli.ExitCodes; -import org.elasticsearch.cli.SettingCommand; -import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cli.UserError; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.node.internal.InternalSettingsPreparer; - import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; /** @@ -107,7 +108,7 @@ class InstallPluginCommand extends SettingCommand { static final Set MODULES; static { try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt"); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { Set modules = new HashSet<>(); String line = reader.readLine(); while (line != null) { @@ -124,7 +125,7 @@ class InstallPluginCommand extends SettingCommand { static final Set OFFICIAL_PLUGINS; static { try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt"); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { Set plugins = new TreeSet<>(); // use tree set to get sorting for help command String line = reader.readLine(); while (line != null) { @@ -141,6 +142,7 @@ class InstallPluginCommand extends SettingCommand { private final OptionSpec batchOption; private final OptionSpec arguments; + public static final Set DIR_AND_EXECUTABLE_PERMS; public static final Set FILE_PERMS; @@ -273,13 +275,49 @@ class InstallPluginCommand extends SettingCommand { terminal.println(VERBOSE, "Retrieving zip from " + urlString); URL url = new URL(urlString); Path zip = Files.createTempFile(tmpDir, null, ".zip"); - try (InputStream in = url.openStream()) { + URLConnection urlConnection = url.openConnection(); + int contentLength = urlConnection.getContentLength(); + try (InputStream in = new TerminalProgressInputStream(urlConnection.getInputStream(), contentLength, terminal)) { // must overwrite since creating the temp file above actually created the file Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING); } return zip; } + /** + * content length might be -1 for unknown and progress only makes sense if the content length is greater than 0 + */ + private class TerminalProgressInputStream extends ProgressInputStream { + + private final Terminal terminal; + private int width = 50; + private final boolean enabled; + + public TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) { + super(is, expectedTotalSize); + this.terminal = terminal; + this.enabled = expectedTotalSize > 0; + } + + @Override + public void onProgress(int percent) { + if (enabled) { + int currentPosition = percent * width / 100; + StringBuilder sb = new StringBuilder("\r["); + sb.append(String.join("=", Collections.nCopies(currentPosition, ""))); + if (currentPosition > 0 && percent < 100) { + sb.append(">"); + } + sb.append(String.join(" ", Collections.nCopies(width - currentPosition, ""))); + sb.append("] %s   "); + if (percent == 100) { + sb.append("\n"); + } + terminal.print(Terminal.Verbosity.NORMAL, String.format(Locale.ROOT, sb.toString(), percent + "%")); + } + } + } + /** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */ private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception { Path zip = downloadZip(terminal, urlString, tmpDir); diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java index 3a88c4d0083..3ce60882cce 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; -import java.util.Collections; - /** * A cli tool for adding, removing and listing plugins for elasticsearch. */ diff --git a/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java b/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java new file mode 100644 index 00000000000..16e1f203bb3 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * An input stream that allows to add a listener to monitor progress + * The listener is triggered whenever a full percent is increased + * The listener is never triggered twice on the same percentage + * The listener will always return 99 percent, if the expectedTotalSize is exceeded, until it is finished + * + * Only used by the InstallPluginCommand, thus package private here + */ +abstract class ProgressInputStream extends FilterInputStream { + + private final int expectedTotalSize; + private int currentPercent; + private int count = 0; + + public ProgressInputStream(InputStream is, int expectedTotalSize) { + super(is); + this.expectedTotalSize = expectedTotalSize; + this.currentPercent = 0; + } + + @Override + public int read() throws IOException { + int read = in.read(); + checkProgress(read == -1 ? -1 : 1); + return read; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int byteCount = super.read(b, off, len); + checkProgress(byteCount); + return byteCount; + } + + @Override + public int read(byte b[]) throws IOException { + return read(b, 0, b.length); + } + + void checkProgress(int byteCount) { + // are we done? + if (byteCount == -1) { + currentPercent = 100; + onProgress(currentPercent); + } else { + count += byteCount; + // rounding up to 100% would mean we say we are done, before we are... + // this also catches issues, when expectedTotalSize was guessed wrong + int percent = Math.min(99, (int) Math.floor(100.0*count/expectedTotalSize)); + if (percent > currentPercent) { + currentPercent = percent; + onProgress(percent); + } + } + } + + public void onProgress(int percent) {} +} diff --git a/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java b/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java new file mode 100644 index 00000000000..81e937d26a9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasSize; + +public class ProgressInputStreamTests extends ESTestCase { + + private List progresses = new ArrayList<>(); + + public void testThatProgressListenerIsCalled() throws Exception { + ProgressInputStream is = newProgressInputStream(0); + is.checkProgress(-1); + + assertThat(progresses, hasSize(1)); + assertThat(progresses, hasItems(100)); + } + + public void testThatProgressListenerIsCalledOnUnexpectedCompletion() throws Exception { + ProgressInputStream is = newProgressInputStream(2); + is.checkProgress(-1); + assertThat(progresses, hasItems(100)); + } + + public void testThatProgressListenerReturnsMaxValueOnWrongExpectedSize() throws Exception { + ProgressInputStream is = newProgressInputStream(2); + + is.checkProgress(1); + assertThat(progresses, hasItems(50)); + + is.checkProgress(3); + assertThat(progresses, hasItems(50, 99)); + + is.checkProgress(-1); + assertThat(progresses, hasItems(50, 99, 100)); + } + + public void testOneByte() throws Exception { + ProgressInputStream is = newProgressInputStream(1); + is.checkProgress(1); + is.checkProgress(-1); + + assertThat(progresses, hasItems(99, 100)); + + } + + public void testOddBytes() throws Exception { + int odd = (randomIntBetween(100, 200) / 2) + 1; + ProgressInputStream is = newProgressInputStream(odd); + for (int i = 0; i < odd; i++) { + is.checkProgress(1); + } + is.checkProgress(-1); + + assertThat(progresses, hasSize(odd+1)); + assertThat(progresses, hasItem(100)); + } + + public void testEvenBytes() throws Exception { + int even = (randomIntBetween(100, 200) / 2); + ProgressInputStream is = newProgressInputStream(even); + + for (int i = 0; i < even; i++) { + is.checkProgress(1); + } + is.checkProgress(-1); + + assertThat(progresses, hasSize(even+1)); + assertThat(progresses, hasItem(100)); + } + + public void testOnProgressCannotBeCalledMoreThanOncePerPercent() throws Exception { + int count = randomIntBetween(150, 300); + ProgressInputStream is = newProgressInputStream(count); + + for (int i = 0; i < count; i++) { + is.checkProgress(1); + } + is.checkProgress(-1); + + assertThat(progresses, hasSize(100)); + } + + private ProgressInputStream newProgressInputStream(int expectedSize) { + return new ProgressInputStream(null, expectedSize) { + @Override + public void onProgress(int percent) { + progresses.add(percent); + } + }; + } +} \ No newline at end of file diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 1e21288e39c..987cc7c9758 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -51,7 +51,7 @@ sudo bin/elasticsearch-plugin install analysis-icu ----------------------------------- This command will install the version of the plugin that matches your -Elasticsearch version. +Elasticsearch version and also show a progress bar while downloading. [float] === Custom URL or file system @@ -117,8 +117,8 @@ The `plugin` scripts supports a number of other command line parameters: === Silent/Verbose mode The `--verbose` parameter outputs more debug information, while the `--silent` -parameter turns off all output. The script may return the following exit -codes: +parameter turns off all output including the progress bar. The script may +return the following exit codes: [horizontal] `0`:: everything was OK diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index 53406f1aad9..cba9334fbca 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -26,7 +26,7 @@ apply plugin: 'elasticsearch.standalone-test' dependencies { - testCompile 'com.google.jimfs:jimfs:1.0' + testCompile 'com.google.jimfs:jimfs:1.1' } // TODO: give each evil test its own fresh JVM for more isolation. diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 2b9b6ec6ab9..e5117fa0aa0 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -25,6 +25,7 @@ import com.google.common.jimfs.Jimfs; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserError; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.Tuple; @@ -70,6 +71,7 @@ import java.util.zip.ZipOutputStream; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.not; @LuceneTestCase.SuppressFileSystems("*") public class InstallPluginCommandTests extends ESTestCase { @@ -179,6 +181,10 @@ public class InstallPluginCommandTests extends ESTestCase { /** creates a plugin .zip and returns the url for testing */ static String createPlugin(String name, Path structure) throws IOException { + return createPlugin(name, structure, false); + } + + static String createPlugin(String name, Path structure, boolean createSecurityPolicyFile) throws IOException { PluginTestUtil.writeProperties(structure, "description", "fake desc", "name", name, @@ -186,6 +192,10 @@ public class InstallPluginCommandTests extends ESTestCase { "elasticsearch.version", Version.CURRENT.toString(), "java.version", System.getProperty("java.specification.version"), "classname", "FakePlugin"); + if (createSecurityPolicyFile) { + String securityPolicyContent = "grant {\n permission java.lang.RuntimePermission \"setFactory\";\n};\n"; + Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8)); + } writeJar(structure.resolve("plugin.jar"), "FakePlugin"); return writeZip(structure, "elasticsearch"); } @@ -583,7 +593,41 @@ public class InstallPluginCommandTests extends ESTestCase { assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin")); } - // TODO: test batch flag? + public void testBatchFlag() throws Exception { + MockTerminal terminal = new MockTerminal(); + installPlugin(terminal, true); + assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions")); + } + + public void testQuietFlagDisabled() throws Exception { + MockTerminal terminal = new MockTerminal(); + terminal.setVerbosity(randomFrom(Terminal.Verbosity.NORMAL, Terminal.Verbosity.VERBOSE)); + installPlugin(terminal, false); + assertThat(terminal.getOutput(), containsString("100%")); + } + + public void testQuietFlagEnabled() throws Exception { + MockTerminal terminal = new MockTerminal(); + terminal.setVerbosity(Terminal.Verbosity.SILENT); + installPlugin(terminal, false); + assertThat(terminal.getOutput(), not(containsString("100%"))); + } + + private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception { + Tuple env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + // if batch is enabled, we also want to add a security policy + String pluginZip = createPlugin("fake", pluginDir, isBatch); + + Map settings = new HashMap<>(); + settings.put("path.home", env.v1().toString()); + new InstallPluginCommand() { + @Override + void jarHellCheck(Path candidate, Path pluginsDir) throws Exception { + } + }.execute(terminal, pluginZip, isBatch, settings); + } + // TODO: test checksum (need maven/official below) // TODO: test maven, official, and staging install...need tests with fixtures... }