Merge pull request #18583 from rjernst/official_plugins

Use resource files for list of modules and plugins

core/build.gradle

@@ -121,6 +121,36 @@ forbiddenPatterns {
   exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
 }
 
+task generateModulesList {
+  List<String> modules = project(':modules').subprojects.collect { it.name }
+  File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
+  processResources.from(modulesFile) 
+  inputs.property('modules', modules) 
+  outputs.file(modulesFile)
+  doLast {
+    modulesFile.parentFile.mkdirs()
+    modulesFile.setText(modules.join('\n'), 'UTF-8')
+  }
+}
+
+task generatePluginsList {
+  List<String> plugins = project(':plugins').subprojects
+    .findAll { it.name.contains('example') == false }
+    .collect { it.name }
+  File pluginsFile = new File(buildDir, 'generated-resources/plugins.txt')
+  processResources.from(pluginsFile) 
+  inputs.property('plugins', plugins) 
+  outputs.file(pluginsFile)
+  doLast {
+    pluginsFile.parentFile.mkdirs()
+    pluginsFile.setText(plugins.join('\n'), 'UTF-8')
+  }
+}
+
+processResources {
+  dependsOn generateModulesList, generatePluginsList
+}
+
 thirdPartyAudit.excludes = [
   // uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
   'org.jboss.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',

core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -19,23 +19,6 @@
 
 package org.elasticsearch.plugins;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Build;
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.Command;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -62,12 +45,28 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.JarHell;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+
 import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
 /**
  * A command for the plugin cli to install a plugin into elasticsearch.
@@ -103,37 +102,40 @@ class InstallPluginCommand extends SettingCommand {
 
     private static final String PROPERTY_SUPPORT_STAGING_URLS = "es.plugins.staging";
 
-    // TODO: make this a resource file generated by gradle
+    /** The builtin modules, which are plugins, but cannot be installed or removed. */
-    static final Set<String> MODULES = unmodifiableSet(newHashSet(
+    static final Set<String> MODULES;
-            "ingest-grok",
+    static {
-            "lang-expression",
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
-            "lang-groovy",
+             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
-            "lang-painless",
+            Set<String> modules = new HashSet<>();
-            "reindex"));
+            String line = reader.readLine();
+            while (line != null) {
+                modules.add(line.trim());
+                line = reader.readLine();
+            }
+            MODULES = Collections.unmodifiableSet(modules);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 
-    // TODO: make this a resource file generated by gradle
+    /** The official plugins that can be installed simply by name. */
-    static final Set<String> OFFICIAL_PLUGINS = unmodifiableSet(new LinkedHashSet<>(Arrays.asList(
+    static final Set<String> OFFICIAL_PLUGINS;
-            "analysis-icu",
+    static {
-            "analysis-kuromoji",
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
-            "analysis-phonetic",
+             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
-            "analysis-smartcn",
+            Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
-            "analysis-stempel",
+            String line = reader.readLine();
-            "discovery-azure",
+            while (line != null) {
-            "discovery-ec2",
+                plugins.add(line.trim());
-            "discovery-gce",
+                line = reader.readLine();
-            "ingest-attachment",
+            }
-            "ingest-geoip",
+            plugins.add("x-pack");
-            "lang-javascript",
+            OFFICIAL_PLUGINS = Collections.unmodifiableSet(plugins);
-            "lang-python",
+        } catch (IOException e) {
-            "mapper-attachments",
+            throw new RuntimeException(e);
-            "mapper-murmur3",
+        }
-            "mapper-size",
+    }
-            "repository-azure",
-            "repository-gcs",
-            "repository-hdfs",
-            "repository-s3",
-            "store-smb",
-            "x-pack")));
 
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;

qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -36,8 +36,10 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
 import org.junit.After;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -532,6 +534,34 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
     }
 
+    public void testOfficialPluginsHelpSorted() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
+        try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) {
+            String line = reader.readLine();
+
+            // first find the beginning of our list of official plugins
+            while (line.endsWith("may be installed by name:") == false) {
+                line = reader.readLine();
+            }
+
+            // now check each line compares greater than the last, until we reach an empty line
+            String prev = reader.readLine();
+            line = reader.readLine();
+            while (line != null && line.trim().isEmpty() == false) {
+                assertTrue(prev + " < " + line, prev.compareTo(line) < 0);
+                prev = line;
+                line = reader.readLine();
+            }
+        }
+    }
+
+    public void testOfficialPluginsIncludesXpack() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
+        assertTrue(terminal.getOutput(), terminal.getOutput().contains("x-pack"));
+    }
+
     // TODO: test batch flag?
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...