Package ingest-geoip as a module (#36898)

This commit moves ingest-geoip from being a plugin to being a module
that is packaged with Elasticsearch distributions.

distribution/docker/build.gradle

@@ -12,7 +12,6 @@ configurations {
 }
 
 dependencies {
-  dockerPlugins project(path: ":plugins:ingest-geoip", configuration: 'zip')
   dockerPlugins project(path: ":plugins:ingest-user-agent", configuration: 'zip')
   dockerSource project(path: ":distribution:archives:tar")
   ossDockerSource project(path: ":distribution:archives:oss-tar")
@@ -24,7 +23,6 @@ ext.expansions = { oss ->
     'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
     'jdkVersion' : '11.0.1',
     'license': oss ? 'Apache-2.0' : 'Elastic License',
-    'ingest-geoip' : "ingest-geoip-${VersionProperties.elasticsearch}.zip",
     'ingest-user-agent' : "ingest-user-agent-${VersionProperties.elasticsearch}.zip",
     'version' : VersionProperties.elasticsearch
   ]

distribution/docker/src/docker/Dockerfile

@@ -30,9 +30,8 @@ RUN groupadd -g 1000 elasticsearch && \
 
 WORKDIR /usr/share/elasticsearch
 
-COPY ${elasticsearch} ${ingest-geoip} ${ingest-user-agent} /opt/
+COPY ${elasticsearch} ${ingest-user-agent} /opt/
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
-RUN elasticsearch-plugin install --batch file:///opt/${ingest-geoip}
 RUN elasticsearch-plugin install --batch file:///opt/${ingest-user-agent}
 RUN mkdir -p config data logs
 RUN chmod 0775 config data logs

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -222,6 +222,10 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
             throw new UserException(ExitCodes.USAGE, "plugin id is required");
         }
 
+        if ("ingest-geoip".equals(pluginId)) {
+            handleInstallIngestGeoIp();
+        }
+
         if ("x-pack".equals(pluginId)) {
             handleInstallXPack(buildFlavor());
         }
@@ -231,6 +235,12 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         install(terminal, isBatch, extractedZip, env);
     }
 
+    private static void handleInstallIngestGeoIp() throws UserException {
+        throw new UserException(
+                ExitCodes.OK,
+                "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch");
+    }
+
     Build.Flavor buildFlavor() {
         return Build.CURRENT.flavor();
     }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java

@@ -111,6 +111,14 @@ class RemovePluginCommand extends EnvironmentAwareCommand {
          */
         if ((!Files.exists(pluginDir) && !Files.exists(pluginConfigDir) && !Files.exists(removing))
                 || (!Files.exists(pluginDir) && Files.exists(pluginConfigDir) && !purge)) {
+
+            // special case for ingest-geoip since it is a module now but could have been installed from a previous when it was a plugin
+            if ("ingest-geoip".equals(pluginName)) {
+                throw new UserException(
+                        ExitCodes.OK,
+                        "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch");
+            }
+
             final String message = String.format(
                     Locale.ROOT, "plugin [%s] not found; run 'elasticsearch-plugin list' to get list of installed plugins", pluginName);
             throw new UserException(ExitCodes.CONFIG, message);

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -757,6 +757,17 @@ public class InstallPluginCommandTests extends ESTestCase {
         }
     }
 
+    public void testInstallGeoIp() throws IOException {
+        final Environment environment = createEnv(fs, temp).v2();
+        final UserException exception =
+                expectThrows(UserException.class, () -> new InstallPluginCommand().execute(terminal, "ingest-geoip", false, environment));
+        assertThat(exception.exitCode, equalTo(ExitCodes.OK));
+        assertThat(
+                exception,
+                hasToString(containsString(
+                        "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch")));
+    }
+
     public void testInstallXPack() throws IOException {
         runInstallXPackTest(Build.Flavor.DEFAULT, UserException.class, "this distribution of Elasticsearch contains X-Pack by default");
         runInstallXPackTest(

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java

@@ -29,6 +29,7 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.BufferedReader;
@@ -251,6 +252,33 @@ public class RemovePluginCommandTests extends ESTestCase {
         assertEquals("plugin name is required", e.getMessage());
     }
 
+    /**
+     * The ingest-geoip plugin receives special handling because we have re-packaged it as a module; this test ensures that we are still
+     * able to uninstall an old installation of ingest-geoip.
+     *
+     * @throws Exception if an exception is thrown creating or removing the plugin
+     */
+    public void testRemoveIngestGeoIp() throws Exception {
+        createPlugin(
+                "ingest-geoip",
+                VersionUtils.randomVersionBetween(
+                        random(),
+                        Version.CURRENT.minimumIndexCompatibilityVersion(),
+                        Version.V_6_6_0));
+        removePlugin("ingest-geoip", home, randomBoolean());
+        assertThat(Files.exists(env.pluginsFile().resolve("ingest-geoip")), equalTo(false));
+        assertRemoveCleaned(env);
+    }
+
+    public void testRemoveIngestGeoIpWhenNotInstalled() {
+        final UserException e = expectThrows(UserException.class, () -> removePlugin("ingest-geoip", home, randomBoolean()));
+        assertThat(e.exitCode, equalTo(ExitCodes.OK));
+        assertThat(
+                e,
+                hasToString(Matchers.containsString(
+                        "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch")));
+    }
+
     public void testRemoveWhenRemovingMarker() throws Exception {
         createPlugin("fake");
         Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));

docs/plugins/ingest.asciidoc

@@ -13,14 +13,6 @@ The core ingest plugins are:
 The ingest attachment plugin lets Elasticsearch extract file attachments in common formats (such as PPT, XLS, and PDF) by
 using the Apache text extraction library http://lucene.apache.org/tika/[Tika].
 
-<<ingest-geoip>>::
-
-The GeoIP processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases.
-This processor adds this information by default under the `geoip` field.
-+
-The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available
-under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/.
-
 <<ingest-user-agent>>::
 
 A processor that extracts details from the User-Agent header value.
@@ -34,6 +26,4 @@ The following plugin has been contributed by our community:
 
 include::ingest-attachment.asciidoc[]
 
-include::ingest-geoip.asciidoc[]
-
 include::ingest-user-agent.asciidoc[]

docs/reference/cat/plugins.asciidoc

@@ -25,7 +25,6 @@ U7321H6 discovery-azure-classic {version_qualified} The Azure Classic Discovery
 U7321H6 discovery-ec2           {version_qualified} The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.
 U7321H6 discovery-gce           {version_qualified} The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.
 U7321H6 ingest-attachment       {version_qualified} Ingest processor that uses Apache Tika to extract contents
-U7321H6 ingest-geoip            {version_qualified} Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database
 U7321H6 ingest-user-agent       {version_qualified} Ingest processor that extracts information from a user agent
 U7321H6 mapper-annotated-text   {version_qualified} The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.
 U7321H6 mapper-murmur3          {version_qualified} The Mapper Murmur3 plugin allows to compute hashes of a field's values at index-time and to store them in the index.

docs/reference/cluster/nodes-info.asciidoc

@@ -159,13 +159,6 @@ The result will look similar to:
           "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
           "has_native_controller": false
         },
-        {
-          "name": "ingest-geoip",
-          "version": "{version}",
-          "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
-          "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
-          "has_native_controller": false
-        },
         {
           "name": "ingest-user-agent",
           "version": "{version}",

docs/reference/cluster/stats.asciidoc

@@ -186,13 +186,6 @@ Will return, for example:
           "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
           "has_native_controller": false
         },
-        {
-          "name": "ingest-geoip",
-          "version": "{version}",
-          "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
-          "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
-          "has_native_controller": false
-        },
         {
           "name": "ingest-user-agent",
           "version": "{version}",

docs/reference/ingest/ingest-node.asciidoc

@@ -1310,10 +1310,10 @@ doesn't exist on all nodes. If you rely on custom processor plugins make sure to
 
 [source,yaml]
 --------------------------------------------------
-plugin.mandatory: ingest-attachment,ingest-geoip
+plugin.mandatory: ingest-attachment
 --------------------------------------------------
 
-A node will not start if either of these plugins are not available.
+A node will not start if this plugin is not available.
 
 The <<ingest-stats,node stats API>> can be used to fetch ingest usage statistics, globally and on a per
 pipeline basis. Useful to find out which pipelines are used the most or spent the most time on preprocessing.
@@ -1334,6 +1334,7 @@ include::processors/dot-expand.asciidoc[]
 include::processors/drop.asciidoc[]
 include::processors/fail.asciidoc[]
 include::processors/foreach.asciidoc[]
+include::processors/geoip.asciidoc[]
 include::processors/grok.asciidoc[]
 include::processors/gsub.asciidoc[]
 include::processors/join.asciidoc[]

docs/reference/ingest/processors/geoip.asciidoc

@@ -1,20 +1,17 @@
 [[ingest-geoip]]
-=== Ingest Geoip Processor Plugin
+=== Ingest Geoip Processor
 
 The GeoIP processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases.
 This processor adds this information by default under the `geoip` field. The `geoip` processor can resolve both IPv4 and
 IPv6 addresses.
 
-The ingest-geoip plugin ships by default with the GeoLite2 City, GeoLite2 Country and GeoLite2 ASN geoip2 databases from Maxmind made available
+The ingest-geoip module ships by default with the GeoLite2 City, GeoLite2 Country and GeoLite2 ASN geoip2 databases from Maxmind made available
 under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
 
 The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory,
 and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be stored
 uncompressed. The geoip config directory is located at `$ES_HOME/config/ingest-geoip` and holds the shipped databases too.
 
-:plugin_name: ingest-geoip
-include::install_remove.asciidoc[]
-
 [[using-ingest-geoip]]
 ==== Using the Geoip Processor in a Pipeline
 
@@ -25,7 +22,7 @@ include::install_remove.asciidoc[]
 | Name                   | Required  | Default                                                                            | Description
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
-| `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
+| `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip module ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
@@ -91,7 +88,7 @@ Which returns:
 
 Here is an example that uses the default country database and adds the
 geographical information to the `geo` field based on the `ip` field`. Note that
-this database is included in the plugin download. So this:
+this database is included in the module. So this:
 
 [source,js]
 --------------------------------------------------
@@ -190,7 +187,7 @@ Which returns:
 
 [[ingest-geoip-mappings-note]]
 ===== Recognizing Location as a Geopoint
-Although this plugin enriches your document with a `location` field containing
+Although this processor enriches your document with a `location` field containing
 the estimated latitude and longitude of the IP address, this field will not be
 indexed as a {ref}/geo-point.html[`geo_point`] type in Elasticsearch without explicitely defining it
 as such in the mapping.

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java

@@ -90,7 +90,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
     private Path getGeoIpDirectory(Processor.Parameters parameters) {
         final Path geoIpDirectory;
         if (parameters.env.settings().get("ingest.geoip.database_path") == null) {
-            geoIpDirectory = parameters.env.pluginsFile().resolve("ingest-geoip");
+            geoIpDirectory = parameters.env.modulesFile().resolve("ingest-geoip");
         } else {
             geoIpDirectory = PathUtils.get(parameters.env.settings().get("ingest.geoip.database_path"));
         }

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java

@@ -36,4 +36,3 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
-

modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yml

@@ -1,4 +1,4 @@
-"Ingest plugin installed":
+"ingest-geoip installed":
     - skip:
         reason: "contains is a newly added assertion"
         features: contains
@@ -10,5 +10,5 @@
     - do:
         nodes.info: {}
 
-    - contains:  { nodes.$master.plugins: { name: ingest-geoip } }
+    - contains:  { nodes.$master.modules: { name: ingest-geoip } }
     - contains:  { nodes.$master.ingest.processors: { type: geoip } }

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -22,12 +22,8 @@ apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
     testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
-    testCompile project(path: ':plugins:ingest-geoip', configuration: 'runtime')
+    testCompile project(path: ':modules:ingest-geoip', configuration: 'runtime')
     testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
     testCompile project(path: ':modules:lang-painless', configuration: 'runtime')
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
-
-integTestCluster {
-    plugin ':plugins:ingest-geoip'
-}

qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -231,10 +231,6 @@ fi
     install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-4.0.0.jar poi-ooxml-4.0.0.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
 }
 
-@test "[$GROUP] install ingest-geoip plugin" {
-    install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
-}
-
 @test "[$GROUP] install ingest-user-agent plugin" {
     install_and_check_plugin ingest user-agent
 }
@@ -243,6 +239,10 @@ fi
     check_module ingest-common jcodings-*.jar joni-*.jar
 }
 
+@test "[$GROUP] check ingest-geoip module" {
+    check_module ingest-geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
+}
+
 @test "[$GROUP] check lang-expression module" {
     # we specify the version on the asm-5.0.4.jar so that the test does
     # not spuriously pass if the jar is missing but the other asm jars
@@ -364,10 +364,6 @@ fi
     remove_plugin ingest-attachment
 }
 
-@test "[$GROUP] remove ingest-geoip plugin" {
-    remove_plugin ingest-geoip
-}
-
 @test "[$GROUP] remove ingest-user-agent plugin" {
     remove_plugin ingest-user-agent
 }