Package ingest-user-agent as a module (#36956)

This commit moves ingest-user-agent from being a plugin to being a
module that is packaged with Elasticsearch distributions.

distribution/docker/build.gradle

@@ -12,7 +12,6 @@ configurations {
 }
 
 dependencies {
-  dockerPlugins project(path: ":plugins:ingest-user-agent", configuration: 'zip')
   dockerSource project(path: ":distribution:archives:tar")
   ossDockerSource project(path: ":distribution:archives:oss-tar")
 }
@@ -23,7 +22,6 @@ ext.expansions = { oss ->
     'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
     'jdkVersion' : '11.0.1',
     'license': oss ? 'Apache-2.0' : 'Elastic License',
-    'ingest-user-agent' : "ingest-user-agent-${VersionProperties.elasticsearch}.zip",
     'version' : VersionProperties.elasticsearch
   ]
 }

distribution/docker/src/docker/Dockerfile

@@ -30,9 +30,8 @@ RUN groupadd -g 1000 elasticsearch && \
 
 WORKDIR /usr/share/elasticsearch
 
-COPY ${elasticsearch} ${ingest-user-agent} /opt/
+COPY ${elasticsearch} /opt/
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
-RUN elasticsearch-plugin install --batch file:///opt/${ingest-user-agent}
 RUN mkdir -p config data logs
 RUN chmod 0775 config data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -222,8 +222,10 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
             throw new UserException(ExitCodes.USAGE, "plugin id is required");
         }
 
-        if ("ingest-geoip".equals(pluginId)) {
+        if ("ingest-geoip".equals(pluginId) || "ingest-user-agent".equals(pluginId)) {
-            handleInstallIngestGeoIp();
+            throw new UserException(
+                    ExitCodes.OK,
+                    "[" + pluginId + "] is no longer a plugin but instead a module packaged with this distribution of Elasticsearch");
         }
 
         if ("x-pack".equals(pluginId)) {
@@ -235,12 +237,6 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         install(terminal, isBatch, extractedZip, env);
     }
 
-    private static void handleInstallIngestGeoIp() throws UserException {
-        throw new UserException(
-                ExitCodes.OK,
-                "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch");
-    }
-
     Build.Flavor buildFlavor() {
         return Build.CURRENT.flavor();
     }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java

@@ -21,11 +21,11 @@ package org.elasticsearch.plugins;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
@@ -112,11 +112,14 @@ class RemovePluginCommand extends EnvironmentAwareCommand {
         if ((!Files.exists(pluginDir) && !Files.exists(pluginConfigDir) && !Files.exists(removing))
                 || (!Files.exists(pluginDir) && Files.exists(pluginConfigDir) && !purge)) {
 
-            // special case for ingest-geoip since it is a module now but could have been installed from a previous when it was a plugin
+            /*
-            if ("ingest-geoip".equals(pluginName)) {
+             * This is special case handling for ingest-geoip and ingest-user-agent since they are modules now but could have been installed
+             * from a previous when it was a plugin.
+             */
+            if ("ingest-geoip".equals(pluginName) || "ingest-user-agent".equals(pluginName)) {
                 throw new UserException(
                         ExitCodes.OK,
-                        "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch");
+                        "[" + pluginName + "] is no longer a plugin but instead a module packaged with this distribution of Elasticsearch");
             }
 
             final String message = String.format(

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -757,15 +757,25 @@ public class InstallPluginCommandTests extends ESTestCase {
         }
     }
 
-    public void testInstallGeoIp() throws IOException {
+    public void testInstallIngestGeoIp() throws IOException {
+        runInstallIngestGeoIpOrIngestUserAgentTest("ingest-geoip");
+    }
+
+    public void testInstallIngestUserAgent() throws IOException {
+        runInstallIngestGeoIpOrIngestUserAgentTest("ingest-user-agent");
+    }
+
+    private void runInstallIngestGeoIpOrIngestUserAgentTest(final String pluginId) throws IOException {
+        assert "ingest-geoip".equals(pluginId) || "ingest-user-agent".equals(pluginId) : pluginId;
         final Environment environment = createEnv(fs, temp).v2();
         final UserException exception =
-                expectThrows(UserException.class, () -> new InstallPluginCommand().execute(terminal, "ingest-geoip", false, environment));
+                expectThrows(UserException.class, () -> new InstallPluginCommand().execute(terminal, pluginId, false, environment));
         assertThat(exception.exitCode, equalTo(ExitCodes.OK));
         assertThat(
                 exception,
                 hasToString(containsString(
-                        "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch")));
+                        "[" + pluginId + "] is no longer a plugin but instead a module packaged with this distribution of Elasticsearch")));
+
     }
 
     public void testInstallXPack() throws IOException {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java

@@ -259,24 +259,48 @@ public class RemovePluginCommandTests extends ESTestCase {
      * @throws Exception if an exception is thrown creating or removing the plugin
      */
     public void testRemoveIngestGeoIp() throws Exception {
+        runTestRemoveIngestGeoIpOrIngestUserAgent("ingest-geoip");
+    }
+
+    /**
+     * The ingest-user-agent plugin receives special handling because we have re-packaged it as a module; this test ensures that we are
+     * still able to uninstall an old installation of ingest-user-agent.
+     *
+     * @throws Exception if an exception is thrown creating or removing the plugin
+     */
+    public void testRemoveIngestUserAgent() throws Exception {
+        runTestRemoveIngestGeoIpOrIngestUserAgent("ingest-user-agent");
+    }
+
+    private void runTestRemoveIngestGeoIpOrIngestUserAgent(final String name) throws Exception {
+        assert "ingest-geoip".equals(name) || "ingest-user-agent".equals(name) : name;
         createPlugin(
-                "ingest-geoip",
+                name,
                 VersionUtils.randomVersionBetween(
                         random(),
                         Version.CURRENT.minimumIndexCompatibilityVersion(),
                         Version.V_6_6_0));
-        removePlugin("ingest-geoip", home, randomBoolean());
+        removePlugin(name, home, randomBoolean());
-        assertThat(Files.exists(env.pluginsFile().resolve("ingest-geoip")), equalTo(false));
+        assertThat(Files.exists(env.pluginsFile().resolve(name)), equalTo(false));
         assertRemoveCleaned(env);
     }
 
     public void testRemoveIngestGeoIpWhenNotInstalled() {
-        final UserException e = expectThrows(UserException.class, () -> removePlugin("ingest-geoip", home, randomBoolean()));
+        runTestRemoveIngestGeoIpOrIngestUserAgentWhenNotInstalled("ingest-geoip");
+    }
+
+    public void testRemoveIngestUserAgentWhenNotInstalled() {
+        runTestRemoveIngestGeoIpOrIngestUserAgentWhenNotInstalled("ingest-user-agent");
+    }
+
+    private void runTestRemoveIngestGeoIpOrIngestUserAgentWhenNotInstalled(final String name) {
+        assert "ingest-geoip".equals(name) || "ingest-user-agent".equals(name) : name;
+        final UserException e = expectThrows(UserException.class, () -> removePlugin(name, home, randomBoolean()));
         assertThat(e.exitCode, equalTo(ExitCodes.OK));
         assertThat(
                 e,
                 hasToString(Matchers.containsString(
-                        "ingest-geoip is no longer a plugin but instead a module packaged with this distribution of Elasticsearch")));
+                        "[" + name + "] is no longer a plugin but instead a module packaged with this distribution of Elasticsearch")));
     }
 
     public void testRemoveWhenRemovingMarker() throws Exception {

docs/plugins/ingest-user-agent.asciidoc

@@ -1,88 +1,7 @@
 [[ingest-user-agent]]
-=== Ingest user agent processor plugin
+=== Ingest `user_agent` Processor Plugin
 
-The `user_agent` processor extracts details from the user agent string a browser sends with its web requests.
+The `user_agent` processor is no longer distributed as a plugin, but is now a module
-This processor adds this information by default under the `user_agent` field.
+distributed by default with Elasticsearch. See
+{ref}/ingest-user-agent.html[Ingest `user_agent` processor] for more details.
 
-The ingest-user-agent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core.
-
-:plugin_name: ingest-user-agent
-include::install_remove.asciidoc[]
-
-[[using-ingest-user-agent]]
-==== Using the user_agent Processor in a Pipeline
-
-[[ingest-user-agent-options]]
-.User-agent options
-[options="header"]
-|======
-| Name                   | Required  | Default                                                                                         | Description
-| `field`                | yes       | -                                                                                               | The field containing the user agent string.
-| `target_field`         | no        | user_agent                                                                                      | The field that will be filled with the user agent details.
-| `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
-| `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
-| `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
-|======
-
-Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
-
-[source,js]
---------------------------------------------------
-PUT _ingest/pipeline/user_agent
-{
-  "description" : "Add user agent information",
-  "processors" : [
-    {
-      "user_agent" : {
-        "field" : "agent"
-      }
-    }
-  ]
-}
-PUT my_index/_doc/my_id?pipeline=user_agent
-{
-  "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
-}
-GET my_index/_doc/my_id
---------------------------------------------------
-// CONSOLE
-
-Which returns
-
-[source,js]
---------------------------------------------------
-{
-  "found": true,
-  "_index": "my_index",
-  "_type": "_doc",
-  "_id": "my_id",
-  "_version": 1,
-  "_seq_no": 22,
-  "_primary_term": 1,
-  "_source": {
-    "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
-    "user_agent": {
-      "name": "Chrome",
-      "major": "51",
-      "minor": "0",
-      "patch": "2704",
-      "os_name": "Mac OS X",
-      "os": "Mac OS X 10.10.5",
-      "os_major": "10",
-      "os_minor": "10",
-      "device": "Other"
-    }
-  }
-}
---------------------------------------------------
-// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term": 1/"_primary_term" : $body._primary_term/]
-
-===== Using a custom regex file
-To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-user-agent` directory and
-has to have a `.yaml` filename extension. The file has to be present at node startup, any changes to it or any new files added
-while the node is running will not have any effect.
-
-In practice, it will make most sense for any custom regex file to be a variant of the default file, either a more recent version
-or a customised version.
-
-The default file included in `ingest-user-agent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml

docs/plugins/ingest.asciidoc

@@ -24,7 +24,10 @@ details.
 
 <<ingest-user-agent>>::
 
-A processor that extracts details from the User-Agent header value.
+A processor that extracts details from the User-Agent header value. The
+`user_agent` processor is no longer distributed as a plugin, but is now a module
+distributed by default with Elasticsearch. See
+{ref}/ingest-user-agent.html[Ingest `user_agent` processor] for more details.
 
 [float]
 === Community contributed ingest plugins

docs/reference/cat/plugins.asciidoc

@@ -25,7 +25,6 @@ U7321H6 discovery-azure-classic {version_qualified} The Azure Classic Discovery
 U7321H6 discovery-ec2           {version_qualified} The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.
 U7321H6 discovery-gce           {version_qualified} The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.
 U7321H6 ingest-attachment       {version_qualified} Ingest processor that uses Apache Tika to extract contents
-U7321H6 ingest-user-agent       {version_qualified} Ingest processor that extracts information from a user agent
 U7321H6 mapper-annotated-text   {version_qualified} The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.
 U7321H6 mapper-murmur3          {version_qualified} The Mapper Murmur3 plugin allows to compute hashes of a field's values at index-time and to store them in the index.
 U7321H6 mapper-size             {version_qualified} The Mapper Size plugin allows document to record their uncompressed size at index time.

docs/reference/cluster/nodes-info.asciidoc

@@ -158,13 +158,6 @@ The result will look similar to:
           "description": "The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.",
           "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
           "has_native_controller": false
-        },
-        {
-          "name": "ingest-user-agent",
-          "version": "{version}",
-          "description": "Ingest processor that extracts information from a user agent",
-          "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
-          "has_native_controller": false
         }
       ],
       "modules": [

docs/reference/cluster/stats.asciidoc

@@ -186,13 +186,6 @@ Will return, for example:
           "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
           "has_native_controller": false
         },
-        {
-          "name": "ingest-user-agent",
-          "version": "{version}",
-          "description": "Ingest processor that extracts information from a user agent",
-          "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
-          "has_native_controller": false
-        },
         ...
       ],
       "network_types": {

docs/reference/ingest/ingest-node.asciidoc

@@ -1351,3 +1351,4 @@ include::processors/sort.asciidoc[]
 include::processors/trim.asciidoc[]
 include::processors/uppercase.asciidoc[]
 include::processors/url-decode.asciidoc[]
+include::processors/user-agent.asciidoc[]

docs/reference/ingest/processors/user-agent.asciidoc

@@ -0,0 +1,85 @@
+[[ingest-user-agent]]
+=== Ingest user agent processor
+
+The `user_agent` processor extracts details from the user agent string a browser sends with its web requests.
+This processor adds this information by default under the `user_agent` field.
+
+The ingest-user-agent module ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core.
+
+[[using-ingest-user-agent]]
+==== Using the user_agent Processor in a Pipeline
+
+[[ingest-user-agent-options]]
+.User-agent options
+[options="header"]
+|======
+| Name                   | Required  | Default                                                                                         | Description
+| `field`                | yes       | -                                                                                               | The field containing the user agent string.
+| `target_field`         | no        | user_agent                                                                                      | The field that will be filled with the user agent details.
+| `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
+| `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
+| `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+|======
+
+Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/user_agent
+{
+  "description" : "Add user agent information",
+  "processors" : [
+    {
+      "user_agent" : {
+        "field" : "agent"
+      }
+    }
+  ]
+}
+PUT my_index/_doc/my_id?pipeline=user_agent
+{
+  "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
+}
+GET my_index/_doc/my_id
+--------------------------------------------------
+// CONSOLE
+
+Which returns
+
+[source,js]
+--------------------------------------------------
+{
+  "found": true,
+  "_index": "my_index",
+  "_type": "_doc",
+  "_id": "my_id",
+  "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
+  "_source": {
+    "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
+    "user_agent": {
+      "name": "Chrome",
+      "major": "51",
+      "minor": "0",
+      "patch": "2704",
+      "os_name": "Mac OS X",
+      "os": "Mac OS X 10.10.5",
+      "os_major": "10",
+      "os_minor": "10",
+      "device": "Other"
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term": 1/"_primary_term" : $body._primary_term/]
+
+===== Using a custom regex file
+To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-user-agent` directory and
+has to have a `.yaml` filename extension. The file has to be present at node startup, any changes to it or any new files added
+while the node is running will not have any effect.
+
+In practice, it will make most sense for any custom regex file to be a variant of the default file, either a more recent version
+or a customised version.
+
+The default file included in `ingest-user-agent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml

modules/ingest-user-agent/build.gradle

@@ -23,5 +23,5 @@ esplugin {
 }
 
 integTestCluster {
-  extraConfigFile 'ingest-user-agent/test-regexes.yml', 'test/test-regexes.yml'
+  extraConfigFile 'ingest-user-agent/test-regexes.yml', 'src/test/test-regexes.yml'
 }

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

@@ -198,4 +198,3 @@ public class UserAgentProcessorTests extends ESTestCase {
         assertThat(target.get("device"), is("Other"));
     }
 }
-

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yml

@@ -1,4 +1,4 @@
-"ingest-user-agent plugin installed":
+"ingest-user-agent installed":
     - skip:
         reason: "contains is a newly added assertion"
         features: contains
@@ -10,5 +10,5 @@
     - do:
         nodes.info: {}
 
-    - contains:  { nodes.$master.plugins: { name: ingest-user-agent } }
+    - contains:  { nodes.$master.modules: { name: ingest-user-agent } }
     - contains:  { nodes.$master.ingest.processors: { type: user_agent } }

qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -231,10 +231,6 @@ fi
     install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-4.0.0.jar poi-ooxml-4.0.0.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
 }
 
-@test "[$GROUP] install ingest-user-agent plugin" {
-    install_and_check_plugin ingest user-agent
-}
-
 @test "[$GROUP] check ingest-common module" {
     check_module ingest-common jcodings-*.jar joni-*.jar
 }
@@ -243,6 +239,10 @@ fi
     check_module ingest-geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
 }
 
+@test "[$GROUP] check ingest-user-agent module" {
+    check_module ingest-user-agent
+}
+
 @test "[$GROUP] check lang-expression module" {
     # we specify the version on the asm-5.0.4.jar so that the test does
     # not spuriously pass if the jar is missing but the other asm jars
@@ -364,10 +364,6 @@ fi
     remove_plugin ingest-attachment
 }
 
-@test "[$GROUP] remove ingest-user-agent plugin" {
-    remove_plugin ingest-user-agent
-}
-
 @test "[$GROUP] remove murmur3 mapper plugin" {
     remove_plugin mapper-murmur3
 }