Make 7.x like 6.7 user agent ecs, but default to true (#38828)

Forward port of https://github.com/elastic/elasticsearch/pull/38757

This change reverts the initial 7.0 commits and replaces them
with the 6.7 variant that still allows for the ecs flag. 
This commit differs from the 6.7 variants in that ecs flag will 
now default to true. 

6.7: `ecs` : default `false`
7.x: `ecs` : default `true`
8.0: no option, but behaves as `true`

* Revert "Ingest node - user agent, move device to an object (#38115)"
This reverts commit 5b008a34aa.

* Revert "Add ECS schema for user-agent ingest processor (#37727) (#37984)"
This reverts commit cac6b8e06f.

* cherry-pick 5dfe1935345da3799931fd4a3ebe0b6aa9c17f57 
Add ECS schema for user-agent ingest processor (#37727)

* cherry-pick ec8ddc890a34853ee8db6af66f608b0ad0cd1099 
Ingest node - user agent, move device to an object (#38115) (#38121)
  
* cherry-pick f63cbdb9b426ba24ee4d987ca767ca05a22f2fbb (with manual merge fixes)
Dep. check for ECS changes to User Agent processor (#38362)

* make true the default for the ecs option, and update 7.0 references and tests

docs/reference/ingest/processors/user-agent.asciidoc

@@ -19,6 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab
 | `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
 | `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
 | `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+| `ecs`                  | no        | `true`                                                                                         | Whether to return the output in Elastic Common Schema format. NOTE: This setting is deprecated and will be removed in a future version.
 |======
 
 Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:

docs/reference/migration/migrate_7_0/settings.asciidoc

@@ -201,6 +201,6 @@ Elastic Stack to handle the indexing part.
 
 [float]
 [[ingest-user-agent-ecs-always]]
-==== Ingest User Agent processor always uses `ecs` output format
-The deprecated `ecs` setting for the user agent ingest processor has been
-removed. https://github.com/elastic/ecs[ECS] format is now the default.
+==== Ingest User Agent processor defaults uses `ecs` output format
+https://github.com/elastic/ecs[ECS] format is now the default.
+The `ecs` setting for the user agent ingest processor now defaults to true.

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -27,9 +27,11 @@ import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.useragent.UserAgentParser.Details;
 import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName;
 
+import java.lang.reflect.Field;
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -51,15 +53,17 @@ public class UserAgentProcessor extends AbstractProcessor {
     private final Set<Property> properties;
     private final UserAgentParser parser;
     private final boolean ignoreMissing;
+    private final boolean useECS;
 
     public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties,
-                              boolean ignoreMissing) {
+                              boolean ignoreMissing, boolean useECS) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.parser = parser;
         this.properties = properties;
         this.ignoreMissing = ignoreMissing;
+        this.useECS = useECS;
     }
 
     boolean isIgnoreMissing() {
@@ -80,6 +84,7 @@ public class UserAgentProcessor extends AbstractProcessor {
 
         Map<String, Object> uaDetails = new HashMap<>();
 
+        if (useECS) {
             // Parse the user agent in the ECS (Elastic Common Schema) format
             for (Property property : this.properties) {
                 switch (property) {
@@ -144,6 +149,72 @@ public class UserAgentProcessor extends AbstractProcessor {
                         break;
                 }
             }
+        } else {
+            // Deprecated format, removed in 8.0
+            for (Property property : this.properties) {
+                switch (property) {
+                    case NAME:
+                        if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
+                            uaDetails.put("name", uaClient.userAgent.name);
+                        } else {
+                            uaDetails.put("name", "Other");
+                        }
+                        break;
+                    case MAJOR:
+                        if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
+                            uaDetails.put("major", uaClient.userAgent.major);
+                        }
+                        break;
+                    case MINOR:
+                        if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
+                            uaDetails.put("minor", uaClient.userAgent.minor);
+                        }
+                        break;
+                    case PATCH:
+                        if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
+                            uaDetails.put("patch", uaClient.userAgent.patch);
+                        }
+                        break;
+                    case BUILD:
+                        if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
+                            uaDetails.put("build", uaClient.userAgent.build);
+                        }
+                        break;
+                    case OS:
+                        if (uaClient.operatingSystem != null) {
+                            uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
+                        } else {
+                            uaDetails.put("os", "Other");
+                        }
+
+                        break;
+                    case OS_NAME:
+                        if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
+                            uaDetails.put("os_name", uaClient.operatingSystem.name);
+                        } else {
+                            uaDetails.put("os_name", "Other");
+                        }
+                        break;
+                    case OS_MAJOR:
+                        if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
+                            uaDetails.put("os_major", uaClient.operatingSystem.major);
+                        }
+                        break;
+                    case OS_MINOR:
+                        if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
+                            uaDetails.put("os_minor", uaClient.operatingSystem.minor);
+                        }
+                        break;
+                    case DEVICE:
+                        if (uaClient.device != null && uaClient.device.name != null) {
+                            uaDetails.put("device", uaClient.device.name);
+                        } else {
+                            uaDetails.put("device", "Other");
+                        }
+                        break;
+                }
+            }
+        }
 
         ingestDocument.setFieldValue(targetField, uaDetails);
         return ingestDocument;
@@ -201,6 +272,10 @@ public class UserAgentProcessor extends AbstractProcessor {
         return parser;
     }
 
+    public boolean isUseECS() {
+        return useECS;
+    }
+
     public static final class Factory implements Processor.Factory {
 
         private final Map<String, UserAgentParser> userAgentParsers;
@@ -217,10 +292,7 @@ public class UserAgentProcessor extends AbstractProcessor {
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            Object ecsValue = config.remove("ecs");
-            if (ecsValue != null) {
-                deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
-            }
+            boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", true);
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
@@ -242,22 +314,53 @@ public class UserAgentProcessor extends AbstractProcessor {
                 properties = EnumSet.allOf(Property.class);
             }
 
-            return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing);
+            if (useECS == false) {
+                deprecationLogger.deprecated("setting [ecs] to false for non-common schema " +
+                    "format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format");
+            }
+
+            return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing, useECS);
         }
     }
 
     enum Property {
 
         NAME,
+        // Deprecated in 6.7 (superceded by VERSION), to be removed in 8.0
+        @Deprecated MAJOR,
+        @Deprecated MINOR,
+        @Deprecated PATCH,
         OS,
+        // Deprecated in 6.7 (superceded by just using OS), to be removed in 8.0
+        @Deprecated OS_NAME,
+        @Deprecated OS_MAJOR,
+        @Deprecated OS_MINOR,
         DEVICE,
+        @Deprecated BUILD, // Same deprecated as OS_* above
         ORIGINAL,
         VERSION;
 
+        private static Set<Property> DEPRECATED_PROPERTIES;
+
+        static {
+            Set<Property> deprecated = new HashSet<>();
+            for (Field field : Property.class.getFields()) {
+                if (field.isEnumConstant() && field.isAnnotationPresent(Deprecated.class)) {
+                    deprecated.add(valueOf(field.getName()));
+                }
+            }
+            DEPRECATED_PROPERTIES = deprecated;
+        }
+
         public static Property parseProperty(String propertyName) {
             try {
-                return valueOf(propertyName.toUpperCase(Locale.ROOT));
-            } catch (IllegalArgumentException e) {
+                Property value = valueOf(propertyName.toUpperCase(Locale.ROOT));
+                if (DEPRECATED_PROPERTIES.contains(value)) {
+                    deprecationLogger.deprecated("the [{}] property is deprecated for the user-agent processor", propertyName);
+                }
+                return value;
+            }
+            catch (IllegalArgumentException e) {
                 throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
                         Arrays.toString(EnumSet.allOf(Property.class).toArray()));
             }

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest.useragent;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.BeforeClass;
 
@@ -27,17 +28,21 @@ import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.lang.reflect.Field;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -84,12 +89,12 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         UserAgentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
-        assertThat(processor.getTargetField(), equalTo("user_agent"));
         assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
         assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
         assertFalse(processor.isIgnoreMissing());
+        assertTrue(processor.isUseECS());
     }
 
     public void testBuildWithIgnoreMissing() throws Exception {
@@ -98,6 +103,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("ignore_missing", true);
+        config.put("ecs", true);
 
         String processorTag = randomAlphaOfLength(10);
 
@@ -118,6 +124,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_target_field");
+        config.put("ecs", true);
 
         UserAgentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
@@ -130,6 +137,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("regex_file", regexWithoutDevicesFilename);
+        config.put("ecs", true);
 
         UserAgentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
@@ -155,8 +163,17 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         Set<UserAgentProcessor.Property> properties = EnumSet.noneOf(UserAgentProcessor.Property.class);
         List<String> fieldNames = new ArrayList<>();
         int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length);
+        Set<String> warnings = new HashSet<>();
+        Set<UserAgentProcessor.Property> deprecated = Arrays.stream(UserAgentProcessor.Property.class.getFields())
+            .filter(Field::isEnumConstant)
+            .filter(field -> field.isAnnotationPresent(Deprecated.class))
+            .map(field -> UserAgentProcessor.Property.valueOf(field.getName()))
+            .collect(Collectors.toSet());
         for (int i = 0; i < numFields; i++) {
             UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i];
+            if (deprecated.contains(property)) {
+                warnings.add("the [" + property.name().toLowerCase(Locale.ROOT) + "] property is deprecated for the user-agent processor");
+            }
             properties.add(property);
             fieldNames.add(property.name().toLowerCase(Locale.ROOT));
         }
@@ -164,10 +181,14 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
+        config.put("ecs", true);
 
         UserAgentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
+        if (warnings.size() > 0) {
+            assertWarnings(warnings.toArray(Strings.EMPTY_ARRAY));
+        }
     }
 
     public void testInvalidProperty() throws Exception {
@@ -178,8 +199,8 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("properties", Collections.singletonList("invalid"));
 
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
-        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " +
-            "ORIGINAL, VERSION]"));
+        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
+                + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD, ORIGINAL, VERSION]"));
     }
 
     public void testInvalidPropertiesType() throws Exception {

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

@@ -48,12 +48,12 @@ public class UserAgentProcessorTests extends ESTestCase {
         UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000));
 
         processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", parser,
-                EnumSet.allOf(UserAgentProcessor.Property.class), false);
+                EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
     }
 
     public void testNullValueWithIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), true, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("source_field", null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
@@ -63,7 +63,7 @@ public class UserAgentProcessorTests extends ESTestCase {
 
     public void testNonExistentWithIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), true, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         processor.execute(ingestDocument);
@@ -72,7 +72,7 @@ public class UserAgentProcessorTests extends ESTestCase {
 
     public void testNullWithoutIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("source_field", null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
@@ -82,7 +82,7 @@ public class UserAgentProcessorTests extends ESTestCase {
 
     public void testNonExistentWithoutIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
@@ -186,6 +186,7 @@ public class UserAgentProcessorTests extends ESTestCase {
         assertNull(target.get("build"));
 
         assertNull(target.get("os"));
+
         Map<String, String> device = new HashMap<>();
         device.put("name", "Other");
         assertThat(target.get("device"), is(device));

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml

@@ -46,6 +46,7 @@
               {
                 "user_agent" : {
                   "field" : "field1",
+                  "ecs": true,
                   "target_field": "field2",
                   "properties": ["os"]
                 }
@@ -69,5 +70,62 @@
   - match: { _source.field2.os.full: "Mac OS X 10.9.2" }
   - is_false: _source.user_agent
   - is_false: _source.field2.name
+  - is_false: _source.field2.os_name
+  - is_false: _source.field2.os_major
+  - is_false: _source.field2.os_minor
+  - is_false: _source.field2.major
+  - is_false: _source.field2.minor
+  - is_false: _source.field2.patch
+  - is_false: _source.field2.device
+
+---
+"Test user agent processor with non-ECS schema":
+  - skip:
+      features: warnings
+
+  - do:
+      warnings:
+        - "setting [ecs] to false for non-common schema format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format"
+        - "the [os_major] property is deprecated for the user-agent processor"
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "user_agent" : {
+                  "field" : "field1",
+                  "ecs": false,
+                  "target_field": "field2",
+                  "properties": ["os", "os_major"]
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
+  - match: { _source.field2.os: "Mac OS X 10.9.2" }
+  - match: { _source.field2.os_major: "10" }
+  - is_false: _source.user_agent
+  - is_false: _source.field2.name
+  - is_false: _source.field2.os_name
+  - is_false: _source.field2.os_minor
+  - is_false: _source.field2.major
+  - is_false: _source.field2.minor
+  - is_false: _source.field2.patch
   - is_false: _source.field2.device
-  - is_false: _source.field2.original

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml

@@ -10,6 +10,7 @@
               {
                 "user_agent" : {
                   "field": "field1",
+                  "ecs": true,
                   "regex_file": "test-regexes.yml"
                 }
               }

server/src/main/java/org/elasticsearch/ingest/IngestService.java

@@ -343,7 +343,7 @@ public class IngestService implements ClusterStateApplier {
         return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor));
     }
 
-    static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
+    public static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
         IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE);
         Map<String, PipelineConfiguration> pipelines;
         if (currentIngestMetadata != null) {

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.ingest.PipelineConfiguration;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class ClusterDeprecationChecks {
+
+    @SuppressWarnings("unchecked")
+    static DeprecationIssue checkUserAgentPipelines(ClusterState state) {
+        List<PipelineConfiguration> pipelines = IngestService.getPipelines(state);
+
+        List<String> pipelinesWithDeprecatedEcsConfig = pipelines.stream()
+            .filter(Objects::nonNull)
+            .filter(pipeline -> {
+                Map<String, Object> pipelineConfig = pipeline.getConfigAsMap();
+
+                List<Map<String, Map<String, Object>>> processors =
+                    (List<Map<String, Map<String, Object>>>) pipelineConfig.get("processors");
+                return processors.stream()
+                    .filter(Objects::nonNull)
+                    .filter(processor -> processor.containsKey("user_agent"))
+                    .map(processor -> processor.get("user_agent"))
+                    .anyMatch(processorConfig -> processorConfig.containsKey("ecs"));
+            })
+            .map(PipelineConfiguration::getId)
+            .sorted() // Make the warning consistent for testing purposes
+            .collect(Collectors.toList());
+        if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) {
+            return new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                "User-Agent ingest plugin will always use ECS-formatted output",
+                "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
+                    "#ingest-user-agent-ecs-always",
+                "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " uses the [ecs] option which needs to be removed to work in 8.0");
+        }
+        return null;
+
+    }
+}

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -31,7 +31,10 @@ public class DeprecationChecks {
     }
 
     static List<Function<ClusterState, DeprecationIssue>> CLUSTER_SETTINGS_CHECKS =
-        Collections.emptyList();
+        Collections.unmodifiableList(Arrays.asList(
+            ClusterDeprecationChecks::checkUserAgentPipelines
+        ));
+
 
     static List<BiFunction<Settings, PluginsAndModules, DeprecationIssue>> NODE_SETTINGS_CHECKS =
         Collections.unmodifiableList(Arrays.asList(

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS;
+
+public class ClusterDeprecationChecksTests extends ESTestCase {
+
+    public void testUserAgentEcsCheck() {
+        PutPipelineRequest ecsFalseRequest = new PutPipelineRequest("ecs_false",
+            new BytesArray("{\n" +
+                "  \"description\" : \"This has ecs set to false\",\n" +
+                "  \"processors\" : [\n" +
+                "    {\n" +
+                "      \"user_agent\" : {\n" +
+                "        \"field\" : \"agent\",\n" +
+                "        \"ecs\" : false\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}"), XContentType.JSON);
+        PutPipelineRequest ecsNullRequest = new PutPipelineRequest("ecs_null",
+            new BytesArray("{\n" +
+                "  \"description\" : \"This has ecs set to false\",\n" +
+                "  \"processors\" : [\n" +
+                "    {\n" +
+                "      \"user_agent\" : {\n" +
+                "        \"field\" : \"agent\"\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}"), XContentType.JSON);
+        PutPipelineRequest ecsTrueRequest = new PutPipelineRequest("ecs_true",
+            new BytesArray("{\n" +
+                "  \"description\" : \"This has ecs set to false\",\n" +
+                "  \"processors\" : [\n" +
+                "    {\n" +
+                "      \"user_agent\" : {\n" +
+                "        \"field\" : \"agent\",\n" +
+                "        \"ecs\" : true\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}"), XContentType.JSON);
+
+        ClusterState state = ClusterState.builder(new ClusterName("test")).build();
+        state = IngestService.innerPut(ecsTrueRequest, state);
+        state = IngestService.innerPut(ecsFalseRequest, state);
+        state = IngestService.innerPut(ecsNullRequest, state);
+
+        final ClusterState finalState = state;
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState));
+
+        DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING,
+            "User-Agent ingest plugin will always use ECS-formatted output",
+            "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
+                "#ingest-user-agent-ecs-always",
+            "Ingest pipelines [ecs_false, ecs_true] uses the [ecs] option which needs to be removed to work in 8.0");
+        assertEquals(singletonList(expected), issues);
+    }
+}