From 73add726d758f6b7d1d23e0e9cbc8c7da67f0045 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 16 Dec 2019 11:30:06 +0000
Subject: [PATCH] [7.x][ML] Fix exception when field is not included and
 excluded at the same time (#50192) (#50223)

Executing the data frame analytics _explain API with a config that contains
a field that is not in the includes list but at the same time is the excludes
list results to trying to remove the field twice from the iterator. That causes
an `IllegalStateException`. This commit fixes this issue and adds a test that
captures the scenario.

Backport of #50192
---
 .../extractor/ExtractedFieldsDetector.java    |  8 +++----
 .../ExtractedFieldsDetectorTests.java         | 21 +++++++++++++++++++
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
index 82bbecbd358..b4bc63f5c06 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
@@ -235,14 +235,14 @@ public class ExtractedFieldsDetector {
                 if (IGNORE_FIELDS.contains(field)) {
                     throw ExceptionsHelper.badRequestException("field [{}] cannot be analyzed", field);
                 }
+                if (excludes.contains(field)) {
+                    fieldsIterator.remove();
+                    addExcludedField(field, "field in excludes list", fieldSelection);
+                }
             } else {
                 fieldsIterator.remove();
                 addExcludedField(field, "field not in includes list", fieldSelection);
             }
-            if (excludes.contains(field)) {
-                fieldsIterator.remove();
-                addExcludedField(field, "field in excludes list", fieldSelection);
-            }
         }
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
index 9c55b2a9ac9..6b882d03f29 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
@@ -233,6 +233,27 @@ public class ExtractedFieldsDetectorTests extends ESTestCase {
         );
     }
 
+    public void testDetect_GivenFieldIsNotIncludedAndIsExcluded() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("foo", "float")
+            .addAggregatableField("bar", "float")
+            .build();
+        analyzedFields = new FetchSourceContext(true, new String[]  {"foo"}, new String[] {"bar"});
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildOutlierDetectionConfig(), false, 100, fieldCapabilities, Collections.emptyMap());
+        Tuple<ExtractedFields, List<FieldSelection>> fieldExtraction = extractedFieldsDetector.detect();
+
+        List<ExtractedField> allFields = fieldExtraction.v1().getAllFields();
+        assertThat(allFields, hasSize(1));
+        assertThat(allFields.stream().map(ExtractedField::getName).collect(Collectors.toList()), contains("foo"));
+
+        assertFieldSelectionContains(fieldExtraction.v2(),
+            FieldSelection.excluded("bar", Collections.singleton("float"), "field not in includes list"),
+            FieldSelection.included("foo", Collections.singleton("float"), false, FieldSelection.FeatureType.NUMERICAL)
+        );
+    }
+
     public void testDetect_GivenRegressionAndRequiredFieldHasInvalidType() {
         FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
             .addAggregatableField("some_float", "float")