honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x][ML] DFA jobs should accept excluding an unsupported field (#49535) (#49544) 

Before this change excluding an unsupported field resulted in
an error message that explained the excluded field could not be
detected as if it doesn't exist. This error message is confusing.

This commit commit changes this so that there is no error in this
scenario. When excluding a field that does exist but has been
automatically been excluded from the analysis there is no harm
(unlike excluding a missing field which could be a typo).

Backport of #49535
This commit is contained in:
Dimitris Athanasiou 2019-11-25 15:13:00 +02:00 committed by GitHub
parent 8c374014ae
commit aca38f6882
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 37 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
View File

@ -203,7 +203,7 @@ public class ExtractedFieldsDetector {
.expand(includes, false);
// If the exclusion set does not match anything, that means the fields are already not present
// no need to raise if nothing matched
Set<String> excludedSet = NameResolver.newUnaliased(fields,
Set<String> excludedSet = NameResolver.newUnaliased(fieldCapabilitiesResponse.get().keySet(),
(ex) -> new ResourceNotFoundException(
Messages.getMessage(Messages.DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER, ex)))
.expand(excludes, true);

36
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
View File

@ -300,6 +300,42 @@ public class ExtractedFieldsDetectorTests extends ESTestCase {
assertThat(e.getMessage(), equalTo("No field [_id] could be detected"));
}
public void testDetect_GivenExcludedFieldIsMissing() {
FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
.addAggregatableField("foo", "float")
.build();
FetchSourceContext analyzedFields = new FetchSourceContext(true, new String[]{"*"}, new String[] {"bar"});
ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
SOURCE_INDEX, buildOutlierDetectionConfig(analyzedFields), false, 100, fieldCapabilities, Collections.emptyMap());
ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
assertThat(e.getMessage(), equalTo("No field [bar] could be detected"));
}
public void testDetect_GivenExcludedFieldIsUnsupported() {
FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
.addAggregatableField("numeric", "float")
.addAggregatableField("categorical", "keyword")
.build();
FetchSourceContext analyzedFields = new FetchSourceContext(true, null, new String[] {"categorical"});
ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
SOURCE_INDEX, buildOutlierDetectionConfig(analyzedFields), false, 100, fieldCapabilities, Collections.emptyMap());
Tuple<ExtractedFields, List<FieldSelection>> fieldExtraction = extractedFieldsDetector.detect();
List<ExtractedField> allFields = fieldExtraction.v1().getAllFields();
assertThat(allFields.size(), equalTo(1));
assertThat(allFields.get(0).getName(), equalTo("numeric"));
assertFieldSelectionContains(fieldExtraction.v2(),
FieldSelection.excluded("categorical", Collections.singleton("keyword"),
"unsupported type; supported types are [boolean, byte, double, float, half_float, integer, long, scaled_float, short]"),
FieldSelection.included("numeric", Collections.singleton("float"), false, FieldSelection.FeatureType.NUMERICAL)
);
}
public void testDetect_ShouldSortFieldsAlphabetically() {
int fieldCount = randomIntBetween(10, 20);
List<String> fields = new ArrayList<>();