honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML][Transform] Filter null objects from field caps request (#62945) (#62971) 

If the transform grouping is a script then exclude the field from the source index
mappings fields caps request. A null object caused an NPE in the serialisation of 
FieldCapabilitiesIndexRequest.
This commit is contained in:
David Kyle 2020-09-29 09:07:01 +01:00 committed by GitHub
parent 7f6c1ff5b4
commit f23603dafd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -77,6 +78,11 @@ public final class SchemaUtil {
.getGroups()
.forEach(
(destinationFieldName, group) -> {
// skip any fields that use scripts as there will be no source mapping
if (group.getScriptConfig() != null) {
return;
}
// We will always need the field name for the grouping to create the mapping
fieldNamesForGrouping.put(destinationFieldName, group.getField());
// Sometimes the group config will supply a desired mapping as well
@ -107,7 +113,7 @@ public final class SchemaUtil {
getSourceFieldMappings(
client,
source,
allFieldNames.values().toArray(new String[0]),
allFieldNames.values().stream().filter(Objects::nonNull).toArray(String[]::new),
ActionListener.wrap(
sourceMappings -> listener.onResponse(
resolveMappings(
@ -182,7 +188,8 @@ public final class SchemaUtil {
} else if (destinationMapping != null) {
targetMapping.put(targetFieldName, destinationMapping);
} else {
logger.warn("Failed to deduce mapping for [{}], fall back to dynamic mapping.", targetFieldName);
logger.warn("Failed to deduce mapping for [{}], fall back to dynamic mapping. " +
"Create the destination index with complete mappings first to avoid deducing the mappings", targetFieldName);
}
});
@ -192,7 +199,8 @@ public final class SchemaUtil {
if (destinationMapping != null) {
targetMapping.put(targetFieldName, destinationMapping);
} else {
logger.warn("Failed to deduce mapping for [{}], fall back to keyword.", targetFieldName);
logger.warn("Failed to deduce mapping for [{}], fall back to keyword. " +
"Create the destination index with complete mappings first to avoid deducing the mappings", targetFieldName);
targetMapping.put(targetFieldName, KeywordFieldMapper.CONTENT_TYPE);
}
});

8
x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java
View File

@ -127,11 +127,13 @@ public class AggregationSchemaAndResultTests extends ESTestCase {
AggregationConfig aggregationConfig = new AggregationConfig(Collections.emptyMap(), aggs);
GroupConfig groupConfig = GroupConfigTests.randomGroupConfig();
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggregationConfig, null);
long numGroupsWithoutScripts = groupConfig.getGroups().values().stream()
.filter(singleGroupSource -> singleGroupSource.getScriptConfig() == null).count();
this.<Map<String, String>>assertAsync(
listener -> SchemaUtil.deduceMappings(client, pivotConfig, new String[] { "source-index" }, listener),
mappings -> {
assertEquals(groupConfig.getGroups().size() + 10, mappings.size());
assertEquals(numGroupsWithoutScripts + 10, mappings.size());
assertEquals("long", mappings.get("max_rating"));
assertEquals("double", mappings.get("avg_rating"));
assertEquals("long", mappings.get("count_rating"));
@ -189,11 +191,13 @@ public class AggregationSchemaAndResultTests extends ESTestCase {
AggregationConfig aggregationConfig = new AggregationConfig(Collections.emptyMap(), aggs);
GroupConfig groupConfig = GroupConfigTests.randomGroupConfig();
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggregationConfig, null);
long numGroupsWithoutScripts = groupConfig.getGroups().values().stream()
.filter(singleGroupSource -> singleGroupSource.getScriptConfig() == null).count();
this.<Map<String, String>>assertAsync(
listener -> SchemaUtil.deduceMappings(client, pivotConfig, new String[] { "source-index" }, listener),
mappings -> {
assertEquals(groupConfig.getGroups().size() + 12, mappings.size());
assertEquals(numGroupsWithoutScripts + 12, mappings.size());
assertEquals("long", mappings.get("filter_1"));
assertEquals("object", mappings.get("filter_2"));
assertEquals("long", mappings.get("filter_2.max_drinks_2"));