[Rollup] Histo group config should support scaled_floats (#32048)

Metric config already whitelist scaled_floats, but it wasn't added to
the histo group config.  This centralizes the mapping types map
so that both metrics and histo (and any future configs) use the same
map.

Fixes #32035
This commit is contained in:
Zachary Tong 2018-07-13 16:10:39 -04:00 committed by GitHub
parent 42ca520377
commit 867c49600c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 18 additions and 20 deletions

View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.core.rollup; package org.elasticsearch.xpack.core.rollup;
import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseField;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
@ -15,6 +16,8 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuil
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class RollupField { public class RollupField {
// Fields that are used both in core Rollup actions and Rollup plugin // Fields that are used both in core Rollup actions and Rollup plugin
@ -34,6 +37,16 @@ public class RollupField {
public static final List<String> SUPPORTED_METRICS = Arrays.asList(MaxAggregationBuilder.NAME, MinAggregationBuilder.NAME, public static final List<String> SUPPORTED_METRICS = Arrays.asList(MaxAggregationBuilder.NAME, MinAggregationBuilder.NAME,
SumAggregationBuilder.NAME, AvgAggregationBuilder.NAME, ValueCountAggregationBuilder.NAME); SumAggregationBuilder.NAME, AvgAggregationBuilder.NAME, ValueCountAggregationBuilder.NAME);
// these mapper types are used by the configs (metric, histo, etc) to validate field mappings
public static final List<String> NUMERIC_FIELD_MAPPER_TYPES;
static {
List<String> types = Stream.of(NumberFieldMapper.NumberType.values())
.map(NumberFieldMapper.NumberType::typeName)
.collect(Collectors.toList());
types.add("scaled_float"); // have to add manually since scaled_float is in a module
NUMERIC_FIELD_MAPPER_TYPES = types;
}
/** /**
* Format to the appropriate Rollup field name convention * Format to the appropriate Rollup field name convention
* *

View File

@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
import org.elasticsearch.search.aggregations.bucket.composite.HistogramValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.composite.HistogramValuesSourceBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
@ -30,7 +29,6 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
/** /**
* The configuration object for the histograms in the rollup config * The configuration object for the histograms in the rollup config
@ -51,10 +49,6 @@ public class HistoGroupConfig implements Writeable, ToXContentFragment {
private static final ParseField INTERVAL = new ParseField("interval"); private static final ParseField INTERVAL = new ParseField("interval");
private static final ParseField FIELDS = new ParseField("fields"); private static final ParseField FIELDS = new ParseField("fields");
private static final List<String> MAPPER_TYPES = Stream.of(NumberFieldMapper.NumberType.values())
.map(NumberFieldMapper.NumberType::typeName)
.collect(Collectors.toList());
private final long interval; private final long interval;
private final String[] fields; private final String[] fields;
@ -126,7 +120,7 @@ public class HistoGroupConfig implements Writeable, ToXContentFragment {
Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field); Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field);
if (fieldCaps != null && fieldCaps.isEmpty() == false) { if (fieldCaps != null && fieldCaps.isEmpty() == false) {
fieldCaps.forEach((key, value) -> { fieldCaps.forEach((key, value) -> {
if (MAPPER_TYPES.contains(key)) { if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) {
if (value.isAggregatable() == false) { if (value.isAggregatable() == false) {
validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " + validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
"but is not."); "but is not.");

View File

@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
@ -32,7 +31,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
/** /**
* The configuration object for the metrics portion of a rollup job config * The configuration object for the metrics portion of a rollup job config
@ -66,15 +64,6 @@ public class MetricConfig implements Writeable, ToXContentFragment {
private static final ParseField AVG = new ParseField("avg"); private static final ParseField AVG = new ParseField("avg");
private static final ParseField VALUE_COUNT = new ParseField("value_count"); private static final ParseField VALUE_COUNT = new ParseField("value_count");
private static final List<String> MAPPER_TYPES;
static {
List<String> types = Stream.of(NumberFieldMapper.NumberType.values())
.map(NumberFieldMapper.NumberType::typeName)
.collect(Collectors.toList());
types.add("scaled_float"); // have to add manually since scaled_float is in a module
MAPPER_TYPES = types;
}
public static final ObjectParser<MetricConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, MetricConfig.Builder::new); public static final ObjectParser<MetricConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, MetricConfig.Builder::new);
static { static {
@ -153,7 +142,7 @@ public class MetricConfig implements Writeable, ToXContentFragment {
Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field); Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field);
if (fieldCaps != null && fieldCaps.isEmpty() == false) { if (fieldCaps != null && fieldCaps.isEmpty() == false) {
fieldCaps.forEach((key, value) -> { fieldCaps.forEach((key, value) -> {
if (MAPPER_TYPES.contains(key)) { if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) {
if (value.isAggregatable() == false) { if (value.isAggregatable() == false) {
validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " + validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
"but is not."); "but is not.");

View File

@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.AbstractSerializingTestCase;
import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
import org.elasticsearch.xpack.core.rollup.RollupField;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
@ -111,7 +112,8 @@ public class HistoGroupConfigSerializingTests extends AbstractSerializingTestCas
// Have to mock fieldcaps because the ctor's aren't public... // Have to mock fieldcaps because the ctor's aren't public...
FieldCapabilities fieldCaps = mock(FieldCapabilities.class); FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true); when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("long", fieldCaps)); String mappingType = randomFrom(RollupField.NUMERIC_FIELD_MAPPER_TYPES);
responseMap.put("my_field", Collections.singletonMap(mappingType, fieldCaps));
HistoGroupConfig config = new HistoGroupConfig.Builder() HistoGroupConfig config = new HistoGroupConfig.Builder()
.setFields(Collections.singletonList("my_field")) .setFields(Collections.singletonList("my_field"))