honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adds support for date_nanos in Rollup Metric and DateHistogram Configs (#59349) (#59577) 

Closes #44505.
This commit is contained in:
Tal Levy 2020-07-14 22:37:48 -07:00 committed by GitHub
parent 2dd086445c
commit 4bb91b61e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 96 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java
View File

@ -61,7 +61,8 @@ public class RollupField {
NUMERIC_FIELD_MAPPER_TYPES = types;
}
public static final String DATE_FIELD_MAPPER_TYPE = DateFieldMapper.CONTENT_TYPE;
public static final List<String> DATE_FIELD_MAPPER_TYPES = Arrays.asList(DateFieldMapper.CONTENT_TYPE,
DateFieldMapper.DATE_NANOS_CONTENT_TYPE);
/**
* Format to the appropriate Rollup field name convention

34
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java
View File

@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.xpack.core.rollup.RollupField;
import java.io.IOException;
import java.time.ZoneId;
@ -314,24 +315,31 @@ public class DateHistogramGroupConfig implements Writeable, ToXContentObject {
public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
ActionRequestValidationException validationException) {
Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field);
if (fieldCaps != null && fieldCaps.isEmpty() == false) {
if (fieldCaps.containsKey("date") && fieldCaps.size() == 1) {
if (fieldCaps.get("date").isAggregatable()) {
return;
} else {
validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
"but is not.");
boolean matchesDateType = false;
for (String dateType : RollupField.DATE_FIELD_MAPPER_TYPES) {
if (fieldCaps.containsKey(dateType) && fieldCaps.size() == 1) {
matchesDateType |= true;
if (fieldCaps.get(dateType).isAggregatable()) {
return;
} else {
validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
"but is not.");
}
}
} else {
validationException.addValidationError("The field referenced by a date_histo group must be a [date] type across all " +
"indices in the index pattern. Found: " + fieldCaps.keySet().toString() + " for field [" + field + "]");
}
}
validationException.addValidationError("Could not find a [date] field with name [" + field + "] in any of the indices matching " +
if (matchesDateType == false) {
validationException.addValidationError("The field referenced by a date_histo group must be one of type [" +
Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] across all " +
"indices in the index pattern. Found: " + fieldCaps.keySet().toString() + " for field [" + field + "]");
}
} else {
validationException.addValidationError("Could not find one of [" +
Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] fields with name [" +
field + "] in any of the indices matching " +
"the index pattern.");
}
}
@Override

12
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
View File

@ -115,19 +115,21 @@ public class MetricConfig implements Writeable, ToXContentObject {
}
if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) {
// nothing to do as all metrics are supported by SUPPORTED_NUMERIC_METRICS currently
} else if (RollupField.DATE_FIELD_MAPPER_TYPE.equals(key)) {
} else if (RollupField.DATE_FIELD_MAPPER_TYPES.contains(key)) {
if (RollupField.SUPPORTED_DATE_METRICS.containsAll(metrics) == false) {
validationException.addValidationError(
buildSupportedMetricError("date", RollupField.SUPPORTED_DATE_METRICS));
buildSupportedMetricError(key, RollupField.SUPPORTED_DATE_METRICS));
}
} else {
validationException.addValidationError("The field referenced by a metric group must be a [numeric] or [date] type, " +
validationException.addValidationError("The field referenced by a metric group must be a [numeric] or [" +
Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] type, " +
"but found " + fieldCaps.keySet().toString() + " for field [" + field + "]");
}
});
} else {
validationException.addValidationError("Could not find a [numeric] or [date] field with name [" + field + "] in any of the " +
"indices matching the index pattern.");
validationException.addValidationError("Could not find a [numeric] or [" +
Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) +
"] field with name [" + field + "] in any of the " + "indices matching the index pattern.");
}
}

32
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;
@ -29,6 +30,7 @@ import java.util.stream.IntStream;
import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
import static com.carrotsearch.randomizedtesting.generators.RandomPicks.randomFrom;
import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween;
import static org.elasticsearch.test.ESTestCase.randomSubsetOf;
import static org.elasticsearch.test.ESTestCase.randomZone;
public class ConfigTestHelpers {
@ -69,7 +71,10 @@ public class ConfigTestHelpers {
}
public static DateHistogramGroupConfig randomDateHistogramGroupConfig(final Random random) {
final String field = randomField(random);
return randomDateHistogramGroupConfigWithField(random, randomField(random));
}
public static DateHistogramGroupConfig randomDateHistogramGroupConfigWithField(final Random random, final String field) {
final DateHistogramInterval delay = random.nextBoolean() ? randomInterval() : null;
final String timezone = random.nextBoolean() ? randomZone().getId() : null;
if (random.nextBoolean()) {
@ -128,26 +133,11 @@ public class ConfigTestHelpers {
public static MetricConfig randomMetricConfig(final Random random) {
final String field = randomAsciiAlphanumOfLengthBetween(random, 15, 25); // large names so we don't accidentally collide
final List<String> metrics = new ArrayList<>();
if (random.nextBoolean()) {
metrics.add("min");
}
if (random.nextBoolean()) {
metrics.add("max");
}
if (random.nextBoolean()) {
metrics.add("sum");
}
if (random.nextBoolean()) {
metrics.add("avg");
}
if (random.nextBoolean()) {
metrics.add("value_count");
}
if (metrics.size() == 0) {
metrics.add("min");
}
return new MetricConfig(field, Collections.unmodifiableList(metrics));
return randomMetricConfigWithFieldAndMetrics(random, field, RollupField.SUPPORTED_METRICS);
}
public static MetricConfig randomMetricConfigWithFieldAndMetrics(final Random random, String field, Collection<String> metrics) {
return new MetricConfig(field, Collections.unmodifiableList(randomSubsetOf(randomIntBetween(random, 1, metrics.size()), metrics)));
}
public static TermsGroupConfig randomTermsGroupConfig(final Random random) {

16
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
View File

@ -68,8 +68,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
new DateHistogramInterval("1d"), null, null);
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " +
"indices matching the index pattern."));
assertThat(e.validationErrors().get(0), equalTo("Could not find one of [date,date_nanos] fields with name [my_field] in " +
"any of the indices matching the index pattern."));
}
public void testValidateNomatchingField() {
@ -83,8 +83,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
new DateHistogramInterval("1d"), null, null);
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " +
"indices matching the index pattern."));
assertThat(e.validationErrors().get(0), equalTo("Could not find one of [date,date_nanos] fields with name [my_field] in " +
"any of the indices matching the index pattern."));
}
public void testValidateFieldWrongType() {
@ -98,8 +98,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
new DateHistogramInterval("1d"), null, null);
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " +
"indices in the index pattern. Found: [keyword] for field [my_field]"));
assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be one of type " +
"[date,date_nanos] across all indices in the index pattern. Found: [keyword] for field [my_field]"));
}
public void testValidateFieldMixtureTypes() {
@ -116,8 +116,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
new DateHistogramInterval("1d"), null, null);
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " +
"indices in the index pattern. Found: [date, keyword] for field [my_field]"));
assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be one of type " +
"[date,date_nanos] across all indices in the index pattern. Found: [date, keyword] for field [my_field]"));
}
public void testValidateFieldMatchingNotAggregatable() {

127
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
View File

@ -14,10 +14,11 @@ import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
import org.elasticsearch.xpack.core.rollup.RollupField;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static java.util.Collections.singletonList;
import static org.hamcrest.Matchers.equalTo;
@ -49,11 +50,11 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
MetricConfig config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date] field with name [my_field] in any" +
" of the indices matching the index pattern."));
assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date,date_nanos] field with name [my_field] " +
"in any of the indices matching the index pattern."));
}
public void testValidateNomatchingField() {
public void testValidateNoMatchingField() {
ActionRequestValidationException e = new ActionRequestValidationException();
Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
@ -63,8 +64,8 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
MetricConfig config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date] field with name [my_field] in any" +
" of the indices matching the index pattern."));
assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date,date_nanos] field with name [my_field] " +
"in any of the indices matching the index pattern."));
}
public void testValidateFieldWrongType() {
@ -77,7 +78,7 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
MetricConfig config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat("The field referenced by a metric group must be a [numeric] or [date] type," +
assertThat("The field referenced by a metric group must be a [numeric] or [date,date_nanos] type," +
" but found [keyword] for field [my_field]", is(in(e.validationErrors())));
}
@ -95,90 +96,54 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
}
public void testValidateDateFieldUnsupportedMetric() {
ActionRequestValidationException e = new ActionRequestValidationException();
public void testValidateDateFieldsUnsupportedMetric() {
Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
// Have to mock fieldcaps because the ctor's aren't public...
FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("date", fieldCaps));
for (String mappingType : RollupField.DATE_FIELD_MAPPER_TYPES) {
// Have to mock fieldcaps because the ctor's aren't public...
FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap(mappingType, fieldCaps));
Set<String> unsupportedMetrics = new HashSet<>(RollupField.SUPPORTED_METRICS);
unsupportedMetrics.removeAll(RollupField.SUPPORTED_DATE_METRICS);
for (String unsupportedMetric : unsupportedMetrics) {
MetricConfig config = new MetricConfig("my_field", Collections.singletonList(unsupportedMetric));
ActionRequestValidationException e = new ActionRequestValidationException();
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("Only the metrics " + RollupField.SUPPORTED_DATE_METRICS.toString() +
" are supported for [" + mappingType + "] types, but unsupported metrics [" + unsupportedMetric +
"] supplied for field [my_field]"));
}
}
MetricConfig config = new MetricConfig("my_field", Arrays.asList("avg", "max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().get(0), equalTo("Only the metrics " + RollupField.SUPPORTED_DATE_METRICS.toString() +
" are supported for [date] types, but unsupported metrics [avg] supplied for field [my_field]"));
}
public void testValidateMatchingField() {
ActionRequestValidationException e = new ActionRequestValidationException();
Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
// Have to mock fieldcaps because the ctor's aren't public...
FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("long", fieldCaps));
for (String numericType : RollupField.NUMERIC_FIELD_MAPPER_TYPES) {
// Have to mock fieldcaps because the ctor's aren't public...
FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap(numericType, fieldCaps));
MetricConfig config = ConfigTestHelpers
.randomMetricConfigWithFieldAndMetrics(random(), "my_field", RollupField.SUPPORTED_NUMERIC_METRICS);
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
}
MetricConfig config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("double", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("float", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("short", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("byte", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("half_float", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("scaled_float", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("integer", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap("date", fieldCaps));
config = new MetricConfig("my_field", singletonList("max"));
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
for (String dateType : RollupField.DATE_FIELD_MAPPER_TYPES) {
// Have to mock fieldcaps because the ctor's aren't public...
FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
when(fieldCaps.isAggregatable()).thenReturn(true);
responseMap.put("my_field", Collections.singletonMap(dateType, fieldCaps));
MetricConfig config = ConfigTestHelpers
.randomMetricConfigWithFieldAndMetrics(random(), "my_field", RollupField.SUPPORTED_DATE_METRICS);
config.validateMappings(responseMap, e);
assertThat(e.validationErrors().size(), equalTo(0));
}
}
}

2
x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
View File

@ -209,7 +209,7 @@ setup:
"Validation failures":
- do:
catch: /Could not find a \[numeric\] or \[date\] field with name \[field_doesnt_exist\] in any of the indices matching the index pattern/
catch: /Could not find a \[numeric\] or \[date,date_nanos\] field with name \[field_doesnt_exist\] in any of the indices matching the index pattern/
headers:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
rollup.put_job: