honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removes two unused AnalysisConfig options (#35645) 

* ML: Removing result_finalization_window && overlapping_buckets

* Reverting bad method deletions

* Setting to current before backport to try and get a green build

* fixing testBuildAutodetectCommand test

* disabling bwc tests for backport
This commit is contained in:
Benjamin Trent 2018-11-19 08:29:53 -06:00 committed by GitHub
parent 450db7fcf7
commit 6d4a3f8fce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 56 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/AnalysisConfig.java
View File

@ -60,8 +60,6 @@ public class AnalysisConfig implements ToXContentObject {
public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
public static final ParseField DETECTORS = new ParseField("detectors");
public static final ParseField INFLUENCERS = new ParseField("influencers");
public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");
@SuppressWarnings("unchecked")
@ -84,8 +82,6 @@ public class AnalysisConfig implements ToXContentObject {
builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS);
PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
}
@ -100,14 +96,11 @@ public class AnalysisConfig implements ToXContentObject {
private final String summaryCountFieldName;
private final List<Detector> detectors;
private final List<String> influencers;
private final Boolean overlappingBuckets;
private final Long resultFinalizationWindow;
private final Boolean multivariateByFields;
private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
Boolean multivariateByFields) {
List<Detector> detectors, List<String> influencers, Boolean multivariateByFields) {
this.detectors = Collections.unmodifiableList(detectors);
this.bucketSpan = bucketSpan;
this.latency = latency;
@ -116,8 +109,6 @@ public class AnalysisConfig implements ToXContentObject {
this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
this.summaryCountFieldName = summaryCountFieldName;
this.influencers = Collections.unmodifiableList(influencers);
this.overlappingBuckets = overlappingBuckets;
this.resultFinalizationWindow = resultFinalizationWindow;
this.multivariateByFields = multivariateByFields;
}
@ -177,14 +168,6 @@ public class AnalysisConfig implements ToXContentObject {
return influencers;
}
public Boolean getOverlappingBuckets() {
return overlappingBuckets;
}
public Long getResultFinalizationWindow() {
return resultFinalizationWindow;
}
public Boolean getMultivariateByFields() {
return multivariateByFields;
}
@ -255,12 +238,6 @@ public class AnalysisConfig implements ToXContentObject {
}
builder.endArray();
builder.field(INFLUENCERS.getPreferredName(), influencers);
if (overlappingBuckets != null) {
builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
}
if (resultFinalizationWindow != null) {
builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
}
if (multivariateByFields != null) {
builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
}
@ -287,8 +264,6 @@ public class AnalysisConfig implements ToXContentObject {
Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
Objects.equals(detectors, that.detectors) &&
Objects.equals(influencers, that.influencers) &&
Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
Objects.equals(multivariateByFields, that.multivariateByFields);
}
@ -296,8 +271,7 @@ public class AnalysisConfig implements ToXContentObject {
public int hashCode() {
return Objects.hash(
bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
multivariateByFields);
summaryCountFieldName, detectors, influencers, multivariateByFields);
}
public static Builder builder(List<Detector> detectors) {
@ -314,8 +288,6 @@ public class AnalysisConfig implements ToXContentObject {
private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
private String summaryCountFieldName;
private List<String> influencers = new ArrayList<>();
private Boolean overlappingBuckets;
private Long resultFinalizationWindow;
private Boolean multivariateByFields;
public Builder(List<Detector> detectors) {
@ -332,8 +304,6 @@ public class AnalysisConfig implements ToXContentObject {
this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
this.influencers = new ArrayList<>(analysisConfig.influencers);
this.overlappingBuckets = analysisConfig.overlappingBuckets;
this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
this.multivariateByFields = analysisConfig.multivariateByFields;
}
@ -391,16 +361,6 @@ public class AnalysisConfig implements ToXContentObject {
return this;
}
public Builder setOverlappingBuckets(Boolean overlappingBuckets) {
this.overlappingBuckets = overlappingBuckets;
return this;
}
public Builder setResultFinalizationWindow(Long resultFinalizationWindow) {
this.resultFinalizationWindow = resultFinalizationWindow;
return this;
}
public Builder setMultivariateByFields(Boolean multivariateByFields) {
this.multivariateByFields = multivariateByFields;
return this;
@ -409,8 +369,7 @@ public class AnalysisConfig implements ToXContentObject {
public AnalysisConfig build() {
return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
resultFinalizationWindow, multivariateByFields);
latency, summaryCountFieldName, detectors, influencers, multivariateByFields);
}
}
}

6
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java
View File

@ -89,12 +89,6 @@ public class AnalysisConfigTests extends AbstractXContentTestCase<AnalysisConfig
if (randomBoolean()) {
builder.setMultivariateByFields(randomBoolean());
}
if (randomBoolean()) {
builder.setOverlappingBuckets(randomBoolean());
}
if (randomBoolean()) {
builder.setResultFinalizationWindow(randomNonNegativeLong());
}
builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false)));
return builder;

97
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
View File

@ -61,15 +61,11 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
private static final ParseField DETECTORS = new ParseField("detectors");
private static final ParseField INFLUENCERS = new ParseField("influencers");
private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");
public static final String ML_CATEGORY_FIELD = "mlcategory";
public static final Set<String> AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD));
public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L;
// These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> LENIENT_PARSER = createParser(true);
public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> STRICT_PARSER = createParser(false);
@ -94,8 +90,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
parser.declareStringArray(Builder::setInfluencers, INFLUENCERS);
parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
return parser;
@ -112,14 +106,11 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
private final String summaryCountFieldName;
private final List<Detector> detectors;
private final List<String> influencers;
private final Boolean overlappingBuckets;
private final Long resultFinalizationWindow;
private final Boolean multivariateByFields;
private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
Boolean multivariateByFields) {
List<Detector> detectors, List<String> influencers, Boolean multivariateByFields) {
this.detectors = detectors;
this.bucketSpan = bucketSpan;
this.latency = latency;
@ -128,8 +119,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
this.summaryCountFieldName = summaryCountFieldName;
this.influencers = Collections.unmodifiableList(influencers);
this.overlappingBuckets = overlappingBuckets;
this.resultFinalizationWindow = resultFinalizationWindow;
this.multivariateByFields = multivariateByFields;
}
@ -146,8 +135,13 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
summaryCountFieldName = in.readOptionalString();
detectors = Collections.unmodifiableList(in.readList(Detector::new));
influencers = Collections.unmodifiableList(in.readList(StreamInput::readString));
overlappingBuckets = in.readOptionalBoolean();
resultFinalizationWindow = in.readOptionalLong();
// BWC for result_finalization_window and overlapping_buckets
// TODO Remove in 7.0.0
if (in.getVersion().before(Version.CURRENT)) {//setting to current before backport
in.readOptionalBoolean();
in.readOptionalLong();
}
multivariateByFields = in.readOptionalBoolean();
// BWC for removed multiple_bucket_spans
@ -185,8 +179,13 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
out.writeOptionalString(summaryCountFieldName);
out.writeList(detectors);
out.writeStringList(influencers);
out.writeOptionalBoolean(overlappingBuckets);
out.writeOptionalLong(resultFinalizationWindow);
// BWC for result_finalization_window and overlapping_buckets
// TODO Remove in 7.0.0
if (out.getVersion().before(Version.CURRENT)) { //setting to current before backport
out.writeOptionalBoolean(null);
out.writeOptionalLong(null);
}
out.writeOptionalBoolean(multivariateByFields);
// BWC for removed multiple_bucket_spans
@ -291,14 +290,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
.flatMap(Set::stream).collect(Collectors.toSet());
}
public Boolean getOverlappingBuckets() {
return overlappingBuckets;
}
public Long getResultFinalizationWindow() {
return resultFinalizationWindow;
}
public Boolean getMultivariateByFields() {
return multivariateByFields;
}
@ -394,12 +385,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
}
builder.endArray();
builder.field(INFLUENCERS.getPreferredName(), influencers);
if (overlappingBuckets != null) {
builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
}
if (resultFinalizationWindow != null) {
builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
}
if (multivariateByFields != null) {
builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
}
@ -420,8 +405,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
Objects.equals(detectors, that.detectors) &&
Objects.equals(influencers, that.influencers) &&
Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
Objects.equals(multivariateByFields, that.multivariateByFields);
}
@ -429,9 +412,7 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
public int hashCode() {
return Objects.hash(
bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
multivariateByFields
);
summaryCountFieldName, detectors, influencers, multivariateByFields);
}
public static class Builder {
@ -446,8 +427,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
private String summaryCountFieldName;
private List<String> influencers = new ArrayList<>();
private Boolean overlappingBuckets;
private Long resultFinalizationWindow;
private Boolean multivariateByFields;
public Builder(List<Detector> detectors) {
@ -464,8 +443,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
this.influencers = new ArrayList<>(analysisConfig.influencers);
this.overlappingBuckets = analysisConfig.overlappingBuckets;
this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
this.multivariateByFields = analysisConfig.multivariateByFields;
}
@ -517,14 +494,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
this.influencers = ExceptionsHelper.requireNonNull(influencers, INFLUENCERS.getPreferredName());
}
public void setOverlappingBuckets(Boolean overlappingBuckets) {
this.overlappingBuckets = overlappingBuckets;
}
public void setResultFinalizationWindow(Long resultFinalizationWindow) {
this.resultFinalizationWindow = resultFinalizationWindow;
}
public void setMultivariateByFields(Boolean multivariateByFields) {
this.multivariateByFields = multivariateByFields;
}
@ -536,7 +505,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
* <li>Check that if non-null Latency is &lt;= MAX_LATENCY</li>
* <li>Check there is at least one detector configured</li>
* <li>Check all the detectors are configured correctly</li>
* <li>Check that OVERLAPPING_BUCKETS is set appropriately</li>
* <li>Check that MULTIPLE_BUCKETSPANS are set appropriately</li>
* <li>If Per Partition normalization is configured at least one detector
* must have a partition field and no influences can be used</li>
@ -555,17 +523,13 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
verifyMlCategoryIsUsedWhenCategorizationFieldNameIsSet();
verifyCategorizationAnalyzer();
verifyCategorizationFilters();
checkFieldIsNotNegativeIfSpecified(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet();
overlappingBuckets = verifyOverlappingBucketsConfig(overlappingBuckets, detectors);
verifyNoInconsistentNestedFieldNames();
return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
resultFinalizationWindow, multivariateByFields);
latency, summaryCountFieldName, detectors, influencers, multivariateByFields);
}
private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() {
@ -576,13 +540,6 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
}
}
private static void checkFieldIsNotNegativeIfSpecified(String fieldName, Long value) {
if (value != null && value < 0) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, fieldName, 0, value);
throw ExceptionsHelper.badRequestException(msg);
}
}
private void verifyDetectorAreDefined() {
if (detectors == null || detectors.isEmpty()) {
throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS));
@ -697,25 +654,5 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
return false;
}
}
private static Boolean verifyOverlappingBucketsConfig(Boolean overlappingBuckets, List<Detector> detectors) {
// If any detector function is rare/freq_rare, mustn't use overlapping buckets
boolean mustNotUse = false;
List<DetectorFunction> illegalFunctions = new ArrayList<>();
for (Detector d : detectors) {
if (Detector.NO_OVERLAPPING_BUCKETS_FUNCTIONS.contains(d.getFunction())) {
illegalFunctions.add(d.getFunction());
mustNotUse = true;
}
}
if (Boolean.TRUE.equals(overlappingBuckets) && mustNotUse) {
throw ExceptionsHelper.badRequestException(
Messages.getMessage(Messages.JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION, illegalFunctions.toString()));
}
return overlappingBuckets;
}
}
}

19
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
View File

@ -184,25 +184,6 @@ public class Detector implements ToXContentObject, Writeable {
DetectorFunction.HIGH_NON_ZERO_COUNT
);
/**
* The set of functions that must not be used with overlapping buckets
*/
public static final EnumSet<DetectorFunction> NO_OVERLAPPING_BUCKETS_FUNCTIONS = EnumSet.of(
DetectorFunction.RARE,
DetectorFunction.FREQ_RARE
);
/**
* The set of functions that should not be used with overlapping buckets
* as they gain no benefit but have overhead
*/
public static final EnumSet<DetectorFunction> OVERLAPPING_BUCKETS_FUNCTIONS_NOT_NEEDED = EnumSet.of(
DetectorFunction.MIN,
DetectorFunction.MAX,
DetectorFunction.TIME_OF_DAY,
DetectorFunction.TIME_OF_WEEK
);
/**
* Functions that do not support rule conditions:
* <ul>

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
View File

@ -138,8 +138,6 @@ public final class Messages {
public static final String JOB_CONFIG_NO_DETECTORS = "No detectors configured";
public static final String JOB_CONFIG_OVERFIELD_INCOMPATIBLE_FUNCTION =
"over_field_name cannot be used with function ''{0}''";
public static final String JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION =
"Overlapping buckets cannot be used with function ''{0}''";
public static final String JOB_CONFIG_UNKNOWN_FUNCTION = "Unknown function ''{0}''";
public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED =
"Invalid update value for analysis_limits: model_memory_limit cannot be decreased below current usage; " +

110
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
View File

@ -90,12 +90,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
if (randomBoolean()) {
builder.setMultivariateByFields(randomBoolean());
}
if (randomBoolean()) {
builder.setOverlappingBuckets(randomBoolean());
}
if (randomBoolean()) {
builder.setResultFinalizationWindow(randomNonNegativeLong());
}
builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false)));
@ -564,94 +558,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
RecordWriter.CONTROL_FIELD_NAME), e.getMessage());
}
public void testVerify_OverlappingBuckets() {
List<Detector> detectors;
Detector detector;
boolean onByDefault = false;
// Uncomment this when overlappingBuckets turned on by default
if (onByDefault) {
// Test overlappingBuckets unset
AnalysisConfig.Builder analysisConfig = createValidConfig();
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L));
detectors = new ArrayList<>();
detector = new Detector.Builder("count", null).build();
detectors.add(detector);
detector = new Detector.Builder("mean", "value").build();
detectors.add(detector);
analysisConfig.setDetectors(detectors);
AnalysisConfig ac = analysisConfig.build();
assertTrue(ac.getOverlappingBuckets());
// Test overlappingBuckets unset
analysisConfig = createValidConfig();
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L));
detectors = new ArrayList<>();
detector = new Detector.Builder("count", null).build();
detectors.add(detector);
detector = new Detector.Builder("rare", "value").build();
detectors.add(detector);
analysisConfig.setDetectors(detectors);
ac = analysisConfig.build();
assertFalse(ac.getOverlappingBuckets());
// Test overlappingBuckets unset
analysisConfig = createValidConfig();
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L));
detectors = new ArrayList<>();
detector = new Detector.Builder("count", null).build();
detectors.add(detector);
detector = new Detector.Builder("min", "value").build();
detectors.add(detector);
detector = new Detector.Builder("max", "value").build();
detectors.add(detector);
analysisConfig.setDetectors(detectors);
ac = analysisConfig.build();
assertFalse(ac.getOverlappingBuckets());
}
// Test overlappingBuckets set
AnalysisConfig.Builder analysisConfig = createValidConfig();
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L));
detectors = new ArrayList<>();
detector = new Detector.Builder("count", null).build();
detectors.add(detector);
Detector.Builder builder = new Detector.Builder("rare", null);
builder.setByFieldName("value");
detectors.add(builder.build());
analysisConfig.setOverlappingBuckets(false);
analysisConfig.setDetectors(detectors);
assertFalse(analysisConfig.build().getOverlappingBuckets());
// Test overlappingBuckets set
analysisConfig = createValidConfig();
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L));
analysisConfig.setOverlappingBuckets(true);
detectors = new ArrayList<>();
detector = new Detector.Builder("count", null).build();
detectors.add(detector);
builder = new Detector.Builder("rare", null);
builder.setByFieldName("value");
detectors.add(builder.build());
analysisConfig.setDetectors(detectors);
ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, analysisConfig::build);
assertEquals("Overlapping buckets cannot be used with function '[rare]'", e.getMessage());
// Test overlappingBuckets set
analysisConfig = createValidConfig();
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L));
analysisConfig.setOverlappingBuckets(false);
detectors = new ArrayList<>();
detector = new Detector.Builder("count", null).build();
detectors.add(detector);
detector = new Detector.Builder("mean", "value").build();
detectors.add(detector);
analysisConfig.setDetectors(detectors);
AnalysisConfig ac = analysisConfig.build();
assertFalse(ac.getOverlappingBuckets());
}
public void testVerify_GivenMetricAndSummaryCountField() {
Detector d = new Detector.Builder("metric", "my_metric").build();
AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d));
@ -728,7 +634,7 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
@Override
protected AnalysisConfig mutateInstance(AnalysisConfig instance) {
AnalysisConfig.Builder builder = new AnalysisConfig.Builder(instance);
switch (between(0, 10)) {
switch (between(0, 8)) {
case 0:
List<Detector> detectors = new ArrayList<>(instance.getDetectors());
Detector.Builder detector = new Detector.Builder();
@ -806,20 +712,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
builder.setInfluencers(influencers);
break;
case 8:
if (instance.getOverlappingBuckets() == null) {
builder.setOverlappingBuckets(randomBoolean());
} else {
builder.setOverlappingBuckets(instance.getOverlappingBuckets() == false);
}
break;
case 9:
if (instance.getResultFinalizationWindow() == null) {
builder.setResultFinalizationWindow(between(1, 100) * 1000L);
} else {
builder.setResultFinalizationWindow(instance.getResultFinalizationWindow() + (between(1, 100) * 1000));
}
break;
case 10:
if (instance.getMultivariateByFields() == null) {
builder.setMultivariateByFields(randomBoolean());
} else {

16
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java
View File

@ -43,7 +43,6 @@ public class UpdateInterimResultsIT extends MlNativeAutodetectIntegTestCase {
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Collections.singletonList(new Detector.Builder("max", "value").build()));
analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(BUCKET_SPAN_SECONDS));
analysisConfig.setOverlappingBuckets(true);
DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch");
Job.Builder job = new Job.Builder(JOB_ID);
@ -77,10 +76,9 @@ public class UpdateInterimResultsIT extends MlNativeAutodetectIntegTestCase {
// We might need to retry this while waiting for a refresh
assertBusy(() -> {
List<Bucket> firstInterimBuckets = getInterimResults(job.getId());
assertThat("interim buckets were: " + firstInterimBuckets, firstInterimBuckets.size(), equalTo(2));
assertThat(firstInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400039000000L));
assertThat(firstInterimBuckets.get(1).getTimestamp().getTime(), equalTo(1400040000000L));
assertThat(firstInterimBuckets.get(1).getRecords().get(0).getActual().get(0), equalTo(16.0));
assertThat("interim buckets were: " + firstInterimBuckets, firstInterimBuckets.size(), equalTo(1));
assertThat(firstInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400040000000L));
assertThat(firstInterimBuckets.get(0).getRecords().get(0).getActual().get(0), equalTo(16.0));
});
// push 1 more record, flush (with interim), check same interim result
@ -90,9 +88,7 @@ public class UpdateInterimResultsIT extends MlNativeAutodetectIntegTestCase {
assertBusy(() -> {
List<Bucket> secondInterimBuckets = getInterimResults(job.getId());
assertThat(secondInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400039000000L));
assertThat(secondInterimBuckets.get(1).getTimestamp().getTime(), equalTo(1400040000000L));
assertThat(secondInterimBuckets.get(1).getRecords().get(0).getActual().get(0), equalTo(16.0));
assertThat(secondInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400040000000L));
});
// push rest of data, close, verify no interim results
@ -103,11 +99,11 @@ public class UpdateInterimResultsIT extends MlNativeAutodetectIntegTestCase {
// Verify interim results have been replaced with finalized results
GetBucketsAction.Request bucketRequest = new GetBucketsAction.Request(job.getId());
bucketRequest.setTimestamp("1400039500000");
bucketRequest.setTimestamp("1400040000000");
bucketRequest.setExpand(true);
List<Bucket> bucket = client().execute(GetBucketsAction.INSTANCE, bucketRequest).get().getBuckets().results();
assertThat(bucket.size(), equalTo(1));
assertThat(bucket.get(0).getRecords().get(0).getActual().get(0), equalTo(14.0));
assertThat(bucket.get(0).getRecords().get(0).getActual().get(0), equalTo(16.0));
}
private String createData(int halfBuckets) {

11
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java
View File

@ -68,7 +68,6 @@ public class AutodetectBuilder {
private static final String MODEL_PLOT_CONFIG_ARG = "--modelplotconfig=";
private static final String FIELD_CONFIG_ARG = "--fieldconfig=";
static final String LATENCY_ARG = "--latency=";
static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow=";
static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields";
static final String PERSIST_INTERVAL_ARG = "--persistInterval=";
static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval=";
@ -202,15 +201,7 @@ public class AutodetectBuilder {
if (analysisConfig != null) {
addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command);
addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command);
addIfNotNull(analysisConfig.getSummaryCountFieldName(),
SUMMARY_COUNT_FIELD_ARG, command);
if (Boolean.TRUE.equals(analysisConfig.getOverlappingBuckets())) {
Long window = analysisConfig.getResultFinalizationWindow();
if (window == null) {
window = AnalysisConfig.DEFAULT_RESULT_FINALIZATION_WINDOW;
}
command.add(RESULT_FINALIZATION_WINDOW_ARG + window);
}
addIfNotNull(analysisConfig.getSummaryCountFieldName(), SUMMARY_COUNT_FIELD_ARG, command);
if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) {
command.add(MULTIVARIATE_BY_FIELDS_ARG);
}

5
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java
View File

@ -54,8 +54,8 @@ public class AutodetectBuilderTests extends ESTestCase {
acBuilder.setBucketSpan(TimeValue.timeValueSeconds(120));
acBuilder.setLatency(TimeValue.timeValueSeconds(360));
acBuilder.setSummaryCountFieldName("summaryField");
acBuilder.setOverlappingBuckets(true);
acBuilder.setMultivariateByFields(true);
job.setAnalysisConfig(acBuilder);
DataDescription.Builder dd = new DataDescription.Builder();
@ -65,12 +65,11 @@ public class AutodetectBuilderTests extends ESTestCase {
job.setDataDescription(dd);
List<String> command = autodetectBuilder(job.build()).buildAutodetectCommand();
assertEquals(12, command.size());
assertEquals(11, command.size());
assertTrue(command.contains(AutodetectBuilder.AUTODETECT_PATH));
assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "120"));
assertTrue(command.contains(AutodetectBuilder.LATENCY_ARG + "360"));
assertTrue(command.contains(AutodetectBuilder.SUMMARY_COUNT_FIELD_ARG + "summaryField"));
assertTrue(command.contains(AutodetectBuilder.RESULT_FINALIZATION_WINDOW_ARG + "2"));
assertTrue(command.contains(AutodetectBuilder.MULTIVARIATE_BY_FIELDS_ARG));
assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG));

5
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml
View File

@ -1,4 +1,9 @@
---
setup:
- skip:
version: "all"
reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645"
---
"Test get old cluster job":
- do:
xpack.ml.get_jobs:

5
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml
View File

@ -1,4 +1,9 @@
---
setup:
- skip:
version: "all"
reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645"
---
"Test old cluster datafeed":
- do:
xpack.ml.get_datafeeds:

5
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml
View File

@ -1,4 +1,9 @@
---
setup:
- skip:
version: "all"
reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645"
---
"Put job on the old cluster and post some data":
- do:

5
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml
View File

@ -1,4 +1,9 @@
---
setup:
- skip:
version: "all"
reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645"
---
"Put job and datafeed in old cluster":
- do:

3
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
View File

@ -1,4 +1,7 @@
setup:
- skip:
version: "all"
reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645"
- do:
cluster.health:
wait_for_status: green

3
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml
View File

@ -1,4 +1,7 @@
setup:
- skip:
version: "all"
reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645"
- do:
cluster.health:
wait_for_status: green