From 4da1c5b9dceeb2900764f771034dc6a365748c53 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 11 Apr 2017 12:57:58 +0100 Subject: [PATCH] [ML] Remove batch_span and period from analysis config (elastic/x-pack-elasticsearch#1043) relates elastic/x-pack-elasticsearch#1040 Original commit: elastic/x-pack-elasticsearch@aed5e9912c6a7469f662733b792b6fc8ffb55410 --- .../xpack/ml/job/config/AnalysisConfig.java | 73 ++----------------- .../xpack/ml/job/process/ProcessCtrl.java | 4 - .../datafeed/DatafeedJobValidatorTests.java | 1 - .../ml/job/config/AnalysisConfigTests.java | 57 --------------- .../ml/job/process/ProcessCtrlTests.java | 6 +- .../xpack/ml/client/MLTransportClientIT.java | 1 - 6 files changed, 9 insertions(+), 133 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java index dbf22665a43..3ff01962102 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java @@ -39,7 +39,7 @@ import java.util.stream.Collectors; *

* The configuration can contain multiple detectors, a new anomaly detector will * be created for each detector configuration. The fields - * bucketSpan, batchSpan, summaryCountFieldName and categorizationFieldName + * bucketSpan, summaryCountFieldName and categorizationFieldName * apply to all detectors. *

* If a value has not been set it will be null @@ -52,11 +52,9 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { */ private static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config"); private static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); - private static final ParseField BATCH_SPAN = new ParseField("batch_span"); private static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name"); public static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters"); private static final ParseField LATENCY = new ParseField("latency"); - private static final ParseField PERIOD = new ParseField("period"); private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name"); private static final ParseField DETECTORS = new ParseField("detectors"); private static final ParseField INFLUENCERS = new ParseField("influencers"); @@ -79,13 +77,10 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Detector.PARSER.apply(p, c).build(), DETECTORS); PARSER.declareString((builder, val) -> builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN); - PARSER.declareString((builder, val) -> - builder.setBatchSpan(TimeValue.parseTimeValue(val, BATCH_SPAN.getPreferredName())), BATCH_SPAN); PARSER.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME); PARSER.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS); PARSER.declareString((builder, val) -> builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY); - PARSER.declareLong(Builder::setPeriod, PERIOD); PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME); PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS); PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS); @@ -101,11 +96,9 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { * These values apply to all detectors */ private final TimeValue bucketSpan; - private final TimeValue batchSpan; private final String categorizationFieldName; private final List categorizationFilters; private final TimeValue latency; - private final Long period; private final String summaryCountFieldName; private final List detectors; private final List influencers; @@ -115,15 +108,13 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { private final List multipleBucketSpans; private final boolean usePerPartitionNormalization; - private AnalysisConfig(TimeValue bucketSpan, TimeValue batchSpan, String categorizationFieldName, List categorizationFilters, - TimeValue latency, Long period, String summaryCountFieldName, List detectors, + private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List categorizationFilters, + TimeValue latency, String summaryCountFieldName, List detectors, List influencers, Boolean overlappingBuckets, Long resultFinalizationWindow, Boolean multivariateByFields, List multipleBucketSpans, boolean usePerPartitionNormalization) { this.detectors = detectors; this.bucketSpan = bucketSpan; - this.batchSpan = batchSpan; this.latency = latency; - this.period = period; this.categorizationFieldName = categorizationFieldName; this.categorizationFilters = categorizationFilters; this.summaryCountFieldName = summaryCountFieldName; @@ -137,11 +128,9 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { public AnalysisConfig(StreamInput in) throws IOException { bucketSpan = new TimeValue(in); - batchSpan = in.readOptionalWriteable(TimeValue::new); categorizationFieldName = in.readOptionalString(); categorizationFilters = in.readBoolean() ? in.readList(StreamInput::readString) : null; latency = in.readOptionalWriteable(TimeValue::new); - period = in.readOptionalLong(); summaryCountFieldName = in.readOptionalString(); detectors = in.readList(Detector::new); influencers = in.readList(StreamInput::readString); @@ -155,7 +144,6 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { bucketSpan.writeTo(out); - out.writeOptionalWriteable(batchSpan); out.writeOptionalString(categorizationFieldName); if (categorizationFilters != null) { out.writeBoolean(true); @@ -164,7 +152,6 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { out.writeBoolean(false); } out.writeOptionalWriteable(latency); - out.writeOptionalLong(period); out.writeOptionalString(summaryCountFieldName); out.writeList(detectors); out.writeStringList(influencers); @@ -189,15 +176,6 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { return bucketSpan; } - /** - * Interval into which to batch seasonal data - * - * @return The batchspan or null if not set - */ - public TimeValue getBatchSpan() { - return batchSpan; - } - public String getCategorizationFieldName() { return categorizationFieldName; } @@ -215,16 +193,6 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { return latency; } - /** - * The repeat interval for periodic data in multiples of - * {@linkplain #getBatchSpan()} - * - * @return The period or null if not set - */ - public Long getPeriod() { - return period; - } - /** * The name of the field that contains counts for pre-summarised input * @@ -367,9 +335,6 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan.getStringRep()); - if (batchSpan != null) { - builder.field(BATCH_SPAN.getPreferredName(), batchSpan.getStringRep()); - } if (categorizationFieldName != null) { builder.field(CATEGORIZATION_FIELD_NAME.getPreferredName(), categorizationFieldName); } @@ -379,9 +344,6 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { if (latency != null) { builder.field(LATENCY.getPreferredName(), latency.getStringRep()); } - if (period != null) { - builder.field(PERIOD.getPreferredName(), period); - } if (summaryCountFieldName != null) { builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName); } @@ -415,10 +377,8 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { return Objects.equals(latency, that.latency) && usePerPartitionNormalization == that.usePerPartitionNormalization && Objects.equals(bucketSpan, that.bucketSpan) && - Objects.equals(batchSpan, that.batchSpan) && Objects.equals(categorizationFieldName, that.categorizationFieldName) && Objects.equals(categorizationFilters, that.categorizationFilters) && - Objects.equals(period, that.period) && Objects.equals(summaryCountFieldName, that.summaryCountFieldName) && Objects.equals(detectors, that.detectors) && Objects.equals(influencers, that.influencers) && @@ -431,7 +391,7 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { @Override public int hashCode() { return Objects.hash( - bucketSpan, batchSpan, categorizationFieldName, categorizationFilters, latency, period, + bucketSpan, categorizationFieldName, categorizationFilters, latency, summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, multivariateByFields, multipleBucketSpans, usePerPartitionNormalization ); @@ -443,9 +403,7 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { private List detectors; private TimeValue bucketSpan = DEFAULT_BUCKET_SPAN; - private TimeValue batchSpan; private TimeValue latency; - private Long period; private String categorizationFieldName; private List categorizationFilters; private String summaryCountFieldName; @@ -463,9 +421,7 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { public Builder(AnalysisConfig analysisConfig) { this.detectors = analysisConfig.detectors; this.bucketSpan = analysisConfig.bucketSpan; - this.batchSpan = analysisConfig.batchSpan; this.latency = analysisConfig.latency; - this.period = analysisConfig.period; this.categorizationFieldName = analysisConfig.categorizationFieldName; this.categorizationFilters = analysisConfig.categorizationFilters; this.summaryCountFieldName = analysisConfig.summaryCountFieldName; @@ -485,18 +441,10 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { this.bucketSpan = bucketSpan; } - public void setBatchSpan(TimeValue batchSpan) { - this.batchSpan = batchSpan; - } - public void setLatency(TimeValue latency) { this.latency = latency; } - public void setPeriod(long period) { - this.period = period; - } - public void setCategorizationFieldName(String categorizationFieldName) { this.categorizationFieldName = categorizationFieldName; } @@ -536,8 +484,7 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { /** * Checks the configuration is valid *

    - *
  1. Check that if non-null BucketSpan, BatchSpan, Latency and Period are - * >= 0
    2. + *
  2. Check that if non-null BucketSpan and Latency are >= 0
    3. *
  3. Check that if non-null Latency is <= MAX_LATENCY
    4. *
  4. Check there is at least one detector configured
    5. *
  5. Check all the detectors are configured correctly
    6. @@ -549,13 +496,9 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { */ public AnalysisConfig build() { TimeUtils.checkPositiveMultiple(bucketSpan, TimeUnit.SECONDS, BUCKET_SPAN); - if (batchSpan != null) { - TimeUtils.checkPositiveMultiple(batchSpan, TimeUnit.SECONDS, BATCH_SPAN); - } if (latency != null) { TimeUtils.checkNonNegativeMultiple(latency, TimeUnit.SECONDS, LATENCY); } - checkFieldIsNotNegativeIfSpecified(PERIOD.getPreferredName(), period); verifyDetectorAreDefined(); verifyFieldName(summaryCountFieldName); @@ -573,8 +516,8 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { checkNoInfluencersAreSet(influencers); } - return new AnalysisConfig(bucketSpan, batchSpan, categorizationFieldName, categorizationFilters, - latency, period, summaryCountFieldName, detectors, influencers, overlappingBuckets, + return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, + latency, summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, multivariateByFields, multipleBucketSpans, usePerPartitionNormalization); } @@ -696,7 +639,7 @@ public class AnalysisConfig extends ToXContentToBytes implements Writeable { return true; } } - return field.chars().anyMatch(ch -> Character.isISOControl(ch)); + return field.chars().anyMatch(Character::isISOControl); } private static boolean isValidRegex(String exp) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java index d1c166cbe21..7049d7093d1 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java @@ -88,11 +88,9 @@ public class ProcessCtrl { /* * Arguments used by autodetect */ - static final String BATCH_SPAN_ARG = "--batchspan="; static final String LATENCY_ARG = "--latency="; static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow="; static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields"; - static final String PERIOD_ARG = "--period="; static final String PERSIST_INTERVAL_ARG = "--persistInterval="; static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval="; static final String SUMMARY_COUNT_FIELD_ARG = "--summarycountfield="; @@ -166,9 +164,7 @@ public class ProcessCtrl { AnalysisConfig analysisConfig = job.getAnalysisConfig(); if (analysisConfig != null) { addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command); - addIfNotNull(analysisConfig.getBatchSpan(), BATCH_SPAN_ARG, command); addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command); - addIfNotNull(analysisConfig.getPeriod(), PERIOD_ARG, command); addIfNotNull(analysisConfig.getSummaryCountFieldName(), SUMMARY_COUNT_FIELD_ARG, command); addIfNotNull(analysisConfig.getMultipleBucketSpans(), diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java index cb1d1a977a1..9c76c337ddd 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java @@ -53,7 +53,6 @@ public class DatafeedJobValidatorTests extends ESTestCase { public void testVerify_GivenNoLatency() { Job.Builder builder = buildJobBuilder("foo"); AnalysisConfig.Builder ac = createAnalysisConfig(); - ac.setBatchSpan(TimeValue.timeValueSeconds(1800)); ac.setBucketSpan(TimeValue.timeValueSeconds(100)); builder.setAnalysisConfig(ac); Job job = builder.build(new Date()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java index c741e6197c5..f7046bf4da4 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java @@ -40,9 +40,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase analysisConfig.build()); - - assertEquals("batch_span cannot be less or equal than 0. Value = -1", e.getMessage()); - } - public void testVerify_GivenNegativeLatency() { AnalysisConfig.Builder analysisConfig = createValidConfig(); analysisConfig.setLatency(TimeValue.timeValueSeconds(-1)); @@ -542,16 +502,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase analysisConfig.build()); - - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "period", 0, -1), e.getMessage()); - } - - public void testVerify_GivenDefaultConfig_ShouldBeInvalidDueToNoDetectors() { AnalysisConfig.Builder analysisConfig = createValidConfig(); analysisConfig.setDetectors(null); @@ -585,7 +535,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase(); detector = new Detector.Builder("count", null).build(); detectors.add(detector); @@ -598,7 +547,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase(); detector = new Detector.Builder("count", null).build(); detectors.add(detector); @@ -611,7 +559,6 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase(); detector = new Detector.Builder("count", null).build(); detectors.add(detector); @@ -793,9 +740,7 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase command = ProcessCtrl.buildAutodetectCommand(env, settings, job.build(), logger, true, pid); - assertEquals(17, command.size()); + assertEquals(15, command.size()); assertTrue(command.contains(ProcessCtrl.AUTODETECT_PATH)); - assertTrue(command.contains(ProcessCtrl.BATCH_SPAN_ARG + "100")); assertTrue(command.contains(ProcessCtrl.BUCKET_SPAN_ARG + "120")); assertTrue(command.contains(ProcessCtrl.LATENCY_ARG + "360")); - assertTrue(command.contains(ProcessCtrl.PERIOD_ARG + "20")); assertTrue(command.contains(ProcessCtrl.SUMMARY_COUNT_FIELD_ARG + "summaryField")); assertTrue(command.contains(ProcessCtrl.RESULT_FINALIZATION_WINDOW_ARG + "2")); assertTrue(command.contains(ProcessCtrl.MULTIVARIATE_BY_FIELDS_ARG)); diff --git a/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java b/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java index ad03f7eb0a1..9311b7f4d51 100644 --- a/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java +++ b/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/MLTransportClientIT.java @@ -37,7 +37,6 @@ public class MLTransportClientIT extends ESXPackSmokeClientTestCase { detectors.add(detector.build()); AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(detectors); - analysisConfig.setBatchSpan(TimeValue.timeValueMinutes(5)); job.setAnalysisConfig(analysisConfig); PutJobAction.Response putJobResponse = mlClient