[ML] Randomize default datafeed query delay (elastic/x-pack-elasticsearch#2475)

Changes the default query delay from 1m to a random
value between 1m and 2m. The motivation is to avoid
having multiple jobs firing their searches at the same
time which may potentially lead to increased load
on the machine.

relates elastic/x-pack-elasticsearch#2472

Original commit: elastic/x-pack-elasticsearch@3224e836fa

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfig.java

@@ -41,6 +41,7 @@ import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -352,12 +353,13 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
     public static class Builder {
 
         private static final int DEFAULT_SCROLL_SIZE = 1000;
-        private static final TimeValue DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
+        private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
+        private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2);
         private static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;
 
         private String id;
         private String jobId;
-        private TimeValue queryDelay = DEFAULT_QUERY_DELAY;
+        private TimeValue queryDelay;
         private TimeValue frequency;
         private List<String> indices = Collections.emptyList();
         private List<String> types = Collections.emptyList();
@@ -460,6 +462,7 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
             }
             validateAggregations();
             setDefaultChunkingConfig();
+            setDefaultQueryDelay();
             return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, types, query, aggregations, scriptFields, scrollSize,
                     chunkingConfig);
         }
@@ -530,6 +533,15 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
             }
         }
 
+        private void setDefaultQueryDelay() {
+            if (queryDelay == null) {
+                Random random = new Random(jobId.hashCode());
+                long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis())
+                        .findFirst().getAsLong();
+                queryDelay = TimeValue.timeValueMillis(delayMillis);
+            }
+        }
+
         private static ElasticsearchException invalidOptionValue(String fieldName, Object value) {
             String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, fieldName, value);
             throw ExceptionsHelper.badRequestException(msg);

plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigTests.java

@@ -33,18 +33,23 @@ import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ml.datafeed.ChunkingConfig.Mode;
+import org.elasticsearch.xpack.ml.job.config.JobTests;
 import org.elasticsearch.xpack.ml.job.messages.Messages;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.TimeZone;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.not;
 
 public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedConfig> {
 
@@ -162,15 +167,36 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
         }
     }
 
-    public void testFillDefaults() {
+    public void testDefaults() {
         DatafeedConfig.Builder expectedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1");
         expectedDatafeedConfig.setIndices(Collections.singletonList("index"));
         expectedDatafeedConfig.setQueryDelay(TimeValue.timeValueMinutes(1));
         expectedDatafeedConfig.setScrollSize(1000);
-        DatafeedConfig.Builder defaultedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1");
+        DatafeedConfig.Builder defaultFeedBuilder = new DatafeedConfig.Builder("datafeed1", "job1");
-        defaultedDatafeedConfig.setIndices(Collections.singletonList("index"));
+        defaultFeedBuilder.setIndices(Collections.singletonList("index"));
+        DatafeedConfig defaultFeed = defaultFeedBuilder.build();
 
-        assertEquals(expectedDatafeedConfig.build(), defaultedDatafeedConfig.build());
+
+        assertThat(defaultFeed.getScrollSize(), equalTo(1000));
+        assertThat(defaultFeed.getQueryDelay().seconds(), greaterThanOrEqualTo(60L));
+        assertThat(defaultFeed.getQueryDelay().seconds(), lessThan(120L));
+    }
+
+    public void testDefaultQueryDelay() {
+        DatafeedConfig.Builder feedBuilder1 = new DatafeedConfig.Builder("datafeed1", "job1");
+        feedBuilder1.setIndices(Arrays.asList("foo"));
+        DatafeedConfig.Builder feedBuilder2 = new DatafeedConfig.Builder("datafeed2", "job1");
+        feedBuilder2.setIndices(Arrays.asList("foo"));
+        DatafeedConfig.Builder feedBuilder3 = new DatafeedConfig.Builder("datafeed3", "job2");
+        feedBuilder3.setIndices(Arrays.asList("foo"));
+        DatafeedConfig feed1 = feedBuilder1.build();
+        DatafeedConfig feed2 = feedBuilder2.build();
+        DatafeedConfig feed3 = feedBuilder3.build();
+
+        // Two datafeeds with the same job id should have the same random query delay
+        assertThat(feed1.getQueryDelay(), equalTo(feed2.getQueryDelay()));
+        // But the query delay of a datafeed with a different job id should differ too
+        assertThat(feed1.getQueryDelay(), not(equalTo(feed3.getQueryDelay())));
     }
 
     public void testCheckValid_GivenNullIndices() throws IOException {