honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Randomize default datafeed query delay (elastic/x-pack-elasticsearch#2475) 

Changes the default query delay from 1m to a random
value between 1m and 2m. The motivation is to avoid
having multiple jobs firing their searches at the same
time which may potentially lead to increased load
on the machine.

relates elastic/x-pack-elasticsearch#2472

Original commit: elastic/x-pack-elasticsearch@3224e836fa
This commit is contained in:
Dimitris Athanasiou 2017-09-13 09:12:39 +01:00 committed by GitHub
parent 2e3aca414b
commit e4753656bc
2 changed files with 44 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfig.java
View File

@ -41,6 +41,7 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
@ -352,12 +353,13 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
public static class Builder {
private static final int DEFAULT_SCROLL_SIZE = 1000;
private static final TimeValue DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2);
private static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;
private String id;
private String jobId;
private TimeValue queryDelay = DEFAULT_QUERY_DELAY;
private TimeValue queryDelay;
private TimeValue frequency;
private List<String> indices = Collections.emptyList();
private List<String> types = Collections.emptyList();
@ -460,6 +462,7 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
}
validateAggregations();
setDefaultChunkingConfig();
setDefaultQueryDelay();
return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, types, query, aggregations, scriptFields, scrollSize,
chunkingConfig);
}
@ -530,6 +533,15 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
}
}
private void setDefaultQueryDelay() {
if (queryDelay == null) {
Random random = new Random(jobId.hashCode());
long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis())
.findFirst().getAsLong();
queryDelay = TimeValue.timeValueMillis(delayMillis);
}
}
private static ElasticsearchException invalidOptionValue(String fieldName, Object value) {
String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, fieldName, value);
throw ExceptionsHelper.badRequestException(msg);

34
plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigTests.java
View File

@ -33,18 +33,23 @@ import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
import org.elasticsearch.test.AbstractSerializingTestCase;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.datafeed.ChunkingConfig.Mode;
import org.elasticsearch.xpack.ml.job.config.JobTests;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.TimeZone;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.not;
public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedConfig> {
@ -162,15 +167,36 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
}
}
public void testFillDefaults() {
public void testDefaults() {
DatafeedConfig.Builder expectedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1");
expectedDatafeedConfig.setIndices(Collections.singletonList("index"));
expectedDatafeedConfig.setQueryDelay(TimeValue.timeValueMinutes(1));
expectedDatafeedConfig.setScrollSize(1000);
DatafeedConfig.Builder defaultedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1");
defaultedDatafeedConfig.setIndices(Collections.singletonList("index"));
DatafeedConfig.Builder defaultFeedBuilder = new DatafeedConfig.Builder("datafeed1", "job1");
defaultFeedBuilder.setIndices(Collections.singletonList("index"));
DatafeedConfig defaultFeed = defaultFeedBuilder.build();
assertEquals(expectedDatafeedConfig.build(), defaultedDatafeedConfig.build());
assertThat(defaultFeed.getScrollSize(), equalTo(1000));
assertThat(defaultFeed.getQueryDelay().seconds(), greaterThanOrEqualTo(60L));
assertThat(defaultFeed.getQueryDelay().seconds(), lessThan(120L));
}
public void testDefaultQueryDelay() {
DatafeedConfig.Builder feedBuilder1 = new DatafeedConfig.Builder("datafeed1", "job1");
feedBuilder1.setIndices(Arrays.asList("foo"));
DatafeedConfig.Builder feedBuilder2 = new DatafeedConfig.Builder("datafeed2", "job1");
feedBuilder2.setIndices(Arrays.asList("foo"));
DatafeedConfig.Builder feedBuilder3 = new DatafeedConfig.Builder("datafeed3", "job2");
feedBuilder3.setIndices(Arrays.asList("foo"));
DatafeedConfig feed1 = feedBuilder1.build();
DatafeedConfig feed2 = feedBuilder2.build();
DatafeedConfig feed3 = feedBuilder3.build();
// Two datafeeds with the same job id should have the same random query delay
assertThat(feed1.getQueryDelay(), equalTo(feed2.getQueryDelay()));
// But the query delay of a datafeed with a different job id should differ too
assertThat(feed1.getQueryDelay(), not(equalTo(feed3.getQueryDelay())));
}
public void testCheckValid_GivenNullIndices() throws IOException {