honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'elastic/master' into feature/sql 

Original commit: elastic/x-pack-elasticsearch@e67ceb1d1e
This commit is contained in:
Igor Motov 2017-11-17 13:33:28 -05:00
parent 87a3e93fae 418a6632e5
commit ffcd54de77
2 changed files with 30 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
View File

@ -17,7 +17,7 @@ import org.junit.After;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -42,8 +42,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
detector.setPartitionFieldName("user"); detector.setPartitionFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1); TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
Arrays.asList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan); analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder(); DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch"); dataDescription.setTimeFormat("epoch");
@ -64,6 +63,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
List<String> data = new ArrayList<>(); List<String> data = new ArrayList<>();
while (timestamp < now) { while (timestamp < now) {
for (int i = 0; i < 10000; i++) { for (int i = 0; i < 10000; i++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
data.add(createJsonRecord(createRecord(timestamp, String.valueOf(i), ""))); data.add(createJsonRecord(createRecord(timestamp, String.valueOf(i), "")));
} }
timestamp += bucketSpan.seconds(); timestamp += bucketSpan.seconds();
@ -76,7 +78,6 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// and a balance of partitions/by fields were created // and a balance of partitions/by fields were created
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
assertThat(modelSizeStats.getModelBytes(), lessThan(35000000L)); assertThat(modelSizeStats.getModelBytes(), lessThan(35000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L)); assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L));
@ -87,6 +88,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
assertThat(modelSizeStats.getTotalPartitionFieldCount(), greaterThan(600L)); assertThat(modelSizeStats.getTotalPartitionFieldCount(), greaterThan(600L));
assertThat(modelSizeStats.getTotalByFieldCount(), lessThan(900L)); assertThat(modelSizeStats.getTotalByFieldCount(), lessThan(900L));
assertThat(modelSizeStats.getTotalByFieldCount(), greaterThan(600L)); assertThat(modelSizeStats.getTotalByFieldCount(), greaterThan(600L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
} }
public void testTooManyByFields() throws Exception { public void testTooManyByFields() throws Exception {
@ -94,8 +96,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
detector.setByFieldName("user"); detector.setByFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1); TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
Arrays.asList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan); analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder(); DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch"); dataDescription.setTimeFormat("epoch");
@ -116,6 +117,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
List<String> data = new ArrayList<>(); List<String> data = new ArrayList<>();
while (timestamp < now) { while (timestamp < now) {
for (int i = 0; i < 10000; i++) { for (int i = 0; i < 10000; i++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
data.add(createJsonRecord(createRecord(timestamp, String.valueOf(i), ""))); data.add(createJsonRecord(createRecord(timestamp, String.valueOf(i), "")));
} }
timestamp += bucketSpan.seconds(); timestamp += bucketSpan.seconds();
@ -127,11 +131,11 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// Assert we haven't violated the limit too much // Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L)); assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L)); assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L));
assertThat(modelSizeStats.getTotalByFieldCount(), lessThan(2100L)); assertThat(modelSizeStats.getTotalByFieldCount(), lessThan(2100L));
assertThat(modelSizeStats.getTotalByFieldCount(), greaterThan(1500L)); assertThat(modelSizeStats.getTotalByFieldCount(), greaterThan(1500L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
} }
public void testTooManyByAndOverFields() throws Exception { public void testTooManyByAndOverFields() throws Exception {
@ -140,8 +144,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
detector.setOverFieldName("user"); detector.setOverFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1); TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
Arrays.asList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan); analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder(); DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch"); dataDescription.setTimeFormat("epoch");
@ -163,6 +166,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
for (int department = 0; department < 10; department++) { for (int department = 0; department < 10; department++) {
List<String> data = new ArrayList<>(); List<String> data = new ArrayList<>();
for (int user = 0; user < 10000; user++) { for (int user = 0; user < 10000; user++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
data.add(createJsonRecord(createRecord( data.add(createJsonRecord(createRecord(
timestamp, String.valueOf(department) + "_" + String.valueOf(user), String.valueOf(department)))); timestamp, String.valueOf(department) + "_" + String.valueOf(user), String.valueOf(department))));
} }
@ -176,21 +182,20 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// Assert we haven't violated the limit too much // Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L)); assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(24000000L)); assertThat(modelSizeStats.getModelBytes(), greaterThan(24000000L));
assertThat(modelSizeStats.getTotalByFieldCount(), equalTo(8L)); assertThat(modelSizeStats.getTotalByFieldCount(), equalTo(7L));
assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(50000L)); assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(40000L));
assertThat(modelSizeStats.getTotalOverFieldCount(), lessThan(60000L)); assertThat(modelSizeStats.getTotalOverFieldCount(), lessThan(50000L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT));
} }
public void testTooManyDistinctOverFields() throws Exception { public void testManyDistinctOverFields() throws Exception {
Detector.Builder detector = new Detector.Builder("sum", "value"); Detector.Builder detector = new Detector.Builder("sum", "value");
detector.setOverFieldName("user"); detector.setOverFieldName("user");
TimeValue bucketSpan = TimeValue.timeValueHours(1); TimeValue bucketSpan = TimeValue.timeValueHours(1);
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
Arrays.asList(detector.build()));
analysisConfig.setBucketSpan(bucketSpan); analysisConfig.setBucketSpan(bucketSpan);
DataDescription.Builder dataDescription = new DataDescription.Builder(); DataDescription.Builder dataDescription = new DataDescription.Builder();
dataDescription.setTimeFormat("epoch"); dataDescription.setTimeFormat("epoch");
@ -198,8 +203,8 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
job.setAnalysisConfig(analysisConfig); job.setAnalysisConfig(analysisConfig);
job.setDataDescription(dataDescription); job.setDataDescription(dataDescription);
// Set the memory limit to 300MB // Set the memory limit to 110MB
AnalysisLimits limits = new AnalysisLimits(70L, null); AnalysisLimits limits = new AnalysisLimits(110L, null);
job.setAnalysisLimits(limits); job.setAnalysisLimits(limits);
registerJob(job); registerJob(job);
@ -212,6 +217,9 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
while (timestamp < now) { while (timestamp < now) {
List<String> data = new ArrayList<>(); List<String> data = new ArrayList<>();
for (int i = 0; i < 10000; i++) { for (int i = 0; i < 10000; i++) {
// It's important that the values used here are either always represented in less than 16 UTF-8 bytes or
// always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string
// optimisation is used will make the results of this test very different for the different platforms.
Map<String, Object> record = new HashMap<>(); Map<String, Object> record = new HashMap<>();
record.put("time", timestamp); record.put("time", timestamp);
record.put("user", user++); record.put("user", user++);
@ -227,10 +235,10 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
// Assert we haven't violated the limit too much // Assert we haven't violated the limit too much
GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0);
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.OK)); assertThat(modelSizeStats.getModelBytes(), lessThan(95000000L));
assertThat(modelSizeStats.getModelBytes(), lessThan(45000000L)); assertThat(modelSizeStats.getModelBytes(), greaterThan(75000000L));
assertThat(modelSizeStats.getModelBytes(), greaterThan(35000000L));
assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(140000L)); assertThat(modelSizeStats.getTotalOverFieldCount(), greaterThan(140000L));
assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.OK));
} }
private static Map<String, Object> createRecord(long timestamp, String user, String department) { private static Map<String, Object> createRecord(long timestamp, String user, String department) {

1
plugin/src/test/resources/org/elasticsearch/transport/handlers
View File

@ -99,6 +99,7 @@ internal:cluster/nodes/indices/shard/store[n]
internal:cluster/shard/failure internal:cluster/shard/failure
internal:cluster/shard/started internal:cluster/shard/started
internal:cluster/snapshot/update_snapshot internal:cluster/snapshot/update_snapshot
internal:cluster/snapshot/update_snapshot_status
internal:discovery/zen/fd/master_ping internal:discovery/zen/fd/master_ping
internal:discovery/zen/fd/ping internal:discovery/zen/fd/ping
internal:discovery/zen/join internal:discovery/zen/join