honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TEST] Give jobs meaningful names in ML integ tests (elastic/x-pack-elasticsearch#1103) 

Original commit: elastic/x-pack-elasticsearch@aaf1807172
This commit is contained in:
David Kyle 2017-04-19 12:00:53 +01:00 committed by GitHub
parent 5223acdd9f
commit d8a70138cd
8 changed files with 96 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
View File

@ -62,7 +62,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class AutodetectResultProcessorIT extends ESSingleNodeTestCase { public class AutodetectResultProcessorIT extends ESSingleNodeTestCase {
private static final String JOB_ID = "foo"; private static final String JOB_ID = "autodetect-result-processor-it-job";
private Renormalizer renormalizer; private Renormalizer renormalizer;
private JobResultsPersister jobResultsPersister; private JobResultsPersister jobResultsPersister;

22
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
View File

@ -47,7 +47,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
internalCluster().ensureAtLeastNumDataNodes(4); internalCluster().ensureAtLeastNumDataNodes(4);
ensureStableCluster(4); ensureStableCluster(4);
Job.Builder job = createJob("job_id"); Job.Builder job = createJob("fail-over-basics-job");
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
@ -83,7 +83,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
internalCluster().ensureAtLeastNumDataNodes(4); internalCluster().ensureAtLeastNumDataNodes(4);
ensureStableCluster(4); ensureStableCluster(4);
Job.Builder job = createScheduledJob("job_id"); Job.Builder job = createScheduledJob("fail-over-basics_with-data-feeder-job");
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
@ -157,7 +157,8 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
} }
ensureStableCluster(3); ensureStableCluster(3);
Job.Builder job = createJob("job_id"); String jobId = "dedicated-ml-node-job";
Job.Builder job = createJob(jobId);
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
@ -167,7 +168,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
assertBusy(() -> { assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
PersistentTask task = tasks.getTask(MlMetadata.jobTaskId("job_id")); PersistentTask task = tasks.getTask(MlMetadata.jobTaskId(jobId));
DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode()); DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
Map<String, String> expectedNodeAttr = new HashMap<>(); Map<String, String> expectedNodeAttr = new HashMap<>();
@ -184,7 +185,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
assertBusy(() -> { assertBusy(() -> {
// job should get and remain in a failed state and // job should get and remain in a failed state and
// the status remains to be opened as from ml we didn't had the chance to set the status to failed: // the status remains to be opened as from ml we didn't had the chance to set the status to failed:
assertJobTask("job_id", JobState.OPENED, false); assertJobTask(jobId, JobState.OPENED, false);
}); });
logger.info("start ml node"); logger.info("start ml node");
@ -192,7 +193,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
ensureStableCluster(3); ensureStableCluster(3);
assertBusy(() -> { assertBusy(() -> {
// job should be re-opened: // job should be re-opened:
assertJobTask("job_id", JobState.OPENED, true); assertJobTask(jobId, JobState.OPENED, true);
}); });
} }
@ -322,7 +323,8 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
.put(MachineLearning.ML_ENABLED.getKey(), true)); .put(MachineLearning.ML_ENABLED.getKey(), true));
ensureStableCluster(2); ensureStableCluster(2);
Job.Builder job = createFareQuoteJob("job_id"); String jobId = "ml-indices-not-available-job";
Job.Builder job = createFareQuoteJob(jobId);
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
@ -330,7 +332,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId()); OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId());
client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet(); client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
PostDataAction.Request postDataRequest = new PostDataAction.Request("job_id"); PostDataAction.Request postDataRequest = new PostDataAction.Request(jobId);
postDataRequest.setContent(new BytesArray( postDataRequest.setContent(new BytesArray(
"{\"airline\":\"AAL\",\"responsetime\":\"132.2046\",\"sourcetype\":\"farequote\",\"time\":\"1403481600\"}\n" + "{\"airline\":\"AAL\",\"responsetime\":\"132.2046\",\"sourcetype\":\"farequote\",\"time\":\"1403481600\"}\n" +
"{\"airline\":\"JZA\",\"responsetime\":\"990.4628\",\"sourcetype\":\"farequote\",\"time\":\"1403481700\"}" "{\"airline\":\"JZA\",\"responsetime\":\"990.4628\",\"sourcetype\":\"farequote\",\"time\":\"1403481700\"}"
@ -338,7 +340,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
PostDataAction.Response response = client().execute(PostDataAction.INSTANCE, postDataRequest).actionGet(); PostDataAction.Response response = client().execute(PostDataAction.INSTANCE, postDataRequest).actionGet();
assertEquals(2, response.getDataCounts().getProcessedRecordCount()); assertEquals(2, response.getDataCounts().getProcessedRecordCount());
CloseJobAction.Request closeJobRequest = new CloseJobAction.Request("job_id"); CloseJobAction.Request closeJobRequest = new CloseJobAction.Request(jobId);
client().execute(CloseJobAction.INSTANCE, closeJobRequest).actionGet(); client().execute(CloseJobAction.INSTANCE, closeJobRequest).actionGet();
assertBusy(() -> { assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
@ -362,7 +364,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
ensureStableCluster(2, nonMlNode); ensureStableCluster(2, nonMlNode);
ensureYellow(); // at least the primary shards of the indices a job uses should be started ensureYellow(); // at least the primary shards of the indices a job uses should be started
client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet(); client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
assertBusy(() -> assertJobTask("job_id", JobState.OPENED, true)); assertBusy(() -> assertJobTask(jobId, JobState.OPENED, true));
} }
private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) { private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) {

14
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
View File

@ -43,8 +43,8 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
public void testFailOver() throws Exception { public void testFailOver() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(3); internalCluster().ensureAtLeastNumDataNodes(3);
ensureStableClusterOnAllNodes(3); ensureStableClusterOnAllNodes(3);
run(() -> { run("fail-over-job", () -> {
GetJobsStatsAction.Request request = new GetJobsStatsAction.Request("job_id"); GetJobsStatsAction.Request request = new GetJobsStatsAction.Request("fail-over-job");
GetJobsStatsAction.Response response = client().execute(GetJobsStatsAction.INSTANCE, request).actionGet(); GetJobsStatsAction.Response response = client().execute(GetJobsStatsAction.INSTANCE, request).actionGet();
DiscoveryNode discoveryNode = response.getResponse().results().get(0).getNode(); DiscoveryNode discoveryNode = response.getResponse().results().get(0).getNode();
internalCluster().stopRandomNode(settings -> discoveryNode.getName().equals(settings.get("node.name"))); internalCluster().stopRandomNode(settings -> discoveryNode.getName().equals(settings.get("node.name")));
@ -67,7 +67,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
.put("node.master", false) .put("node.master", false)
.build()); .build());
ensureStableClusterOnAllNodes(2); ensureStableClusterOnAllNodes(2);
run(() -> { run("lose-dedicated-master-node-job", () -> {
logger.info("Stopping dedicated master node"); logger.info("Stopping dedicated master node");
internalCluster().stopRandomNode(settings -> settings.getAsBoolean("node.master", false)); internalCluster().stopRandomNode(settings -> settings.getAsBoolean("node.master", false));
assertBusy(() -> { assertBusy(() -> {
@ -88,14 +88,14 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
public void testFullClusterRestart() throws Exception { public void testFullClusterRestart() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(3); internalCluster().ensureAtLeastNumDataNodes(3);
ensureStableClusterOnAllNodes(3); ensureStableClusterOnAllNodes(3);
run(() -> { run("full-cluster-restart-job", () -> {
logger.info("Restarting all nodes"); logger.info("Restarting all nodes");
internalCluster().fullRestart(); internalCluster().fullRestart();
logger.info("Restarted all nodes"); logger.info("Restarted all nodes");
}); });
} }
private void run(CheckedRunnable<Exception> disrupt) throws Exception { private void run(String jobId, CheckedRunnable<Exception> disrupt) throws Exception {
client().admin().indices().prepareCreate("data") client().admin().indices().prepareCreate("data")
.addMapping("type", "time", "type=date") .addMapping("type", "time", "type=date")
.get(); .get();
@ -105,7 +105,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
long twoWeeksAgo = weekAgo - 604800000; long twoWeeksAgo = weekAgo - 604800000;
indexDocs(logger, "data", numDocs1, twoWeeksAgo, weekAgo); indexDocs(logger, "data", numDocs1, twoWeeksAgo, weekAgo);
Job.Builder job = createScheduledJob("job_id"); Job.Builder job = createScheduledJob(jobId);
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
@ -142,7 +142,7 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
assertFalse(task.needsReassignment(clusterState.nodes())); assertFalse(task.needsReassignment(clusterState.nodes()));
} }
GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request("job_id"); GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(jobId);
JobStats jobStats = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet() JobStats jobStats = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet()
.getResponse().results().get(0); .getResponse().results().get(0);
assertEquals(JobState.OPENED, jobStats.getState()); assertEquals(JobState.OPENED, jobStats.getState());

110
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
View File

@ -60,11 +60,11 @@ public class MlJobIT extends ESRestTestCase {
"} } } }"; "} } } }";
public void testPutJob_GivenFarequoteConfig() throws Exception { public void testPutJob_GivenFarequoteConfig() throws Exception {
Response response = createFarequoteJob(); Response response = createFarequoteJob("given-farequote-config-job");
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response); String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-farequote-config-job\""));
} }
public void testGetJob_GivenNoSuchJob() throws Exception { public void testGetJob_GivenNoSuchJob() throws Exception {
@ -76,18 +76,20 @@ public class MlJobIT extends ESRestTestCase {
} }
public void testGetJob_GivenJobExists() throws Exception { public void testGetJob_GivenJobExists() throws Exception {
createFarequoteJob(); createFarequoteJob("get-job_given-job-exists-job");
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/farequote/_stats"); Response response = client().performRequest("get",
MachineLearning.BASE_PATH + "anomaly_detectors/get-job_given-job-exists-job/_stats");
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response); String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); assertThat(responseAsString, containsString("\"job_id\":\"get-job_given-job-exists-job\""));
} }
public void testGetJobs_GivenSingleJob() throws Exception { public void testGetJobs_GivenSingleJob() throws Exception {
createFarequoteJob(); String jobId = "get-jobs_given-single-job-job";
createFarequoteJob(jobId);
// Explicit _all // Explicit _all
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/_all"); Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/_all");
@ -95,7 +97,7 @@ public class MlJobIT extends ESRestTestCase {
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response); String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); assertThat(responseAsString, containsString("\"job_id\":\"" + jobId + "\""));
// Implicit _all // Implicit _all
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors"); response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors");
@ -103,13 +105,13 @@ public class MlJobIT extends ESRestTestCase {
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
responseAsString = responseEntityToString(response); responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); assertThat(responseAsString, containsString("\"job_id\":\"" + jobId + "\""));
} }
public void testGetJobs_GivenMultipleJobs() throws Exception { public void testGetJobs_GivenMultipleJobs() throws Exception {
createFarequoteJob("farequote_1"); createFarequoteJob("given-multiple-jobs-job-1");
createFarequoteJob("farequote_2"); createFarequoteJob("given-multiple-jobs-job-2");
createFarequoteJob("farequote_3"); createFarequoteJob("given-multiple-jobs-job-3");
// Explicit _all // Explicit _all
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/_all"); Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/_all");
@ -117,9 +119,9 @@ public class MlJobIT extends ESRestTestCase {
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response); String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":3")); assertThat(responseAsString, containsString("\"count\":3"));
assertThat(responseAsString, containsString("\"job_id\":\"farequote_1\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-1\""));
assertThat(responseAsString, containsString("\"job_id\":\"farequote_2\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-2\""));
assertThat(responseAsString, containsString("\"job_id\":\"farequote_3\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-3\""));
// Implicit _all // Implicit _all
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors"); response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors");
@ -127,13 +129,9 @@ public class MlJobIT extends ESRestTestCase {
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
responseAsString = responseEntityToString(response); responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":3")); assertThat(responseAsString, containsString("\"count\":3"));
assertThat(responseAsString, containsString("\"job_id\":\"farequote_1\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-1\""));
assertThat(responseAsString, containsString("\"job_id\":\"farequote_2\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-2\""));
assertThat(responseAsString, containsString("\"job_id\":\"farequote_3\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-3\""));
}
private Response createFarequoteJob() throws Exception {
return createFarequoteJob("farequote");
} }
private Response createFarequoteJob(String jobId) throws Exception { private Response createFarequoteJob(String jobId) throws Exception {
@ -153,23 +151,27 @@ public class MlJobIT extends ESRestTestCase {
params.put("start", "1200"); // inclusive params.put("start", "1200"); // inclusive
params.put("end", "1400"); // exclusive params.put("end", "1400"); // exclusive
String jobId = "get-bucket-results-job";
ResponseException e = expectThrows(ResponseException.class, ResponseException e = expectThrows(ResponseException.class,
() -> client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/buckets", params)); () -> client().performRequest("get",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/buckets", params));
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
assertThat(e.getMessage(), containsString("No known job with id '1'")); assertThat(e.getMessage(), containsString("No known job with id '" + jobId + "'"));
createFarequoteJob("1"); createFarequoteJob(jobId);
addBucketResult("1", "1234", 1); addBucketResult(jobId, "1234", 1);
addBucketResult("1", "1235", 1); addBucketResult(jobId, "1235", 1);
addBucketResult("1", "1236", 1); addBucketResult(jobId, "1236", 1);
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/buckets", params); Response response = client().performRequest("get",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/buckets", params);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response); String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":3")); assertThat(responseAsString, containsString("\"count\":3"));
params.put("end", "1235"); params.put("end", "1235");
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/buckets", params); response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/buckets", params);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
responseAsString = responseEntityToString(response); responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
@ -180,10 +182,10 @@ public class MlJobIT extends ESRestTestCase {
assertThat(e.getMessage(), containsString("No known job with id '2'")); assertThat(e.getMessage(), containsString("No known job with id '2'"));
e = expectThrows(ResponseException.class, () -> client().performRequest("get", e = expectThrows(ResponseException.class, () -> client().performRequest("get",
MachineLearning.BASE_PATH + "anomaly_detectors/1/results/buckets/1")); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/buckets/1"));
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/buckets/1234"); response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/buckets/1234");
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
responseAsString = responseEntityToString(response); responseAsString = responseEntityToString(response);
assertThat(responseAsString, not(isEmptyString())); assertThat(responseAsString, not(isEmptyString()));
@ -194,23 +196,26 @@ public class MlJobIT extends ESRestTestCase {
params.put("start", "1200"); // inclusive params.put("start", "1200"); // inclusive
params.put("end", "1400"); // exclusive params.put("end", "1400"); // exclusive
String jobId = "get-record-results-job";
ResponseException e = expectThrows(ResponseException.class, ResponseException e = expectThrows(ResponseException.class,
() -> client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/records", params)); () -> client().performRequest("get",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/records", params));
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
assertThat(e.getMessage(), containsString("No known job with id '1'")); assertThat(e.getMessage(), containsString("No known job with id '" + jobId + "'"));
createFarequoteJob("1"); createFarequoteJob(jobId);
addRecordResult("1", "1234", 1, 1); addRecordResult(jobId, "1234", 1, 1);
addRecordResult("1", "1235", 1, 2); addRecordResult(jobId, "1235", 1, 2);
addRecordResult("1", "1236", 1, 3); addRecordResult(jobId, "1236", 1, 3);
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/records", params); Response response = client().performRequest("get",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/records", params);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response); String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":3")); assertThat(responseAsString, containsString("\"count\":3"));
params.put("end", "1235"); params.put("end", "1235");
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/1/results/records", params); response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/results/records", params);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
responseAsString = responseEntityToString(response); responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
@ -225,18 +230,19 @@ public class MlJobIT extends ESRestTestCase {
String jobConfig = String.format(Locale.ROOT, jobTemplate, "index-1"); String jobConfig = String.format(Locale.ROOT, jobTemplate, "index-1");
Response response = client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/repeated-id" , String jobId = "cant-create-job-with-same-id-job";
Response response = client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId ,
Collections.emptyMap(), Collections.emptyMap(),
new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); new StringEntity(jobConfig, ContentType.APPLICATION_JSON));
assertEquals(200, response.getStatusLine().getStatusCode()); assertEquals(200, response.getStatusLine().getStatusCode());
final String jobConfig2 = String.format(Locale.ROOT, jobTemplate, "index-2"); final String jobConfig2 = String.format(Locale.ROOT, jobTemplate, "index-2");
ResponseException e = expectThrows(ResponseException.class, ResponseException e = expectThrows(ResponseException.class,
() ->client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/repeated-id" , () ->client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId,
Collections.emptyMap(), new StringEntity(jobConfig2, ContentType.APPLICATION_JSON))); Collections.emptyMap(), new StringEntity(jobConfig2, ContentType.APPLICATION_JSON)));
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400));
assertThat(e.getMessage(), containsString("The job cannot be created with the Id 'repeated-id'. The Id is already used.")); assertThat(e.getMessage(), containsString("The job cannot be created with the Id '" + jobId + "'. The Id is already used."));
} }
public void testCreateJobsWithIndexNameOption() throws Exception { public void testCreateJobsWithIndexNameOption() throws Exception {
@ -246,7 +252,7 @@ public class MlJobIT extends ESRestTestCase {
" },\n" + " },\n" +
" \"results_index_name\" : \"%s\"}"; " \"results_index_name\" : \"%s\"}";
String jobId1 = "aliased-job-1"; String jobId1 = "create-jobs-with-index-name-option-job-1";
String indexName = "non-default-index"; String indexName = "non-default-index";
String jobConfig = String.format(Locale.ROOT, jobTemplate, indexName); String jobConfig = String.format(Locale.ROOT, jobTemplate, indexName);
@ -254,7 +260,7 @@ public class MlJobIT extends ESRestTestCase {
+ "anomaly_detectors/" + jobId1, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); + "anomaly_detectors/" + jobId1, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON));
assertEquals(200, response.getStatusLine().getStatusCode()); assertEquals(200, response.getStatusLine().getStatusCode());
String jobId2 = "aliased-job-2"; String jobId2 = "create-jobs-with-index-name-option-job-2";
response = client().performRequest("put", MachineLearning.BASE_PATH response = client().performRequest("put", MachineLearning.BASE_PATH
+ "anomaly_detectors/" + jobId2, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); + "anomaly_detectors/" + jobId2, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON));
assertEquals(200, response.getStatusLine().getStatusCode()); assertEquals(200, response.getStatusLine().getStatusCode());
@ -335,9 +341,9 @@ public class MlJobIT extends ESRestTestCase {
" }\n" + " }\n" +
"}"; "}";
String jobId1 = "job-1"; String jobId1 = "create-job-in-shared-index-updates-mapping-job-1";
String byFieldName1 = "responsetime"; String byFieldName1 = "responsetime";
String jobId2 = "job-2"; String jobId2 = "create-job-in-shared-index-updates-mapping-job-2";
String byFieldName2 = "cpu-usage"; String byFieldName2 = "cpu-usage";
String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1); String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1);
@ -374,9 +380,9 @@ public class MlJobIT extends ESRestTestCase {
" },\n" + " },\n" +
" \"results_index_name\" : \"shared-index\"}"; " \"results_index_name\" : \"shared-index\"}";
String jobId1 = "job-1"; String jobId1 = "create-job-in-custom-shared-index-updates-mapping-job-1";
String byFieldName1 = "responsetime"; String byFieldName1 = "responsetime";
String jobId2 = "job-2"; String jobId2 = "create-job-in-custom-shared-index-updates-mapping-job-2";
String byFieldName2 = "cpu-usage"; String byFieldName2 = "cpu-usage";
String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1); String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1);
@ -432,7 +438,7 @@ public class MlJobIT extends ESRestTestCase {
} }
public void testDeleteJob() throws Exception { public void testDeleteJob() throws Exception {
String jobId = "foo"; String jobId = "delete-job-job";
String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
@ -468,7 +474,7 @@ public class MlJobIT extends ESRestTestCase {
} }
public void testDeleteJobAfterMissingIndex() throws Exception { public void testDeleteJobAfterMissingIndex() throws Exception {
String jobId = "foo"; String jobId = "delete-job-after-missing-index-job";
String aliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); String aliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
@ -498,7 +504,7 @@ public class MlJobIT extends ESRestTestCase {
} }
public void testDeleteJobAfterMissingAlias() throws Exception { public void testDeleteJobAfterMissingAlias() throws Exception {
String jobId = "foo"; String jobId = "delete-job-after-missing-alias-job";
String aliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); String aliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
@ -521,7 +527,7 @@ public class MlJobIT extends ESRestTestCase {
} }
public void testMultiIndexDelete() throws Exception { public void testMultiIndexDelete() throws Exception {
String jobId = "foo"; String jobId = "multi-index-delete-job";
String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndex.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndex.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);

16
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
View File

@ -28,34 +28,34 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
startMlCluster(1, 1); startMlCluster(1, 1);
// create and open first job, which succeeds: // create and open first job, which succeeds:
Job.Builder job = createJob("1"); Job.Builder job = createJob("close-failed-job-1");
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())).get(); client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())).get();
assertBusy(() -> { assertBusy(() -> {
GetJobsStatsAction.Response statsResponse = GetJobsStatsAction.Response statsResponse =
client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("1")).actionGet(); client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("close-failed-job-1")).actionGet();
assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED); assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED);
}); });
// create and try to open second job, which fails: // create and try to open second job, which fails:
job = createJob("2"); job = createJob("close-failed-job-2");
putJobRequest = new PutJobAction.Request(job); putJobRequest = new PutJobAction.Request(job);
putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
expectThrows(ElasticsearchStatusException.class, expectThrows(ElasticsearchStatusException.class,
() -> client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request("2")).actionGet()); () -> client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request("close-failed-job-2")).actionGet());
// Ensure that the second job didn't even attempt to be opened and we still have 1 job open: // Ensure that the second job didn't even attempt to be opened and we still have 1 job open:
GetJobsStatsAction.Response statsResponse = GetJobsStatsAction.Response statsResponse =
client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("2")).actionGet(); client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request("close-failed-job-2")).actionGet();
assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.CLOSED); assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.CLOSED);
ClusterState state = client().admin().cluster().prepareState().get().getState(); ClusterState state = client().admin().cluster().prepareState().get().getState();
PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
assertEquals(1, tasks.taskMap().size()); assertEquals(1, tasks.taskMap().size());
// now just double check that the first job is still opened: // now just double check that the first job is still opened:
PersistentTasksCustomMetaData.PersistentTask task = tasks.getTask(MlMetadata.jobTaskId("1")); PersistentTasksCustomMetaData.PersistentTask task = tasks.getTask(MlMetadata.jobTaskId("close-failed-job-1"));
assertEquals(JobState.OPENED, ((JobTaskStatus) task.getStatus()).getState()); assertEquals(JobState.OPENED, ((JobTaskStatus) task.getStatus()).getState());
} }
@ -71,7 +71,7 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
startMlCluster(numNodes, maxNumberOfJobsPerNode); startMlCluster(numNodes, maxNumberOfJobsPerNode);
int clusterWideMaxNumberOfJobs = numNodes * maxNumberOfJobsPerNode; int clusterWideMaxNumberOfJobs = numNodes * maxNumberOfJobsPerNode;
for (int i = 1; i <= (clusterWideMaxNumberOfJobs + 1); i++) { for (int i = 1; i <= (clusterWideMaxNumberOfJobs + 1); i++) {
Job.Builder job = createJob(Integer.toString(i)); Job.Builder job = createJob("max-number-of-jobs-limit-job-" + Integer.toString(i));
PutJobAction.Request putJobRequest = new PutJobAction.Request(job); PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get();
assertTrue(putJobResponse.isAcknowledged()); assertTrue(putJobResponse.isAcknowledged());
@ -92,7 +92,7 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
logger.info("good news everybody --> reached maximum number of allowed opened jobs, after trying to open the {}th job", i); logger.info("good news everybody --> reached maximum number of allowed opened jobs, after trying to open the {}th job", i);
// close the first job and check if the latest job gets opened: // close the first job and check if the latest job gets opened:
CloseJobAction.Request closeRequest = new CloseJobAction.Request("1"); CloseJobAction.Request closeRequest = new CloseJobAction.Request("max-number-of-jobs-limit-job-1");
closeRequest.setCloseTimeout(TimeValue.timeValueSeconds(20L)); closeRequest.setCloseTimeout(TimeValue.timeValueSeconds(20L));
CloseJobAction.Response closeResponse = client().execute(CloseJobAction.INSTANCE, closeRequest).actionGet(); CloseJobAction.Response closeResponse = client().execute(CloseJobAction.INSTANCE, closeRequest).actionGet();
assertTrue(closeResponse.isClosed()); assertTrue(closeResponse.isClosed());

6
qa/ml-basic-multi-node/src/test/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java
View File

@ -35,7 +35,7 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
} }
public void testMiniFarequote() throws Exception { public void testMiniFarequote() throws Exception {
String jobId = "foo1"; String jobId = "mini-farequote-job";
createFarequoteJob(jobId); createFarequoteJob(jobId);
Response response = client().performRequest("post", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open"); Response response = client().performRequest("post", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open");
@ -113,7 +113,7 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
// Ensure all data is searchable // Ensure all data is searchable
client().performRequest("post", "_refresh"); client().performRequest("post", "_refresh");
String jobId = "foo2"; String jobId = "mini-farequote-with-data-feeder-job";
createFarequoteJob(jobId); createFarequoteJob(jobId);
String datafeedId = "bar"; String datafeedId = "bar";
createDatafeed(datafeedId, jobId); createDatafeed(datafeedId, jobId);
@ -159,7 +159,7 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
} }
public void testMiniFarequoteReopen() throws Exception { public void testMiniFarequoteReopen() throws Exception {
String jobId = "foo1_again"; String jobId = "mini-farequote-reopen";
createFarequoteJob(jobId); createFarequoteJob(jobId);
Response response = client().performRequest("post", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open"); Response response = client().performRequest("post", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open");

2
qa/ml-disabled/src/test/java/org/elasticsearch/xpack/ml/integration/MlPluginDisabledIT.java
View File

@ -26,7 +26,7 @@ public class MlPluginDisabledIT extends ESRestTestCase {
public void testActionsFail() throws Exception { public void testActionsFail() throws Exception {
XContentBuilder xContentBuilder = jsonBuilder(); XContentBuilder xContentBuilder = jsonBuilder();
xContentBuilder.startObject(); xContentBuilder.startObject();
xContentBuilder.field("job_id", "foo"); xContentBuilder.field("actions-fail-job", "foo");
xContentBuilder.field("description", "Analysis of response time by airline"); xContentBuilder.field("description", "Analysis of response time by airline");
xContentBuilder.startObject("analysis_config"); xContentBuilder.startObject("analysis_config");

12
qa/ml-single-node-tests/src/test/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
View File

@ -273,9 +273,9 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
" }\n" + " }\n" +
" }"; " }";
client().performRequest("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/painless", Collections.emptyMap(), client().performRequest("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job", Collections.emptyMap(),
new StringEntity(job, ContentType.APPLICATION_JSON)); new StringEntity(job, ContentType.APPLICATION_JSON));
client().performRequest("POST", MachineLearning.BASE_PATH + "anomaly_detectors/painless/_open"); client().performRequest("POST", MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job/_open");
// Create index to hold data // Create index to hold data
Settings.Builder settings = Settings.builder() Settings.Builder settings = Settings.builder()
@ -318,7 +318,7 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
// Create and start datafeed // Create and start datafeed
String body = "{\n" + String body = "{\n" +
" \"job_id\":\"painless\",\n" + " \"job_id\":\"hrd-split-job\",\n" +
" \"indexes\":[\"painless\"],\n" + " \"indexes\":[\"painless\"],\n" +
" \"types\":[\"test\"],\n" + " \"types\":[\"test\"],\n" +
" \"script_fields\": {\n" + " \"script_fields\": {\n" +
@ -328,16 +328,16 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
" }\n" + " }\n" +
" }"; " }";
client().performRequest("PUT", MachineLearning.BASE_PATH + "datafeeds/painless", Collections.emptyMap(), client().performRequest("PUT", MachineLearning.BASE_PATH + "datafeeds/hrd-split-datafeed", Collections.emptyMap(),
new StringEntity(body, ContentType.APPLICATION_JSON)); new StringEntity(body, ContentType.APPLICATION_JSON));
client().performRequest("POST", MachineLearning.BASE_PATH + "datafeeds/painless/_start"); client().performRequest("POST", MachineLearning.BASE_PATH + "datafeeds/hrd-split-datafeed/_start");
boolean passed = awaitBusy(() -> { boolean passed = awaitBusy(() -> {
try { try {
client().performRequest("POST", "/_refresh"); client().performRequest("POST", "/_refresh");
Response response = client().performRequest("GET", Response response = client().performRequest("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/painless/results/records"); MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job/results/records");
String responseBody = EntityUtils.toString(response.getEntity()); String responseBody = EntityUtils.toString(response.getEntity());
if (responseBody.contains("\"count\":2")) { if (responseBody.contains("\"count\":2")) {