honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch remaining ml tests to new style Requests (#33107) 

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack/plugin/ml/qa/native-multi-node-tests`,
`x-pack/plugin/ml/qa/single-node-tests` projects to use the new
versions.
This commit is contained in:
Nik Everett 2018-08-24 16:36:40 -04:00 committed by GitHub
parent a9a66a09dc
commit 8bee6b3a92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 580 additions and 586 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

597
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
View File

@ -5,9 +5,9 @@
*/ */
package org.elasticsearch.xpack.ml.integration; package org.elasticsearch.xpack.ml.integration;
import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils;
import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Request;
import org.apache.http.message.BasicHeader; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response; import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClient;
@ -22,10 +22,7 @@ import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
@ -36,6 +33,7 @@ import java.util.stream.Collectors;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
public class DatafeedJobsRestIT extends ESRestTestCase { public class DatafeedJobsRestIT extends ESRestTestCase {
@ -57,26 +55,24 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
} }
private void setupDataAccessRole(String index) throws IOException { private void setupDataAccessRole(String index) throws IOException {
String json = "{" Request request = new Request("PUT", "/_xpack/security/role/test_data_access");
request.setJsonEntity("{"
+ " \"indices\" : [" + " \"indices\" : ["
+ " { \"names\": [\"" + index + "\"], \"privileges\": [\"read\"] }" + " { \"names\": [\"" + index + "\"], \"privileges\": [\"read\"] }"
+ " ]" + " ]"
+ "}"; + "}");
client().performRequest(request);
client().performRequest("put", "_xpack/security/role/test_data_access", Collections.emptyMap(),
new StringEntity(json, ContentType.APPLICATION_JSON));
} }
private void setupUser(String user, List<String> roles) throws IOException { private void setupUser(String user, List<String> roles) throws IOException {
String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars()); String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars());
String json = "{" Request request = new Request("PUT", "/_xpack/security/user/" + user);
request.setJsonEntity("{"
+ " \"password\" : \"" + password + "\"," + " \"password\" : \"" + password + "\","
+ " \"roles\" : [ " + roles.stream().map(unquoted -> "\"" + unquoted + "\"").collect(Collectors.joining(", ")) + " ]" + " \"roles\" : [ " + roles.stream().map(unquoted -> "\"" + unquoted + "\"").collect(Collectors.joining(", ")) + " ]"
+ "}"; + "}");
client().performRequest(request);
client().performRequest("put", "_xpack/security/user/" + user, Collections.emptyMap(),
new StringEntity(json, ContentType.APPLICATION_JSON));
} }
@Before @Before
@ -92,7 +88,10 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
} }
private void addAirlineData() throws IOException { private void addAirlineData() throws IOException {
String mappings = "{" StringBuilder bulk = new StringBuilder();
Request createEmptyAirlineDataRequest = new Request("PUT", "/airline-data-empty");
createEmptyAirlineDataRequest.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"response\": {" + " \"response\": {"
+ " \"properties\": {" + " \"properties\": {"
@ -102,12 +101,12 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");
client().performRequest("put", "airline-data-empty", Collections.emptyMap(), client().performRequest(createEmptyAirlineDataRequest);
new StringEntity(mappings, ContentType.APPLICATION_JSON));
// Create index with source = enabled, doc_values = enabled, stored = false + multi-field // Create index with source = enabled, doc_values = enabled, stored = false + multi-field
mappings = "{" Request createAirlineDataRequest = new Request("PUT", "/airline-data");
createAirlineDataRequest.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"response\": {" + " \"response\": {"
+ " \"properties\": {" + " \"properties\": {"
@ -123,18 +122,17 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");
client().performRequest("put", "airline-data", Collections.emptyMap(), new StringEntity(mappings, ContentType.APPLICATION_JSON)); client().performRequest(createAirlineDataRequest);
client().performRequest("put", "airline-data/response/1", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"airline-data\", \"_type\": \"response\", \"_id\": 1}}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}", bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"airline-data\", \"_type\": \"response\", \"_id\": 2}}\n");
client().performRequest("put", "airline-data/response/2", Collections.emptyMap(), bulk.append("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}",
ContentType.APPLICATION_JSON));
// Create index with source = enabled, doc_values = disabled (except time), stored = false // Create index with source = enabled, doc_values = disabled (except time), stored = false
mappings = "{" Request createAirlineDataDisabledDocValues = new Request("PUT", "/airline-data-disabled-doc-values");
createAirlineDataDisabledDocValues.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"response\": {" + " \"response\": {"
+ " \"properties\": {" + " \"properties\": {"
@ -144,19 +142,17 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");
client().performRequest("put", "airline-data-disabled-doc-values", Collections.emptyMap(), client().performRequest(createAirlineDataDisabledDocValues);
new StringEntity(mappings, ContentType.APPLICATION_JSON));
client().performRequest("put", "airline-data-disabled-doc-values/response/1", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-doc-values\", \"_type\": \"response\", \"_id\": 1}}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}", bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-doc-values\", \"_type\": \"response\", \"_id\": 2}}\n");
client().performRequest("put", "airline-data-disabled-doc-values/response/2", Collections.emptyMap(), bulk.append("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}",
ContentType.APPLICATION_JSON));
// Create index with source = disabled, doc_values = enabled (except time), stored = true // Create index with source = disabled, doc_values = enabled (except time), stored = true
mappings = "{" Request createAirlineDataDisabledSource = new Request("PUT", "/airline-data-disabled-source");
createAirlineDataDisabledSource.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"response\": {" + " \"response\": {"
+ " \"_source\":{\"enabled\":false}," + " \"_source\":{\"enabled\":false},"
@ -167,19 +163,16 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");
client().performRequest("put", "airline-data-disabled-source", Collections.emptyMap(),
new StringEntity(mappings, ContentType.APPLICATION_JSON));
client().performRequest("put", "airline-data-disabled-source/response/1", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-source\", \"_type\": \"response\", \"_id\": 1}}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}", bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-source\", \"_type\": \"response\", \"_id\": 2}}\n");
client().performRequest("put", "airline-data-disabled-source/response/2", Collections.emptyMap(), bulk.append("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}",
ContentType.APPLICATION_JSON));
// Create index with nested documents // Create index with nested documents
mappings = "{" Request createAirlineDataNested = new Request("PUT", "/nested-data");
createAirlineDataNested.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"response\": {" + " \"response\": {"
+ " \"properties\": {" + " \"properties\": {"
@ -187,18 +180,17 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");
client().performRequest("put", "nested-data", Collections.emptyMap(), new StringEntity(mappings, ContentType.APPLICATION_JSON)); client().performRequest(createAirlineDataNested);
client().performRequest("put", "nested-data/response/1", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"nested-data\", \"_type\": \"response\", \"_id\": 1}}\n");
new StringEntity("{\"time\":\"2016-06-01T00:00:00Z\", \"responsetime\":{\"millis\":135.22}}", bulk.append("{\"time\":\"2016-06-01T00:00:00Z\", \"responsetime\":{\"millis\":135.22}}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"nested-data\", \"_type\": \"response\", \"_id\": 2}}\n");
client().performRequest("put", "nested-data/response/2", Collections.emptyMap(), bulk.append("{\"time\":\"2016-06-01T01:59:00Z\",\"responsetime\":{\"millis\":222.0}}\n");
new StringEntity("{\"time\":\"2016-06-01T01:59:00Z\",\"responsetime\":{\"millis\":222.0}}",
ContentType.APPLICATION_JSON));
// Create index with multiple docs per time interval for aggregation testing // Create index with multiple docs per time interval for aggregation testing
mappings = "{" Request createAirlineDataAggs = new Request("PUT", "/airline-data-aggs");
createAirlineDataAggs.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"response\": {" + " \"response\": {"
+ " \"properties\": {" + " \"properties\": {"
@ -208,43 +200,33 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");
client().performRequest("put", "airline-data-aggs", Collections.emptyMap(), client().performRequest(createAirlineDataAggs);
new StringEntity(mappings, ContentType.APPLICATION_JSON));
client().performRequest("put", "airline-data-aggs/response/1", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 1}}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":100.0}", bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":100.0}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 2}}\n");
client().performRequest("put", "airline-data-aggs/response/2", Collections.emptyMap(), bulk.append("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"AAA\",\"responsetime\":200.0}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"AAA\",\"responsetime\":200.0}", bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 3}}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"BBB\",\"responsetime\":1000.0}\n");
client().performRequest("put", "airline-data-aggs/response/3", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 4}}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"BBB\",\"responsetime\":1000.0}", bulk.append("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"BBB\",\"responsetime\":2000.0}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 5}}\n");
client().performRequest("put", "airline-data-aggs/response/4", Collections.emptyMap(), bulk.append("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"AAA\",\"responsetime\":300.0}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"BBB\",\"responsetime\":2000.0}", bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 6}}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"AAA\",\"responsetime\":400.0}\n");
client().performRequest("put", "airline-data-aggs/response/5", Collections.emptyMap(), bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 7}}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"AAA\",\"responsetime\":300.0}", bulk.append("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"BBB\",\"responsetime\":3000.0}\n");
ContentType.APPLICATION_JSON)); bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_type\": \"response\", \"_id\": 8}}\n");
client().performRequest("put", "airline-data-aggs/response/6", Collections.emptyMap(), bulk.append("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"BBB\",\"responsetime\":4000.0}\n");
new StringEntity("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"AAA\",\"responsetime\":400.0}",
ContentType.APPLICATION_JSON));
client().performRequest("put", "airline-data-aggs/response/7", Collections.emptyMap(),
new StringEntity("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"BBB\",\"responsetime\":3000.0}",
ContentType.APPLICATION_JSON));
client().performRequest("put", "airline-data-aggs/response/8", Collections.emptyMap(),
new StringEntity("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"BBB\",\"responsetime\":4000.0}",
ContentType.APPLICATION_JSON));
// Ensure all data is searchable bulkIndex(bulk.toString());
client().performRequest("post", "_refresh");
} }
private void addNetworkData(String index) throws IOException { private void addNetworkData(String index) throws IOException {
// Create index with source = enabled, doc_values = enabled, stored = false + multi-field // Create index with source = enabled, doc_values = enabled, stored = false + multi-field
String mappings = "{" Request createIndexRequest = new Request("PUT", index);
createIndexRequest.setJsonEntity("{"
+ " \"mappings\": {" + " \"mappings\": {"
+ " \"doc\": {" + " \"doc\": {"
+ " \"properties\": {" + " \"properties\": {"
@ -260,27 +242,25 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ " }" + " }"
+ " }" + " }"
+ " }" + " }"
+ "}"; + "}");;
client().performRequest("put", index, Collections.emptyMap(), new StringEntity(mappings, ContentType.APPLICATION_JSON)); client().performRequest(createIndexRequest);
StringBuilder bulk = new StringBuilder();
String docTemplate = "{\"timestamp\":%d,\"host\":\"%s\",\"network_bytes_out\":%d}"; String docTemplate = "{\"timestamp\":%d,\"host\":\"%s\",\"network_bytes_out\":%d}";
Date date = new Date(1464739200735L); Date date = new Date(1464739200735L);
for (int i=0; i<120; i++) { for (int i = 0; i < 120; i++) {
long byteCount = randomNonNegativeLong(); long byteCount = randomNonNegativeLong();
String jsonDoc = String.format(Locale.ROOT, docTemplate, date.getTime(), "hostA", byteCount); bulk.append("{\"index\": {\"_index\": \"").append(index).append("\", \"_type\": \"doc\"}}\n");
client().performRequest("post", index + "/doc", Collections.emptyMap(), bulk.append(String.format(Locale.ROOT, docTemplate, date.getTime(), "hostA", byteCount)).append('\n');
new StringEntity(jsonDoc, ContentType.APPLICATION_JSON));
byteCount = randomNonNegativeLong(); byteCount = randomNonNegativeLong();
jsonDoc = String.format(Locale.ROOT, docTemplate, date.getTime(), "hostB", byteCount); bulk.append("{\"index\": {\"_index\": \"").append(index).append("\", \"_type\": \"doc\"}}\n");
client().performRequest("post", index + "/doc", Collections.emptyMap(), bulk.append(String.format(Locale.ROOT, docTemplate, date.getTime(), "hostB", byteCount)).append('\n');
new StringEntity(jsonDoc, ContentType.APPLICATION_JSON));
date = new Date(date.getTime() + 10_000); date = new Date(date.getTime() + 10_000);
} }
// Ensure all data is searchable bulkIndex(bulk.toString());
client().performRequest("post", "_refresh");
} }
public void testLookbackOnlyWithMixedTypes() throws Exception { public void testLookbackOnlyWithMixedTypes() throws Exception {
@ -314,11 +294,21 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testLookbackOnlyWithNestedFields() throws Exception { public void testLookbackOnlyWithNestedFields() throws Exception {
String jobId = "test-lookback-only-with-nested-fields"; String jobId = "test-lookback-only-with-nested-fields";
String job = "{\"description\":\"Nested job\", \"analysis_config\" : {\"bucket_span\":\"1h\",\"detectors\" :" Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "[{\"function\":\"mean\",\"field_name\":\"responsetime.millis\"}]}, \"data_description\" : {\"time_field\":\"time\"}" createJobRequest.setJsonEntity("{\n"
+ "}"; + " \"description\": \"Nested job\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"analysis_config\": {\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " \"bucket_span\": \"1h\",\n"
+ " \"detectors\": [\n"
+ " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"responsetime.millis\"\n"
+ " }\n"
+ " ]\n"
+ " },"
+ " \"data_description\": {\"time_field\": \"time\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = jobId + "-datafeed"; String datafeedId = jobId + "-datafeed";
new DatafeedBuilder(datafeedId, jobId, "nested-data", "response").build(); new DatafeedBuilder(datafeedId, jobId, "nested-data", "response").build();
@ -326,8 +316,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
@ -340,14 +331,23 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testInsufficientSearchPrivilegesOnPut() throws Exception { public void testInsufficientSearchPrivilegesOnPut() throws Exception {
String jobId = "privs-put-job"; String jobId = "privs-put-job";
String job = "{\"description\":\"Aggs job\",\"analysis_config\" :{\"bucket_span\":\"1h\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\"," + " \"description\": \"Aggs job\",\n"
+ "\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]}," + " \"analysis_config\": {\n"
+ "\"data_description\" : {\"time_field\":\"time stamp\"}" + " \"bucket_span\": \"1h\",\n "
+ "}"; + " \"summary_count_field_name\": \"doc_count\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, + " \"detectors\": [\n"
Collections.emptyMap(), new StringEntity(job, ContentType.APPLICATION_JSON)); + " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"responsetime\",\n"
+ " \"by_field_name\":\"airline\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\" : {\"time_field\": \"time stamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
// This should be disallowed, because even though the ml_admin user has permission to // This should be disallowed, because even though the ml_admin user has permission to
@ -365,14 +365,23 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testInsufficientSearchPrivilegesOnPreview() throws Exception { public void testInsufficientSearchPrivilegesOnPreview() throws Exception {
String jobId = "privs-preview-job"; String jobId = "privs-preview-job";
String job = "{\"description\":\"Aggs job\",\"analysis_config\" :{\"bucket_span\":\"1h\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\"," + " \"description\": \"Aggs job\",\n"
+ "\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]}," + " \"analysis_config\": {\n"
+ "\"data_description\" : {\"time_field\":\"time stamp\"}" + " \"bucket_span\": \"1h\",\n"
+ "}"; + " \"summary_count_field_name\": \"doc_count\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, + " \"detectors\": [\n"
Collections.emptyMap(), new StringEntity(job, ContentType.APPLICATION_JSON)); + " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"responsetime\",\n"
+ " \"by_field_name\": \"airline\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\" : {\"time_field\": \"time stamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").build(); new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").build();
@ -380,10 +389,11 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
// This should be disallowed, because ml_admin is trying to preview a datafeed created by // This should be disallowed, because ml_admin is trying to preview a datafeed created by
// by another user (x_pack_rest_user in this case) that will reveal the content of an index they // by another user (x_pack_rest_user in this case) that will reveal the content of an index they
// don't have permission to search directly // don't have permission to search directly
ResponseException e = expectThrows(ResponseException.class, () -> Request getFeed = new Request("GET", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_preview");
client().performRequest("get", RequestOptions.Builder options = getFeed.getOptions().toBuilder();
MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_preview", options.addHeader("Authorization", BASIC_AUTH_VALUE_ML_ADMIN);
new BasicHeader("Authorization", BASIC_AUTH_VALUE_ML_ADMIN))); getFeed.setOptions(options);
ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getFeed));
assertThat(e.getMessage(), assertThat(e.getMessage(),
containsString("[indices:data/read/field_caps] is unauthorized for user [ml_admin]")); containsString("[indices:data/read/field_caps] is unauthorized for user [ml_admin]"));
@ -391,13 +401,23 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testLookbackOnlyGivenAggregationsWithHistogram() throws Exception { public void testLookbackOnlyGivenAggregationsWithHistogram() throws Exception {
String jobId = "aggs-histogram-job"; String jobId = "aggs-histogram-job";
String job = "{\"description\":\"Aggs job\",\"analysis_config\" :{\"bucket_span\":\"1h\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]}," + " \"description\": \"Aggs job\",\n"
+ "\"data_description\" : {\"time_field\":\"time stamp\"}" + " \"analysis_config\": {\n"
+ "}"; + " \"bucket_span\": \"1h\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"summary_count_field_name\": \"doc_count\",\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " \"detectors\": [\n"
+ " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"responsetime\",\n"
+ " \"by_field_name\": \"airline\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\": {\"time_field\": \"time stamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
String aggregations = "{\"buckets\":{\"histogram\":{\"field\":\"time stamp\",\"interval\":3600000}," String aggregations = "{\"buckets\":{\"histogram\":{\"field\":\"time stamp\",\"interval\":3600000},"
@ -410,8 +430,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(new Request("GET",
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4"));
assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
@ -419,13 +440,23 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testLookbackOnlyGivenAggregationsWithDateHistogram() throws Exception { public void testLookbackOnlyGivenAggregationsWithDateHistogram() throws Exception {
String jobId = "aggs-date-histogram-job"; String jobId = "aggs-date-histogram-job";
String job = "{\"description\":\"Aggs job\",\"analysis_config\" :{\"bucket_span\":\"3600s\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]}," + " \"description\": \"Aggs job\",\n"
+ "\"data_description\" : {\"time_field\":\"time stamp\"}" + " \"analysis_config\": {\n"
+ "}"; + " \"bucket_span\": \"3600s\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"summary_count_field_name\": \"doc_count\",\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " \"detectors\": [\n"
+ " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"responsetime\",\n"
+ " \"by_field_name\": \"airline\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\": {\"time_field\": \"time stamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
String aggregations = "{\"time stamp\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":\"1h\"}," String aggregations = "{\"time stamp\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":\"1h\"},"
@ -438,8 +469,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(new Request("GET",
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4"));
assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
@ -447,13 +479,22 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testLookbackUsingDerivativeAggWithLargerHistogramBucketThanDataRate() throws Exception { public void testLookbackUsingDerivativeAggWithLargerHistogramBucketThanDataRate() throws Exception {
String jobId = "derivative-agg-network-job"; String jobId = "derivative-agg-network-job";
String job = "{\"analysis_config\" :{\"bucket_span\":\"300s\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"bytes-delta\",\"by_field_name\":\"hostname\"}]}," + " \"analysis_config\": {\n"
+ "\"data_description\" : {\"time_field\":\"timestamp\"}" + " \"bucket_span\": \"300s\",\n"
+ "}"; + " \"summary_count_field_name\": \"doc_count\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"detectors\": [\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"bytes-delta\",\n"
+ " \"by_field_name\": \"hostname\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\": {\"time_field\": \"timestamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
String aggregations = String aggregations =
@ -471,8 +512,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(new Request("GET",
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":40")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":40"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":40")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":40"));
assertThat(jobStatsResponseAsString, containsString("\"out_of_order_timestamp_count\":0")); assertThat(jobStatsResponseAsString, containsString("\"out_of_order_timestamp_count\":0"));
@ -483,13 +525,22 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void testLookbackUsingDerivativeAggWithSmallerHistogramBucketThanDataRate() throws Exception { public void testLookbackUsingDerivativeAggWithSmallerHistogramBucketThanDataRate() throws Exception {
String jobId = "derivative-agg-network-job"; String jobId = "derivative-agg-network-job";
String job = "{\"analysis_config\" :{\"bucket_span\":\"300s\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"bytes-delta\",\"by_field_name\":\"hostname\"}]}," + " \"analysis_config\": {\n"
+ "\"data_description\" : {\"time_field\":\"timestamp\"}" + " \"bucket_span\": \"300s\",\n"
+ "}"; + " \"summary_count_field_name\": \"doc_count\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"detectors\": [\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"bytes-delta\",\n"
+ " \"by_field_name\": \"hostname\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\": {\"time_field\": \"timestamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
String aggregations = String aggregations =
@ -507,21 +558,31 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(new Request("GET",
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":240")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":240"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":240")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":240"));
} }
public void testLookbackWithoutPermissions() throws Exception { public void testLookbackWithoutPermissions() throws Exception {
String jobId = "permission-test-network-job"; String jobId = "permission-test-network-job";
String job = "{\"analysis_config\" :{\"bucket_span\":\"300s\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"bytes-delta\",\"by_field_name\":\"hostname\"}]}," + " \"analysis_config\": {\n"
+ "\"data_description\" : {\"time_field\":\"timestamp\"}" + " \"bucket_span\": \"300s\",\n"
+ "}"; + " \"summary_count_field_name\": \"doc_count\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"detectors\": [\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"bytes-delta\",\n"
+ " \"by_field_name\": \"hostname\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\": {\"time_field\": \"timestamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
String aggregations = String aggregations =
@ -545,29 +606,39 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS); startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(new Request("GET",
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
// We expect that no data made it through to the job // We expect that no data made it through to the job
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":0")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":0"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":0")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":0"));
// There should be a notification saying that there was a problem extracting data // There should be a notification saying that there was a problem extracting data
client().performRequest("post", "_refresh"); client().performRequest(new Request("POST", "/_refresh"));
Response notificationsResponse = client().performRequest("get", AuditorField.NOTIFICATIONS_INDEX + "/_search?q=job_id:" + jobId); Response notificationsResponse = client().performRequest(
String notificationsResponseAsString = responseEntityToString(notificationsResponse); new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?q=job_id:" + jobId));
String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " + assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
"action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\"")); "action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\""));
} }
public void testLookbackWithPipelineBucketAgg() throws Exception { public void testLookbackWithPipelineBucketAgg() throws Exception {
String jobId = "pipeline-bucket-agg-job"; String jobId = "pipeline-bucket-agg-job";
String job = "{\"analysis_config\" :{\"bucket_span\":\"1h\"," Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ "\"summary_count_field_name\":\"doc_count\"," createJobRequest.setJsonEntity("{\n"
+ "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"percentile95_airlines_count\"}]}," + " \"analysis_config\": {\n"
+ "\"data_description\" : {\"time_field\":\"time stamp\"}" + " \"bucket_span\": \"1h\",\n"
+ "}"; + " \"summary_count_field_name\": \"doc_count\",\n"
client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + " \"detectors\": [\n"
new StringEntity(job, ContentType.APPLICATION_JSON)); + " {\n"
+ " \"function\": \"mean\",\n"
+ " \"field_name\": \"percentile95_airlines_count\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"data_description\": {\"time_field\": \"time stamp\"}\n"
+ "}");
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId; String datafeedId = "datafeed-" + jobId;
String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":\"15m\"}," String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":\"15m\"},"
@ -582,8 +653,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); Response jobStatsResponse = client().performRequest(new Request("GET",
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
assertThat(jobStatsResponseAsString, containsString("\"input_field_count\":4")); assertThat(jobStatsResponseAsString, containsString("\"input_field_count\":4"));
assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2")); assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
@ -599,15 +671,15 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build(); new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build();
openJob(client(), jobId); openJob(client(), jobId);
Response response = client().performRequest("post", Request startRequest = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start?start=2016-06-01T00:00:00Z"); startRequest.addParameter("start", "2016-06-01T00:00:00Z");
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); Response response = client().performRequest(startRequest);
assertThat(responseEntityToString(response), equalTo("{\"started\":true}")); assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"started\":true}"));
assertBusy(() -> { assertBusy(() -> {
try { try {
Response getJobResponse = client().performRequest("get", Response getJobResponse = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String responseAsString = responseEntityToString(getJobResponse); String responseAsString = EntityUtils.toString(getJobResponse.getEntity());
assertThat(responseAsString, containsString("\"processed_record_count\":2")); assertThat(responseAsString, containsString("\"processed_record_count\":2"));
assertThat(responseAsString, containsString("\"state\":\"opened\"")); assertThat(responseAsString, containsString("\"state\":\"opened\""));
} catch (Exception e1) { } catch (Exception e1) {
@ -619,9 +691,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
// test a model snapshot is present // test a model snapshot is present
assertBusy(() -> { assertBusy(() -> {
try { try {
Response getJobResponse = client().performRequest("get", Response getJobResponse = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/model_snapshots"); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/model_snapshots"));
String responseAsString = responseEntityToString(getJobResponse); String responseAsString = EntityUtils.toString(getJobResponse.getEntity());
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
} catch (Exception e1) { } catch (Exception e1) {
throw new RuntimeException(e1); throw new RuntimeException(e1);
@ -629,25 +701,25 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
}); });
ResponseException e = expectThrows(ResponseException.class, ResponseException e = expectThrows(ResponseException.class,
() -> client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)); () -> client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)));
response = e.getResponse(); response = e.getResponse();
assertThat(response.getStatusLine().getStatusCode(), equalTo(409)); assertThat(response.getStatusLine().getStatusCode(), equalTo(409));
assertThat(responseEntityToString(response), containsString("Cannot delete job [" + jobId + "] because datafeed [" + datafeedId assertThat(EntityUtils.toString(response.getEntity()),
+ "] refers to it")); containsString("Cannot delete job [" + jobId + "] because datafeed [" + datafeedId + "] refers to it"));
response = client().performRequest("post", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stop"); response = client().performRequest(new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stop"));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
assertThat(responseEntityToString(response), equalTo("{\"stopped\":true}")); assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"stopped\":true}"));
client().performRequest("POST", "/_xpack/ml/anomaly_detectors/" + jobId + "/_close"); client().performRequest(new Request("POST", "/_xpack/ml/anomaly_detectors/" + jobId + "/_close"));
response = client().performRequest("delete", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId); response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"acknowledged\":true}"));
response = client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"acknowledged\":true}"));
} }
public void testForceDeleteWhileDatafeedIsRunning() throws Exception { public void testForceDeleteWhileDatafeedIsRunning() throws Exception {
@ -657,25 +729,26 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build(); new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build();
openJob(client(), jobId); openJob(client(), jobId);
Response response = client().performRequest("post", Request startRequest = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start?start=2016-06-01T00:00:00Z"); startRequest.addParameter("start", "2016-06-01T00:00:00Z");
Response response = client().performRequest(startRequest);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
assertThat(responseEntityToString(response), equalTo("{\"started\":true}")); assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"started\":true}"));
ResponseException e = expectThrows(ResponseException.class, ResponseException e = expectThrows(ResponseException.class,
() -> client().performRequest("delete", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId)); () -> client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId)));
response = e.getResponse(); response = e.getResponse();
assertThat(response.getStatusLine().getStatusCode(), equalTo(409)); assertThat(response.getStatusLine().getStatusCode(), equalTo(409));
assertThat(responseEntityToString(response), containsString("Cannot delete datafeed [" + datafeedId assertThat(EntityUtils.toString(response.getEntity()),
+ "] while its status is started")); containsString("Cannot delete datafeed [" + datafeedId + "] while its status is started"));
response = client().performRequest("delete", Request forceDeleteRequest = new Request("DELETE", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId);
MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "?force=true"); forceDeleteRequest.addParameter("force", "true");
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); response = client().performRequest(forceDeleteRequest);
assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"acknowledged\":true}"));
expectThrows(ResponseException.class, expectThrows(ResponseException.class,
() -> client().performRequest("get", "/_xpack/ml/datafeeds/" + datafeedId)); () -> client().performRequest(new Request("GET", "/_xpack/ml/datafeeds/" + datafeedId)));
} }
private class LookbackOnlyTestHelper { private class LookbackOnlyTestHelper {
@ -727,9 +800,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
startDatafeedAndWaitUntilStopped(datafeedId); startDatafeedAndWaitUntilStopped(datafeedId);
waitUntilJobIsClosed(jobId); waitUntilJobIsClosed(jobId);
Response jobStatsResponse = client().performRequest("get", Response jobStatsResponse = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
if (shouldSucceedInput) { if (shouldSucceedInput) {
assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2")); assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
} else { } else {
@ -748,16 +821,20 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
} }
private void startDatafeedAndWaitUntilStopped(String datafeedId, String authHeader) throws Exception { private void startDatafeedAndWaitUntilStopped(String datafeedId, String authHeader) throws Exception {
Response startDatafeedRequest = client().performRequest("post", Request request = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start?start=2016-06-01T00:00:00Z&end=2016-06-02T00:00:00Z", request.addParameter("start", "2016-06-01T00:00:00Z");
new BasicHeader("Authorization", authHeader)); request.addParameter("end", "2016-06-02T00:00:00Z");
assertThat(startDatafeedRequest.getStatusLine().getStatusCode(), equalTo(200)); RequestOptions.Builder options = request.getOptions().toBuilder();
assertThat(responseEntityToString(startDatafeedRequest), equalTo("{\"started\":true}")); options.addHeader("Authorization", authHeader);
request.setOptions(options);
Response startDatafeedResponse = client().performRequest(request);
assertThat(EntityUtils.toString(startDatafeedResponse.getEntity()), equalTo("{\"started\":true}"));
assertBusy(() -> { assertBusy(() -> {
try { try {
Response datafeedStatsResponse = client().performRequest("get", Response datafeedStatsResponse = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stats"); MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stats"));
assertThat(responseEntityToString(datafeedStatsResponse), containsString("\"state\":\"stopped\"")); assertThat(EntityUtils.toString(datafeedStatsResponse.getEntity()),
containsString("\"state\":\"stopped\""));
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -767,9 +844,9 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
private void waitUntilJobIsClosed(String jobId) throws Exception { private void waitUntilJobIsClosed(String jobId) throws Exception {
assertBusy(() -> { assertBusy(() -> {
try { try {
Response jobStatsResponse = client().performRequest("get", Response jobStatsResponse = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
assertThat(responseEntityToString(jobStatsResponse), containsString("\"state\":\"closed\"")); assertThat(EntityUtils.toString(jobStatsResponse.getEntity()), containsString("\"state\":\"closed\""));
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -777,27 +854,30 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
} }
private Response createJob(String id, String airlineVariant) throws Exception { private Response createJob(String id, String airlineVariant) throws Exception {
String job = "{\n" + " \"description\":\"Analysis of response time by airline\",\n" Request request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + id);
+ " \"analysis_config\" : {\n" + " \"bucket_span\":\"1h\",\n" request.setJsonEntity("{\n"
+ " \"detectors\" :[\n" + " \"description\": \"Analysis of response time by airline\",\n"
+ " {\"function\":\"mean\",\"field_name\":\"responsetime\",\"by_field_name\":\"" + airlineVariant + "\"}]\n" + " \"analysis_config\": {\n"
+ " },\n" + " \"data_description\" : {\n" + " \"bucket_span\": \"1h\",\n"
+ " \"format\":\"xcontent\",\n" + " \"detectors\" :[\n"
+ " \"time_field\":\"time stamp\",\n" + " \"time_format\":\"yyyy-MM-dd'T'HH:mm:ssX\"\n" + " }\n" + " {\n"
+ "}"; + " \"function\": \"mean\",\n"
return client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + id, + " \"field_name\": \"responsetime\",\n"
Collections.emptyMap(), new StringEntity(job, ContentType.APPLICATION_JSON)); + " \"by_field_name\": \"" + airlineVariant + "\"\n"
} + " }\n"
+ " ]\n"
private static String responseEntityToString(Response response) throws Exception { + " },\n"
try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + " \"data_description\": {\n"
return reader.lines().collect(Collectors.joining("\n")); + " \"format\": \"xcontent\",\n"
} + " \"time_field\": \"time stamp\",\n"
+ " \"time_format\": \"yyyy-MM-dd'T'HH:mm:ssX\"\n"
+ " }\n"
+ "}");
return client().performRequest(request);
} }
public static void openJob(RestClient client, String jobId) throws IOException { public static void openJob(RestClient client, String jobId) throws IOException {
Response response = client.performRequest("post", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open"); client.performRequest(new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open"));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
} }
@After @After
@ -850,17 +930,28 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
} }
Response build() throws IOException { Response build() throws IOException {
String datafeedConfig = "{" Request request = new Request("PUT", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId);
request.setJsonEntity("{"
+ "\"job_id\": \"" + jobId + "\",\"indexes\":[\"" + index + "\"],\"types\":[\"" + type + "\"]" + "\"job_id\": \"" + jobId + "\",\"indexes\":[\"" + index + "\"],\"types\":[\"" + type + "\"]"
+ (source ? ",\"_source\":true" : "") + (source ? ",\"_source\":true" : "")
+ (scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields) + (scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields)
+ (aggregations == null ? "" : ",\"aggs\":" + aggregations) + (aggregations == null ? "" : ",\"aggs\":" + aggregations)
+ (chunkingTimespan == null ? "" : + (chunkingTimespan == null ? "" :
",\"chunking_config\":{\"mode\":\"MANUAL\",\"time_span\":\"" + chunkingTimespan + "\"}") ",\"chunking_config\":{\"mode\":\"MANUAL\",\"time_span\":\"" + chunkingTimespan + "\"}")
+ "}"; + "}");
return client().performRequest("put", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId, Collections.emptyMap(), RequestOptions.Builder options = request.getOptions().toBuilder();
new StringEntity(datafeedConfig, ContentType.APPLICATION_JSON), options.addHeader("Authorization", authHeader);
new BasicHeader("Authorization", authHeader)); request.setOptions(options);
return client().performRequest(request);
} }
} }
private void bulkIndex(String bulk) throws IOException {
Request bulkRequest = new Request("POST", "/_bulk");
bulkRequest.setJsonEntity(bulk);
bulkRequest.addParameter("refresh", "true");
bulkRequest.addParameter("pretty", null);
String bulkResponse = EntityUtils.toString(client().performRequest(bulkRequest).getEntity());
assertThat(bulkResponse, not(containsString("\"errors\": false")));
}
} }

466
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
View File

@ -5,8 +5,7 @@
*/ */
package org.elasticsearch.xpack.ml.integration; package org.elasticsearch.xpack.ml.integration;
import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.client.Request; import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response; import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseException;
@ -23,15 +22,10 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFiel
import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
import org.junit.After; import org.junit.After;
import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
@ -55,15 +49,13 @@ public class MlJobIT extends ESRestTestCase {
public void testPutJob_GivenFarequoteConfig() throws Exception { public void testPutJob_GivenFarequoteConfig() throws Exception {
Response response = createFarequoteJob("given-farequote-config-job"); Response response = createFarequoteJob("given-farequote-config-job");
String responseAsString = EntityUtils.toString(response.getEntity());
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"job_id\":\"given-farequote-config-job\"")); assertThat(responseAsString, containsString("\"job_id\":\"given-farequote-config-job\""));
} }
public void testGetJob_GivenNoSuchJob() throws Exception { public void testGetJob_GivenNoSuchJob() throws Exception {
ResponseException e = expectThrows(ResponseException.class, ResponseException e = expectThrows(ResponseException.class, () ->
() -> client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/non-existing-job/_stats")); client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/non-existing-job/_stats")));
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
assertThat(e.getMessage(), containsString("No known job with id 'non-existing-job'")); assertThat(e.getMessage(), containsString("No known job with id 'non-existing-job'"));
@ -72,11 +64,9 @@ public class MlJobIT extends ESRestTestCase {
public void testGetJob_GivenJobExists() throws Exception { public void testGetJob_GivenJobExists() throws Exception {
createFarequoteJob("get-job_given-job-exists-job"); createFarequoteJob("get-job_given-job-exists-job");
Response response = client().performRequest("get", Response response = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/get-job_given-job-exists-job/_stats"); MachineLearning.BASE_PATH + "anomaly_detectors/get-job_given-job-exists-job/_stats"));
String responseAsString = EntityUtils.toString(response.getEntity());
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1")); assertThat(responseAsString, containsString("\"count\":1"));
assertThat(responseAsString, containsString("\"job_id\":\"get-job_given-job-exists-job\"")); assertThat(responseAsString, containsString("\"job_id\":\"get-job_given-job-exists-job\""));
} }
@ -86,20 +76,16 @@ public class MlJobIT extends ESRestTestCase {
createFarequoteJob(jobId); createFarequoteJob(jobId);
// Explicit _all // Explicit _all
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/_all"); String explictAll = EntityUtils.toString(
client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/_all")).getEntity());
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(explictAll, containsString("\"count\":1"));
String responseAsString = responseEntityToString(response); assertThat(explictAll, containsString("\"job_id\":\"" + jobId + "\""));
assertThat(responseAsString, containsString("\"count\":1"));
assertThat(responseAsString, containsString("\"job_id\":\"" + jobId + "\""));
// Implicit _all // Implicit _all
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors"); String implicitAll = EntityUtils.toString(
client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors")).getEntity());
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(implicitAll, containsString("\"count\":1"));
responseAsString = responseEntityToString(response); assertThat(implicitAll, containsString("\"job_id\":\"" + jobId + "\""));
assertThat(responseAsString, containsString("\"count\":1"));
assertThat(responseAsString, containsString("\"job_id\":\"" + jobId + "\""));
} }
public void testGetJobs_GivenMultipleJobs() throws Exception { public void testGetJobs_GivenMultipleJobs() throws Exception {
@ -108,36 +94,37 @@ public class MlJobIT extends ESRestTestCase {
createFarequoteJob("given-multiple-jobs-job-3"); createFarequoteJob("given-multiple-jobs-job-3");
// Explicit _all // Explicit _all
Response response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/_all"); String explicitAll = EntityUtils.toString(
client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/_all")).getEntity());
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(explicitAll, containsString("\"count\":3"));
String responseAsString = responseEntityToString(response); assertThat(explicitAll, containsString("\"job_id\":\"given-multiple-jobs-job-1\""));
assertThat(responseAsString, containsString("\"count\":3")); assertThat(explicitAll, containsString("\"job_id\":\"given-multiple-jobs-job-2\""));
assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-1\"")); assertThat(explicitAll, containsString("\"job_id\":\"given-multiple-jobs-job-3\""));
assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-2\""));
assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-3\""));
// Implicit _all // Implicit _all
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors"); String implicitAll = EntityUtils.toString(
client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors")).getEntity());
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); assertThat(implicitAll, containsString("\"count\":3"));
responseAsString = responseEntityToString(response); assertThat(implicitAll, containsString("\"job_id\":\"given-multiple-jobs-job-1\""));
assertThat(responseAsString, containsString("\"count\":3")); assertThat(implicitAll, containsString("\"job_id\":\"given-multiple-jobs-job-2\""));
assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-1\"")); assertThat(implicitAll, containsString("\"job_id\":\"given-multiple-jobs-job-3\""));
assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-2\""));
assertThat(responseAsString, containsString("\"job_id\":\"given-multiple-jobs-job-3\""));
} }
private Response createFarequoteJob(String jobId) throws IOException { private Response createFarequoteJob(String jobId) throws IOException {
String job = "{\n" + " \"description\":\"Analysis of response time by airline\",\n" Request request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+ " \"analysis_config\" : {\n" + " \"bucket_span\": \"3600s\",\n" request.setJsonEntity(
"{\n"
+ " \"description\":\"Analysis of response time by airline\",\n"
+ " \"analysis_config\" : {\n"
+ " \"bucket_span\": \"3600s\",\n"
+ " \"detectors\" :[{\"function\":\"metric\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]\n" + " \"detectors\" :[{\"function\":\"metric\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]\n"
+ " },\n" + " \"data_description\" : {\n" + " \"field_delimiter\":\",\",\n" + " " + + " },\n" + " \"data_description\" : {\n"
"\"time_field\":\"time\",\n" + " \"field_delimiter\":\",\",\n"
+ " \"time_format\":\"yyyy-MM-dd HH:mm:ssX\"\n" + " }\n" + "}"; + " \"time_field\":\"time\",\n"
+ " \"time_format\":\"yyyy-MM-dd HH:mm:ssX\"\n"
return client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, + " }\n"
Collections.emptyMap(), new StringEntity(job, ContentType.APPLICATION_JSON)); + "}");
return client().performRequest(request);
} }
public void testCantCreateJobWithSameID() throws Exception { public void testCantCreateJobWithSameID() throws Exception {
@ -148,18 +135,14 @@ public class MlJobIT extends ESRestTestCase {
" \"data_description\": {},\n" + " \"data_description\": {},\n" +
" \"results_index_name\" : \"%s\"}"; " \"results_index_name\" : \"%s\"}";
String jobConfig = String.format(Locale.ROOT, jobTemplate, "index-1");
String jobId = "cant-create-job-with-same-id-job"; String jobId = "cant-create-job-with-same-id-job";
Response response = client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId , Request createJob1 = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
Collections.emptyMap(), createJob1.setJsonEntity(String.format(Locale.ROOT, jobTemplate, "index-1"));
new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); client().performRequest(createJob1);
assertEquals(200, response.getStatusLine().getStatusCode());
final String jobConfig2 = String.format(Locale.ROOT, jobTemplate, "index-2"); Request createJob2 = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
ResponseException e = expectThrows(ResponseException.class, createJob2.setJsonEntity(String.format(Locale.ROOT, jobTemplate, "index-2"));
() ->client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId, ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(createJob2));
Collections.emptyMap(), new StringEntity(jobConfig2, ContentType.APPLICATION_JSON)));
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400));
assertThat(e.getMessage(), containsString("The job cannot be created with the Id '" + jobId + "'. The Id is already used.")); assertThat(e.getMessage(), containsString("The job cannot be created with the Id '" + jobId + "'. The Id is already used."));
@ -175,94 +158,78 @@ public class MlJobIT extends ESRestTestCase {
String jobId1 = "create-jobs-with-index-name-option-job-1"; String jobId1 = "create-jobs-with-index-name-option-job-1";
String indexName = "non-default-index"; String indexName = "non-default-index";
String jobConfig = String.format(Locale.ROOT, jobTemplate, indexName); Request createJob1 = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1);
createJob1.setJsonEntity(String.format(Locale.ROOT, jobTemplate, indexName));
Response response = client().performRequest("put", MachineLearning.BASE_PATH client().performRequest(createJob1);
+ "anomaly_detectors/" + jobId1, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON));
assertEquals(200, response.getStatusLine().getStatusCode());
String jobId2 = "create-jobs-with-index-name-option-job-2"; String jobId2 = "create-jobs-with-index-name-option-job-2";
response = client().performRequest("put", MachineLearning.BASE_PATH Request createJob2 = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2);
+ "anomaly_detectors/" + jobId2, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); createJob2.setEntity(createJob1.getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); client().performRequest(createJob2);
// With security enabled GET _aliases throws an index_not_found_exception // With security enabled GET _aliases throws an index_not_found_exception
// if no aliases have been created. In multi-node tests the alias may not // if no aliases have been created. In multi-node tests the alias may not
// appear immediately so wait here. // appear immediately so wait here.
assertBusy(() -> { assertBusy(() -> {
try { try {
Response aliasesResponse = client().performRequest("get", "_aliases"); String aliasesResponse = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
assertEquals(200, aliasesResponse.getStatusLine().getStatusCode()); assertThat(aliasesResponse,
String responseAsString = responseEntityToString(aliasesResponse);
assertThat(responseAsString,
containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{")); containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{"));
assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}")); + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}"));
assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}")); assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}"));
assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}")); + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}"));
assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}")); assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}"));
} catch (ResponseException e) { } catch (ResponseException e) {
throw new AssertionError(e); throw new AssertionError(e);
} }
}); });
Response indicesResponse = client().performRequest("get", "_cat/indices"); String responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, indicesResponse.getStatusLine().getStatusCode());
String responseAsString = responseEntityToString(indicesResponse);
assertThat(responseAsString, assertThat(responseAsString,
containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
String bucketResult = String.format(Locale.ROOT,
"{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
jobId1, "1234", 1);
String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1234", 300); String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1234", 300);
response = client().performRequest("put", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id, Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
Collections.emptyMap(), new StringEntity(bucketResult, ContentType.APPLICATION_JSON)); createResultRequest.setJsonEntity(String.format(Locale.ROOT,
assertEquals(201, response.getStatusLine().getStatusCode());
bucketResult = String.format(Locale.ROOT,
"{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}", "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
jobId1, "1236", 1); jobId1, "1234", 1));
client().performRequest(createResultRequest);
id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1236", 300); id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1236", 300);
response = client().performRequest("put", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id, createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/doc/" + id);
Collections.emptyMap(), new StringEntity(bucketResult, ContentType.APPLICATION_JSON)); createResultRequest.setJsonEntity(String.format(Locale.ROOT,
assertEquals(201, response.getStatusLine().getStatusCode()); "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}",
jobId1, "1236", 1));
client().performRequest(createResultRequest);
client().performRequest("post", "_refresh"); client().performRequest(new Request("POST", "/_refresh"));
response = client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1 + "/results/buckets"); responseAsString = EntityUtils.toString(client().performRequest(
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1 + "/results/buckets")).getEntity());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":2")); assertThat(responseAsString, containsString("\"count\":2"));
response = client().performRequest("get", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_search"); responseAsString = EntityUtils.toString(client().performRequest(
assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_search")).getEntity());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"total\":2")); assertThat(responseAsString, containsString("\"total\":2"));
response = client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
// check that indices still exist, but are empty and aliases are gone // check that indices still exist, but are empty and aliases are gone
response = client().performRequest("get", "_aliases"); responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
response = client().performRequest("get", "_cat/indices"); responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
client().performRequest("post", "_refresh"); client().performRequest(new Request("POST", "/_refresh"));
response = client().performRequest("get", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "/_count"); responseAsString = EntityUtils.toString(client().performRequest(
assertEquals(200, response.getStatusLine().getStatusCode()); new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "/_count")).getEntity());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":0")); assertThat(responseAsString, containsString("\"count\":0"));
} }
@ -278,32 +245,27 @@ public class MlJobIT extends ESRestTestCase {
String byFieldName1 = "responsetime"; String byFieldName1 = "responsetime";
String jobId2 = "create-job-in-shared-index-updates-mapping-job-2"; String jobId2 = "create-job-in-shared-index-updates-mapping-job-2";
String byFieldName2 = "cpu-usage"; String byFieldName2 = "cpu-usage";
String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1);
Response response = client().performRequest("put", MachineLearning.BASE_PATH Request createJob1Request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1);
+ "anomaly_detectors/" + jobId1, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); createJob1Request.setJsonEntity(String.format(Locale.ROOT, jobTemplate, byFieldName1));
assertEquals(200, response.getStatusLine().getStatusCode()); client().performRequest(createJob1Request);
// Check the index mapping contains the first by_field_name // Check the index mapping contains the first by_field_name
response = client().performRequest("get", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX Request getResultsMappingRequest = new Request("GET",
+ AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "/_mapping?pretty"); AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "/_mapping");
assertEquals(200, response.getStatusLine().getStatusCode()); getResultsMappingRequest.addParameter("pretty", null);
String responseAsString = responseEntityToString(response); String resultsMappingAfterJob1 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity());
assertThat(responseAsString, containsString(byFieldName1)); assertThat(resultsMappingAfterJob1, containsString(byFieldName1));
assertThat(responseAsString, not(containsString(byFieldName2))); assertThat(resultsMappingAfterJob1, not(containsString(byFieldName2)));
jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName2); Request createJob2Request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2);
response = client().performRequest("put", MachineLearning.BASE_PATH createJob2Request.setJsonEntity(String.format(Locale.ROOT, jobTemplate, byFieldName2));
+ "anomaly_detectors/" + jobId2, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); client().performRequest(createJob2Request);
assertEquals(200, response.getStatusLine().getStatusCode());
// Check the index mapping now contains both fields // Check the index mapping now contains both fields
response = client().performRequest("get", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX String resultsMappingAfterJob2 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity());
+ AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "/_mapping?pretty"); assertThat(resultsMappingAfterJob2, containsString(byFieldName1));
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(resultsMappingAfterJob2, containsString(byFieldName2));
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(byFieldName1));
assertThat(responseAsString, containsString(byFieldName2));
} }
public void testCreateJobInCustomSharedIndexUpdatesMapping() throws Exception { public void testCreateJobInCustomSharedIndexUpdatesMapping() throws Exception {
@ -318,32 +280,27 @@ public class MlJobIT extends ESRestTestCase {
String byFieldName1 = "responsetime"; String byFieldName1 = "responsetime";
String jobId2 = "create-job-in-custom-shared-index-updates-mapping-job-2"; String jobId2 = "create-job-in-custom-shared-index-updates-mapping-job-2";
String byFieldName2 = "cpu-usage"; String byFieldName2 = "cpu-usage";
String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1);
Response response = client().performRequest("put", MachineLearning.BASE_PATH Request createJob1Request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1);
+ "anomaly_detectors/" + jobId1, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); createJob1Request.setJsonEntity(String.format(Locale.ROOT, jobTemplate, byFieldName1));
assertEquals(200, response.getStatusLine().getStatusCode()); client().performRequest(createJob1Request);
// Check the index mapping contains the first by_field_name // Check the index mapping contains the first by_field_name
response = client().performRequest("get", Request getResultsMappingRequest = new Request("GET",
AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index" + "/_mapping?pretty"); AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index/_mapping");
assertEquals(200, response.getStatusLine().getStatusCode()); getResultsMappingRequest.addParameter("pretty", null);
String responseAsString = responseEntityToString(response); String resultsMappingAfterJob1 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity());
assertThat(responseAsString, containsString(byFieldName1)); assertThat(resultsMappingAfterJob1, containsString(byFieldName1));
assertThat(responseAsString, not(containsString(byFieldName2))); assertThat(resultsMappingAfterJob1, not(containsString(byFieldName2)));
jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName2); Request createJob2Request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2);
response = client().performRequest("put", MachineLearning.BASE_PATH createJob2Request.setJsonEntity(String.format(Locale.ROOT, jobTemplate, byFieldName2));
+ "anomaly_detectors/" + jobId2, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); client().performRequest(createJob2Request);
assertEquals(200, response.getStatusLine().getStatusCode());
// Check the index mapping now contains both fields // Check the index mapping now contains both fields
response = client().performRequest("get", String resultsMappingAfterJob2 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity());
AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index" + "/_mapping?pretty"); assertThat(resultsMappingAfterJob2, containsString(byFieldName1));
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(resultsMappingAfterJob2, containsString(byFieldName2));
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(byFieldName1));
assertThat(responseAsString, containsString(byFieldName2));
} }
public void testCreateJob_WithClashingFieldMappingsFails() throws Exception { public void testCreateJob_WithClashingFieldMappingsFails() throws Exception {
@ -366,17 +323,14 @@ public class MlJobIT extends ESRestTestCase {
byFieldName1 = "response.time"; byFieldName1 = "response.time";
byFieldName2 = "response"; byFieldName2 = "response";
} }
String jobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName1);
Response response = client().performRequest("put", MachineLearning.BASE_PATH Request createJob1Request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1);
+ "anomaly_detectors/" + jobId1, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON)); createJob1Request.setJsonEntity(String.format(Locale.ROOT, jobTemplate, byFieldName1));
assertEquals(200, response.getStatusLine().getStatusCode()); client().performRequest(createJob1Request);
final String failingJobConfig = String.format(Locale.ROOT, jobTemplate, byFieldName2);
ResponseException e = expectThrows(ResponseException.class,
() -> client().performRequest("put", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2,
Collections.emptyMap(), new StringEntity(failingJobConfig, ContentType.APPLICATION_JSON)));
Request createJob2Request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2);
createJob2Request.setJsonEntity(String.format(Locale.ROOT, jobTemplate, byFieldName2));
ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(createJob2Request));
assertThat(e.getMessage(), assertThat(e.getMessage(),
containsString("This job would cause a mapping clash with existing field [response] - " + containsString("This job would cause a mapping clash with existing field [response] - " +
"avoid the clash by assigning a dedicated results index")); "avoid the clash by assigning a dedicated results index"));
@ -387,35 +341,27 @@ public class MlJobIT extends ESRestTestCase {
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
Response response = client().performRequest("get", "_cat/indices"); String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(indicesBeforeDelete, containsString(indexName));
String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(indexName));
response = client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
// check that the index still exists (it's shared by default) // check that the index still exists (it's shared by default)
response = client().performRequest("get", "_cat/indices"); String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(indicesAfterDelete, containsString(indexName));
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(indexName));
assertBusy(() -> { assertBusy(() -> {
try { try {
Response r = client().performRequest("get", indexName + "/_count"); String count = EntityUtils.toString(client().performRequest(new Request("GET", indexName + "/_count")).getEntity());
assertEquals(200, r.getStatusLine().getStatusCode()); assertThat(count, containsString("\"count\":0"));
String responseString = responseEntityToString(r);
assertThat(responseString, containsString("\"count\":0"));
} catch (Exception e) { } catch (Exception e) {
fail(e.getMessage()); fail(e.getMessage());
} }
}); });
// check that the job itself is gone // check that the job itself is gone
expectThrows(ResponseException.class, () -> expectThrows(ResponseException.class, () ->
client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")); client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")));
} }
public void testDeleteJobAfterMissingIndex() throws Exception { public void testDeleteJobAfterMissingIndex() throws Exception {
@ -424,28 +370,22 @@ public class MlJobIT extends ESRestTestCase {
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
Response response = client().performRequest("get", "_cat/indices"); String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(indicesBeforeDelete, containsString(indexName));
String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(indexName));
// Manually delete the index so that we can test that deletion proceeds // Manually delete the index so that we can test that deletion proceeds
// normally anyway // normally anyway
response = client().performRequest("delete", indexName); client().performRequest(new Request("DELETE", indexName));
assertEquals(200, response.getStatusLine().getStatusCode());
response = client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
// check index was deleted // check index was deleted
response = client().performRequest("get", "_cat/indices"); String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(indicesAfterDelete, not(containsString(aliasName)));
responseAsString = responseEntityToString(response); assertThat(indicesAfterDelete, not(containsString(indexName)));
assertThat(responseAsString, not(containsString(aliasName)));
assertThat(responseAsString, not(containsString(indexName)));
expectThrows(ResponseException.class, () -> expectThrows(ResponseException.class, () ->
client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")); client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")));
} }
public void testDeleteJobAfterMissingAliases() throws Exception { public void testDeleteJobAfterMissingAliases() throws Exception {
@ -460,11 +400,9 @@ public class MlJobIT extends ESRestTestCase {
// appear immediately so wait here. // appear immediately so wait here.
assertBusy(() -> { assertBusy(() -> {
try { try {
Response aliasesResponse = client().performRequest(new Request("get", "_cat/aliases")); String aliases = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/aliases")).getEntity());
assertEquals(200, aliasesResponse.getStatusLine().getStatusCode()); assertThat(aliases, containsString(readAliasName));
String responseAsString = responseEntityToString(aliasesResponse); assertThat(aliases, containsString(writeAliasName));
assertThat(responseAsString, containsString(readAliasName));
assertThat(responseAsString, containsString(writeAliasName));
} catch (ResponseException e) { } catch (ResponseException e) {
throw new AssertionError(e); throw new AssertionError(e);
} }
@ -472,17 +410,14 @@ public class MlJobIT extends ESRestTestCase {
// Manually delete the aliases so that we can test that deletion proceeds // Manually delete the aliases so that we can test that deletion proceeds
// normally anyway // normally anyway
Response response = client().performRequest("delete", indexName + "/_alias/" + readAliasName); client().performRequest(new Request("DELETE", indexName + "/_alias/" + readAliasName));
assertEquals(200, response.getStatusLine().getStatusCode()); client().performRequest(new Request("DELETE", indexName + "/_alias/" + writeAliasName));
response = client().performRequest("delete", indexName + "/_alias/" + writeAliasName);
assertEquals(200, response.getStatusLine().getStatusCode());
// check aliases were deleted // check aliases were deleted
expectThrows(ResponseException.class, () -> client().performRequest("get", indexName + "/_alias/" + readAliasName)); expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", indexName + "/_alias/" + readAliasName)));
expectThrows(ResponseException.class, () -> client().performRequest("get", indexName + "/_alias/" + writeAliasName)); expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", indexName + "/_alias/" + writeAliasName)));
response = client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
} }
public void testMultiIndexDelete() throws Exception { public void testMultiIndexDelete() throws Exception {
@ -490,86 +425,63 @@ public class MlJobIT extends ESRestTestCase {
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
Response response = client().performRequest("put", indexName + "-001"); client().performRequest(new Request("PUT", indexName + "-001"));
assertEquals(200, response.getStatusLine().getStatusCode()); client().performRequest(new Request("PUT", indexName + "-002"));
response = client().performRequest("put", indexName + "-002"); String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(indicesBeforeDelete, containsString(indexName));
assertThat(indicesBeforeDelete, containsString(indexName + "-001"));
response = client().performRequest("get", "_cat/indices"); assertThat(indicesBeforeDelete, containsString(indexName + "-002"));
assertEquals(200, response.getStatusLine().getStatusCode());
String responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString(indexName));
assertThat(responseAsString, containsString(indexName + "-001"));
assertThat(responseAsString, containsString(indexName + "-002"));
// Add some documents to each index to make sure the DBQ clears them out // Add some documents to each index to make sure the DBQ clears them out
String recordResult = Request createDoc0 = new Request("PUT", indexName + "/doc/" + 123);
String.format(Locale.ROOT, createDoc0.setJsonEntity(String.format(Locale.ROOT,
"{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"bucket_span\":%d, \"result_type\":\"record\"}", "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"bucket_span\":%d, \"result_type\":\"record\"}",
jobId, 123, 1); jobId, 123, 1));
client().performRequest("put", indexName + "/doc/" + 123, client().performRequest(createDoc0);
Collections.singletonMap("refresh", "true"), new StringEntity(recordResult, ContentType.APPLICATION_JSON)); Request createDoc1 = new Request("PUT", indexName + "-001/doc/" + 123);
client().performRequest("put", indexName + "-001/doc/" + 123, createDoc1.setEntity(createDoc0.getEntity());
Collections.singletonMap("refresh", "true"), new StringEntity(recordResult, ContentType.APPLICATION_JSON)); client().performRequest(createDoc1);
client().performRequest("put", indexName + "-002/doc/" + 123, Request createDoc2 = new Request("PUT", indexName + "-002/doc/" + 123);
Collections.singletonMap("refresh", "true"), new StringEntity(recordResult, ContentType.APPLICATION_JSON)); createDoc2.setEntity(createDoc0.getEntity());
client().performRequest(createDoc2);
// Also index a few through the alias for the first job // Also index a few through the alias for the first job
client().performRequest("put", indexName + "/doc/" + 456, Request createDoc3 = new Request("PUT", indexName + "/doc/" + 456);
Collections.singletonMap("refresh", "true"), new StringEntity(recordResult, ContentType.APPLICATION_JSON)); createDoc3.setEntity(createDoc0.getEntity());
client().performRequest(createDoc3);
client().performRequest(new Request("POST", "/_refresh"));
client().performRequest("post", "_refresh");
// check for the documents // check for the documents
response = client().performRequest("get", indexName+ "/_count"); assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "/_count")).getEntity()),
assertEquals(200, response.getStatusLine().getStatusCode()); containsString("\"count\":2"));
responseAsString = responseEntityToString(response); assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "-001/_count")).getEntity()),
assertThat(responseAsString, containsString("\"count\":2")); containsString("\"count\":1"));
assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "-002/_count")).getEntity()),
response = client().performRequest("get", indexName + "-001/_count"); containsString("\"count\":1"));
assertEquals(200, response.getStatusLine().getStatusCode());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1"));
response = client().performRequest("get", indexName + "-002/_count");
assertEquals(200, response.getStatusLine().getStatusCode());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":1"));
// Delete // Delete
response = client().performRequest("delete", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
client().performRequest("post", "_refresh"); client().performRequest(new Request("POST", "/_refresh"));
// check that the indices still exist but are empty // check that the indices still exist but are empty
response = client().performRequest("get", "_cat/indices"); String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
assertEquals(200, response.getStatusLine().getStatusCode()); assertThat(indicesAfterDelete, containsString(indexName));
responseAsString = responseEntityToString(response); assertThat(indicesAfterDelete, containsString(indexName + "-001"));
assertThat(responseAsString, containsString(indexName)); assertThat(indicesAfterDelete, containsString(indexName + "-002"));
assertThat(responseAsString, containsString(indexName + "-001"));
assertThat(responseAsString, containsString(indexName + "-002"));
response = client().performRequest("get", indexName + "/_count"); assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "/_count")).getEntity()),
assertEquals(200, response.getStatusLine().getStatusCode()); containsString("\"count\":0"));
responseAsString = responseEntityToString(response); assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "-001/_count")).getEntity()),
assertThat(responseAsString, containsString("\"count\":0")); containsString("\"count\":0"));
assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "-002/_count")).getEntity()),
response = client().performRequest("get", indexName + "-001/_count"); containsString("\"count\":0"));
assertEquals(200, response.getStatusLine().getStatusCode());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":0"));
response = client().performRequest("get", indexName + "-002/_count");
assertEquals(200, response.getStatusLine().getStatusCode());
responseAsString = responseEntityToString(response);
assertThat(responseAsString, containsString("\"count\":0"));
expectThrows(ResponseException.class, () -> expectThrows(ResponseException.class, () ->
client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")); client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")));
} }
public void testDelete_multipleRequest() throws Exception { public void testDelete_multipleRequest() throws Exception {
@ -590,7 +502,7 @@ public class MlJobIT extends ESRestTestCase {
if (forceDelete) { if (forceDelete) {
url += "?force=true"; url += "?force=true";
} }
Response response = client().performRequest("delete", url); Response response = client().performRequest(new Request("DELETE", url));
responses.put(Thread.currentThread().getId(), response); responses.put(Thread.currentThread().getId(), response);
} catch (ResponseException re) { } catch (ResponseException re) {
responseExceptions.put(Thread.currentThread().getId(), re); responseExceptions.put(Thread.currentThread().getId(), re);
@ -640,11 +552,12 @@ public class MlJobIT extends ESRestTestCase {
} }
for (Response response : responses.values()) { for (Response response : responses.values()) {
assertEquals(responseEntityToString(response), 200, response.getStatusLine().getStatusCode()); assertEquals(EntityUtils.toString(response.getEntity()), 200, response.getStatusLine().getStatusCode());
} }
assertNotNull(recreationResponse.get()); assertNotNull(recreationResponse.get());
assertEquals(responseEntityToString(recreationResponse.get()), 200, recreationResponse.get().getStatusLine().getStatusCode()); assertEquals(EntityUtils.toString(recreationResponse.get().getEntity()),
200, recreationResponse.get().getStatusLine().getStatusCode());
if (recreationException.get() != null) { if (recreationException.get() != null) {
assertNull(recreationException.get().getMessage(), recreationException.get()); assertNull(recreationException.get().getMessage(), recreationException.get());
@ -656,7 +569,7 @@ public class MlJobIT extends ESRestTestCase {
// but in the case that it does not the job that is recreated may get deleted. // but in the case that it does not the job that is recreated may get deleted.
// It is not a error if the job does not exist but the following assertions // It is not a error if the job does not exist but the following assertions
// will fail in that case. // will fail in that case.
client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
// Check that the job aliases exist. These are the last thing to be deleted when a job is deleted, so // Check that the job aliases exist. These are the last thing to be deleted when a job is deleted, so
// if there's been a race between deletion and recreation these are what will be missing. // if there's been a race between deletion and recreation these are what will be missing.
@ -682,15 +595,8 @@ public class MlJobIT extends ESRestTestCase {
} }
private String getAliases() throws IOException { private String getAliases() throws IOException {
Response response = client().performRequest("get", "_aliases"); Response response = client().performRequest(new Request("GET", "/_aliases"));
assertEquals(200, response.getStatusLine().getStatusCode()); return EntityUtils.toString(response.getEntity());
return responseEntityToString(response);
}
private static String responseEntityToString(Response response) throws IOException {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
return reader.lines().collect(Collectors.joining("\n"));
}
} }
@After @After

103
x-pack/plugin/ml/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
View File

@ -5,9 +5,8 @@
*/ */
package org.elasticsearch.xpack.ml.transforms; package org.elasticsearch.xpack.ml.transforms;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response; import org.elasticsearch.client.Response;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
@ -18,7 +17,6 @@ import org.elasticsearch.xpack.ml.utils.DomainSplitFunction;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -185,9 +183,10 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0); .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
createIndex("painless", settings.build()); createIndex("painless", settings.build());
client().performRequest("PUT", "painless/test/1", Collections.emptyMap(), Request createDoc = new Request("PUT", "/painless/test/1");
new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)); createDoc.setJsonEntity("{\"test\": \"test\"}");
client().performRequest("POST", "painless/_refresh"); createDoc.addParameter("refresh", "true");
client().performRequest(createDoc);
Pattern pattern = Pattern.compile("domain_split\":\\[(.*?),(.*?)\\]"); Pattern pattern = Pattern.compile("domain_split\":\\[(.*?),(.*?)\\]");
@ -198,7 +197,9 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
String mapAsJson = Strings.toString(jsonBuilder().map(params)); String mapAsJson = Strings.toString(jsonBuilder().map(params));
logger.info("params={}", mapAsJson); logger.info("params={}", mapAsJson);
StringEntity body = new StringEntity("{\n" + Request searchRequest = new Request("GET", "/painless/test/_search");
searchRequest.setJsonEntity(
"{\n" +
" \"query\" : {\n" + " \"query\" : {\n" +
" \"match_all\": {}\n" + " \"match_all\": {}\n" +
" },\n" + " },\n" +
@ -212,10 +213,8 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
" }\n" + " }\n" +
" }\n" + " }\n" +
" }\n" + " }\n" +
"}", ContentType.APPLICATION_JSON); "}");
String responseBody = EntityUtils.toString(client().performRequest(searchRequest).getEntity());
Response response = client().performRequest("GET", "painless/test/_search", Collections.emptyMap(), body);
String responseBody = EntityUtils.toString(response.getEntity());
Matcher m = pattern.matcher(responseBody); Matcher m = pattern.matcher(responseBody);
String actualSubDomain = ""; String actualSubDomain = "";
@ -242,24 +241,23 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32966") @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32966")
public void testHRDSplit() throws Exception { public void testHRDSplit() throws Exception {
// Create job // Create job
String job = "{\n" + Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job");
" \"description\":\"Domain splitting\",\n" + createJobRequest.setJsonEntity(
" \"analysis_config\" : {\n" + "{\n" +
" \"bucket_span\":\"3600s\",\n" + " \"description\":\"Domain splitting\",\n" +
" \"detectors\" :[{\"function\":\"count\", \"by_field_name\" : \"domain_split\"}]\n" + " \"analysis_config\" : {\n" +
" },\n" + " \"bucket_span\":\"3600s\",\n" +
" \"data_description\" : {\n" + " \"detectors\" :[{\"function\":\"count\", \"by_field_name\" : \"domain_split\"}]\n" +
" \"field_delimiter\":\",\",\n" + " },\n" +
" \"time_field\":\"time\"\n" + " \"data_description\" : {\n" +
" \n" + " \"field_delimiter\":\",\",\n" +
" }\n" + " \"time_field\":\"time\"\n" +
" }"; " \n" +
" }\n" +
client().performRequest("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job", Collections.emptyMap(), "}");
new StringEntity(job, ContentType.APPLICATION_JSON)); client().performRequest(createJobRequest);
client().performRequest("POST", MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job/_open"); client().performRequest(new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job/_open"));
// Create index to hold data // Create index to hold data
Settings.Builder settings = Settings.builder() Settings.Builder settings = Settings.builder()
@ -284,44 +282,43 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
if (i == 64) { if (i == 64) {
// Anomaly has 100 docs, but we don't care about the value // Anomaly has 100 docs, but we don't care about the value
for (int j = 0; j < 100; j++) { for (int j = 0; j < 100; j++) {
client().performRequest("PUT", "painless/test/" + time.toDateTimeISO() + "_" + j, Request createDocRequest = new Request("PUT", "/painless/test/" + time.toDateTimeISO() + "_" + j);
Collections.emptyMap(), createDocRequest.setJsonEntity("{\"domain\": \"" + "bar.bar.com\", \"time\": \"" + time.toDateTimeISO() + "\"}");
new StringEntity("{\"domain\": \"" + "bar.bar.com\", \"time\": \"" + time.toDateTimeISO() client().performRequest(createDocRequest);
+ "\"}", ContentType.APPLICATION_JSON));
} }
} else { } else {
// Non-anomalous values will be what's seen when the anomaly is reported // Non-anomalous values will be what's seen when the anomaly is reported
client().performRequest("PUT", "painless/test/" + time.toDateTimeISO(), Request createDocRequest = new Request("PUT", "/painless/test/" + time.toDateTimeISO());
Collections.emptyMap(), createDocRequest.setJsonEntity("{\"domain\": \"" + test.hostName + "\", \"time\": \"" + time.toDateTimeISO() + "\"}");
new StringEntity("{\"domain\": \"" + test.hostName + "\", \"time\": \"" + time.toDateTimeISO() client().performRequest(createDocRequest);
+ "\"}", ContentType.APPLICATION_JSON));
} }
} }
client().performRequest("POST", "painless/_refresh"); client().performRequest(new Request("POST", "/painless/_refresh"));
// Create and start datafeed // Create and start datafeed
String body = "{\n" + Request createFeedRequest = new Request("PUT", MachineLearning.BASE_PATH + "datafeeds/hrd-split-datafeed");
" \"job_id\":\"hrd-split-job\",\n" + createFeedRequest.setJsonEntity(
" \"indexes\":[\"painless\"],\n" + "{\n" +
" \"types\":[\"test\"],\n" + " \"job_id\":\"hrd-split-job\",\n" +
" \"script_fields\": {\n" + " \"indexes\":[\"painless\"],\n" +
" \"domain_split\": {\n" + " \"types\":[\"test\"],\n" +
" \"script\": \"return domainSplit(doc['domain'].value, params);\"\n" + " \"script_fields\": {\n" +
" }\n" + " \"domain_split\": {\n" +
" }\n" + " \"script\": \"return domainSplit(doc['domain'].value, params);\"\n" +
" }"; " }\n" +
" }\n" +
"}");
client().performRequest("PUT", MachineLearning.BASE_PATH + "datafeeds/hrd-split-datafeed", Collections.emptyMap(), client().performRequest(createFeedRequest);
new StringEntity(body, ContentType.APPLICATION_JSON)); client().performRequest(new Request("POST", MachineLearning.BASE_PATH + "datafeeds/hrd-split-datafeed/_start"));
client().performRequest("POST", MachineLearning.BASE_PATH + "datafeeds/hrd-split-datafeed/_start");
boolean passed = awaitBusy(() -> { boolean passed = awaitBusy(() -> {
try { try {
client().performRequest("POST", "/_refresh"); client().performRequest(new Request("POST", "/_refresh"));
Response response = client().performRequest("GET", Response response = client().performRequest(new Request("GET",
MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job/results/records"); MachineLearning.BASE_PATH + "anomaly_detectors/hrd-split-job/results/records"));
String responseBody = EntityUtils.toString(response.getEntity()); String responseBody = EntityUtils.toString(response.getEntity());
if (responseBody.contains("\"count\":2")) { if (responseBody.contains("\"count\":2")) {