honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Transition to typeless (mapping) APIs (#39573) 

ML has historically used doc as the single mapping type but reindex in 7.x
will change the mapping to _doc. Switching to the typeless APIs handles 
case where the mapping type is either doc or _doc. This change removes
deprecated typed usages.
This commit is contained in:
David Kyle 2019-03-04 13:52:05 +00:00 committed by GitHub
parent 9ddaabba88
commit a58145f9e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 225 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
public final class MlMetaIndex {
/**
@ -21,14 +22,12 @@ public final class MlMetaIndex {
*/
public static final String INDEX_NAME = ".ml-meta";
public static final String TYPE = "doc";
private MlMetaIndex() {}
public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)

5
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View File

@ -27,6 +27,7 @@ import java.io.IOException;
import java.util.SortedMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -71,7 +72,7 @@ public class AnnotationIndex {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, annotationsMapping);
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
@ -111,7 +112,7 @@ public class AnnotationIndex {
public static XContentBuilder annotationsMapping() throws IOException {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(ElasticsearchMappings.DOC_TYPE);
.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Annotation.ANNOTATION.getPreferredName())

61
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
View File

@ -18,7 +18,7 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
@ -62,6 +62,7 @@ import java.util.List;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -86,8 +87,6 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
*/
public class ElasticsearchMappings {
public static final String DOC_TYPE = "doc";
/**
* String constants used in mappings
*/
@ -137,7 +136,7 @@ public class ElasticsearchMappings {
public static XContentBuilder configMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
@ -420,14 +419,14 @@ public class ElasticsearchMappings {
.endObject();
}
public static XContentBuilder resultsMapping() throws IOException {
return resultsMapping(Collections.emptyList());
public static XContentBuilder resultsMapping(String mappingType) throws IOException {
return resultsMapping(mappingType, Collections.emptyList());
}
public static XContentBuilder resultsMapping(Collection<String> extraTermFields) throws IOException {
public static XContentBuilder resultsMapping(String mappingType, Collection<String> extraTermFields) throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(mappingType);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
@ -456,10 +455,11 @@ public class ElasticsearchMappings {
// end properties
builder.endObject();
// end type
builder.endObject();
// end mapping
builder.endObject();
// end doc
builder.endObject();
return builder;
}
@ -575,18 +575,25 @@ public class ElasticsearchMappings {
addModelSizeStatsFieldsToMapping(builder);
}
public static XContentBuilder termFieldsMapping(String type, Collection<String> termFields) {
/**
* Generate a keyword mapping for {@code termFields} for the default type
* {@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME}
*
* If the returned mapping is used in index creation and the new index has a matching template
* then the mapping type ({@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME})
* must match the mapping type of the template otherwise the mappings will not be merged correctly.
*
* @param termFields Fields to generate mapping for
* @return The mapping
*/
public static XContentBuilder termFieldsMapping(Collection<String> termFields) {
try {
XContentBuilder builder = jsonBuilder().startObject();
if (type != null) {
builder.startObject(type);
}
builder.startObject(SINGLE_MAPPING_NAME);
builder.startObject(PROPERTIES);
addTermFields(builder, termFields);
builder.endObject();
if (type != null) {
builder.endObject();
}
return builder.endObject();
} catch (IOException e) {
throw new RuntimeException(e);
@ -872,7 +879,7 @@ public class ElasticsearchMappings {
public static XContentBuilder stateMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.field(ENABLED, false);
builder.endObject();
@ -960,8 +967,8 @@ public class ElasticsearchMappings {
}
public static XContentBuilder auditMessageMapping() throws IOException {
XContentBuilder builder = jsonBuilder().startObject()
.startObject(AuditMessage.TYPE.getPreferredName());
XContentBuilder builder = jsonBuilder().startObject();
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
@ -987,6 +994,7 @@ public class ElasticsearchMappings {
.endObject()
.endObject()
.endObject();
return builder;
}
@ -994,12 +1002,12 @@ public class ElasticsearchMappings {
List<String> indicesToUpdate = new ArrayList<>();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> currentMapping = state.metaData().findMappings(concreteIndices,
new String[] {DOC_TYPE}, MapperPlugin.NOOP_FIELD_FILTER);
new String[0], MapperPlugin.NOOP_FIELD_FILTER);
for (String index : concreteIndices) {
ImmutableOpenMap<String, MappingMetaData> innerMap = currentMapping.get(index);
if (innerMap != null) {
MappingMetaData metaData = innerMap.get(DOC_TYPE);
MappingMetaData metaData = innerMap.valuesIt().next();
try {
@SuppressWarnings("unchecked")
Map<String, Object> meta = (Map<String, Object>) metaData.sourceAsMap().get("_meta");
@ -1038,7 +1046,8 @@ public class ElasticsearchMappings {
return indicesToUpdate.toArray(new String[indicesToUpdate.size()]);
}
public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContentBuilder, IOException> mappingSupplier,
public static void addDocMappingIfMissing(String alias,
CheckedBiFunction<String, Collection<String>, XContentBuilder, IOException> mappingSupplier,
Client client, ClusterState state, ActionListener<Boolean> listener) {
AliasOrIndex aliasOrIndex = state.metaData().getAliasAndIndexLookup().get(alias);
if (aliasOrIndex == null) {
@ -1058,9 +1067,13 @@ public class ElasticsearchMappings {
}
if (indicesThatRequireAnUpdate.length > 0) {
try (XContentBuilder mapping = mappingSupplier.get()) {
// Use the mapping type of the first index in the update
IndexMetaData indexMetaData = state.metaData().index(indicesThatRequireAnUpdate[0]);
String mappingType = indexMetaData.mapping().type();
try (XContentBuilder mapping = mappingSupplier.apply(mappingType, Collections.emptyList())) {
PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate);
putMappingRequest.type(DOC_TYPE);
putMappingRequest.type(mappingType);
putMappingRequest.source(mapping);
executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest,
ActionListener.wrap(response -> {

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
View File

@ -22,7 +22,7 @@ import java.util.Date;
import java.util.Objects;
public class AuditMessage implements ToXContentObject, Writeable {
public static final ParseField TYPE = new ParseField("audit_message");
private static final ParseField TYPE = new ParseField("audit_message");
public static final ParseField MESSAGE = new ParseField("message");
public static final ParseField LEVEL = new ParseField("level");

11
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
View File

@ -45,6 +45,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
public class ElasticsearchMappingsTests extends ESTestCase {
@ -117,11 +119,12 @@ public class ElasticsearchMappingsTests extends ESTestCase {
@SuppressWarnings("unchecked")
public void testTermFieldMapping() throws IOException {
XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(null, Arrays.asList("apple", "strawberry",
XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(Arrays.asList("apple", "strawberry",
AnomalyRecord.BUCKET_SPAN.getPreferredName()));
XContentParser parser = createParser(builder);
Map<String, Object> properties = (Map<String, Object>) parser.map().get(ElasticsearchMappings.PROPERTIES);
Map<String, Object> mapping = (Map<String, Object>) parser.map().get(SINGLE_MAPPING_NAME);
Map<String, Object> properties = (Map<String, Object>) mapping.get(ElasticsearchMappings.PROPERTIES);
Map<String, Object> instanceMapping = (Map<String, Object>) properties.get("apple");
assertNotNull(instanceMapping);
@ -217,7 +220,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
}
mapping.put("_meta", meta);
indexMetaData.putMapping(new MappingMetaData(ElasticsearchMappings.DOC_TYPE, mapping));
indexMetaData.putMapping(new MappingMetaData("_doc", mapping));
metaDataBuilder.put(indexMetaData);
}
@ -230,7 +233,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
private Set<String> collectResultsDocFieldNames() throws IOException {
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
return collectFieldNames(ElasticsearchMappings.resultsMapping());
return collectFieldNames(ElasticsearchMappings.resultsMapping("_doc"));
}
private Set<String> collectConfigDocFieldNames() throws IOException {

14
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
View File

@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ -35,38 +36,37 @@ import static org.hamcrest.Matchers.is;
public class CategorizationIT extends MlNativeAutodetectIntegTestCase {
private static final String DATA_INDEX = "log-data";
private static final String DATA_TYPE = "log";
private long nowMillis;
@Before
public void setUpData() {
client().admin().indices().prepareCreate(DATA_INDEX)
.addMapping(DATA_TYPE, "time", "type=date,format=epoch_millis",
.addMapping(SINGLE_MAPPING_NAME, "time", "type=date,format=epoch_millis",
"msg", "type=text")
.get();
nowMillis = System.currentTimeMillis();
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
IndexRequest indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
IndexRequest indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis(),
"msg", "Node 1 started");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis() + 1,
"msg", "Failed to shutdown [error org.aaaa.bbbb.Cccc line 54 caused " +
"by foo exception]");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis(),
"msg", "Node 2 started");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis() + 1,
"msg", "Failed to shutdown [error but this time completely different]");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis, "msg", "Node 3 started");
bulkRequestBuilder.add(indexRequest);

36
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
View File

@ -308,7 +308,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
client().performRequest(createJobRequest);
String datafeedId = jobId + "-datafeed";
new DatafeedBuilder(datafeedId, jobId, "nested-data", "response").build();
new DatafeedBuilder(datafeedId, jobId, "nested-data").build();
openJob(client(), jobId);
startDatafeedAndWaitUntilStopped(datafeedId);
@ -351,7 +351,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
// create a datafeed they DON'T have permission to search the index the datafeed is
// configured to read
ResponseException e = expectThrows(ResponseException.class, () ->
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response")
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs")
.setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN)
.build());
@ -419,7 +419,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
ResponseException e = expectThrows(ResponseException.class, () ->
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup", "doc")
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup")
.setAggregations(aggregations)
.setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS) //want to search, but no admin access
.build());
@ -449,7 +449,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
client().performRequest(createJobRequest);
String datafeedId = "datafeed-" + jobId;
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").build();
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").build();
// This should be disallowed, because ml_admin is trying to preview a datafeed created by
// by another user (x_pack_rest_user in this case) that will reveal the content of an index they
@ -490,7 +490,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+ "\"airline\":{\"terms\":{\"field\":\"airline\",\"size\":10},"
+ " \"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}";
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").setAggregations(aggregations).build();
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").setAggregations(aggregations).build();
openJob(client(), jobId);
startDatafeedAndWaitUntilStopped(datafeedId);
@ -529,7 +529,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+ "\"airline\":{\"terms\":{\"field\":\"airline\",\"size\":10},"
+ " \"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}";
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").setAggregations(aggregations).build();
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").setAggregations(aggregations).build();
openJob(client(), jobId);
startDatafeedAndWaitUntilStopped(datafeedId);
@ -568,7 +568,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"aggs\": {\"timestamp\":{\"max\":{\"field\":\"timestamp\"}},"
+ "\"bytes-delta\":{\"derivative\":{\"buckets_path\":\"avg_bytes_out\"}},"
+ "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";
new DatafeedBuilder(datafeedId, jobId, "network-data", "doc")
new DatafeedBuilder(datafeedId, jobId, "network-data")
.setAggregations(aggregations)
.setChunkingTimespan("300s")
.build();
@ -614,7 +614,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"aggs\": {\"timestamp\":{\"max\":{\"field\":\"timestamp\"}},"
+ "\"bytes-delta\":{\"derivative\":{\"buckets_path\":\"avg_bytes_out\"}},"
+ "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";
new DatafeedBuilder(datafeedId, jobId, "network-data", "doc")
new DatafeedBuilder(datafeedId, jobId, "network-data")
.setAggregations(aggregations)
.setChunkingTimespan("300s")
.build();
@ -658,7 +658,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";
// At the time we create the datafeed the user can access the network-data index that we have access to
new DatafeedBuilder(datafeedId, jobId, "network-data", "doc")
new DatafeedBuilder(datafeedId, jobId, "network-data")
.setAggregations(aggregations)
.setChunkingTimespan("300s")
.setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS)
@ -712,7 +712,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"airlines\":{\"terms\":{\"field\":\"airline.keyword\",\"size\":10}},"
+ "\"percentile95_airlines_count\":{\"percentiles_bucket\":" +
"{\"buckets_path\":\"airlines._count\", \"percents\": [95]}}}}}";
new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").setAggregations(aggregations).build();
new DatafeedBuilder(datafeedId, jobId, "airline-data").setAggregations(aggregations).build();
openJob(client(), jobId);
@ -801,7 +801,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"aggregations\":{"
+ "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+ "\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}";
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup", "response").setAggregations(aggregations).build();
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup").setAggregations(aggregations).build();
openJob(client(), jobId);
startDatafeedAndWaitUntilStopped(datafeedId);
@ -872,7 +872,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
// At the time we create the datafeed the user can access the network-data index that we have access to
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup", "doc")
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup")
.setAggregations(aggregations)
.setChunkingTimespan("300s")
.setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS)
@ -919,7 +919,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
+ "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+ "\"airlineFilter\":{\"filter\":{\"term\": {\"airline\":\"AAA\"}},"
+ " \"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}";
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").setAggregations(aggregations).build();
new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").setAggregations(aggregations).build();
openJob(client(), jobId);
startDatafeedAndWaitUntilStopped(datafeedId);
@ -936,7 +936,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
String jobId = "job-realtime-1";
createJob(jobId, "airline");
String datafeedId = jobId + "-datafeed";
new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build();
new DatafeedBuilder(datafeedId, jobId, "airline-data").build();
openJob(client(), jobId);
Request startRequest = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
@ -994,7 +994,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
String jobId = "job-realtime-2";
createJob(jobId, "airline");
String datafeedId = jobId + "-datafeed";
new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build();
new DatafeedBuilder(datafeedId, jobId, "airline-data").build();
openJob(client(), jobId);
Request startRequest = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
@ -1059,7 +1059,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
public void execute() throws Exception {
createJob(jobId, airlineVariant);
String datafeedId = "datafeed-" + jobId;
new DatafeedBuilder(datafeedId, jobId, dataIndex, "response")
new DatafeedBuilder(datafeedId, jobId, dataIndex)
.setScriptedFields(addScriptedFields ?
"{\"airline\":{\"script\":{\"lang\":\"painless\",\"inline\":\"doc['airline'].value\"}}}" : null)
.build();
@ -1159,18 +1159,16 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
String datafeedId;
String jobId;
String index;
String type;
boolean source;
String scriptedFields;
String aggregations;
String authHeader = BASIC_AUTH_VALUE_SUPER_USER;
String chunkingTimespan;
DatafeedBuilder(String datafeedId, String jobId, String index, String type) {
DatafeedBuilder(String datafeedId, String jobId, String index) {
this.datafeedId = datafeedId;
this.jobId = jobId;
this.index = index;
this.type = type;
}
DatafeedBuilder setSource(boolean enableSource) {

10
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
View File

@ -40,6 +40,7 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@ -49,12 +50,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
private static final String DATA_INDEX = "delete-expired-data-test-data";
private static final String DATA_TYPE = "doc";
@Before
public void setUpData() throws IOException {
client().admin().indices().prepareCreate(DATA_INDEX)
.addMapping(DATA_TYPE, "time", "type=date,format=epoch_millis")
.addMapping(SINGLE_MAPPING_NAME, "time", "type=date,format=epoch_millis")
.get();
// We are going to create data for last 2 days
@ -68,7 +68,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
long timestamp = nowMillis - TimeValue.timeValueHours(totalBuckets - bucket).getMillis();
int bucketRate = bucket == anomalousBucket ? anomalousRate : normalRate;
for (int point = 0; point < bucketRate; point++) {
IndexRequest indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
IndexRequest indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", timestamp);
bulkRequestBuilder.add(indexRequest);
}
@ -98,7 +98,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for (int i = 0; i < 10010; i++) {
String docId = "non_existing_job_" + randomFrom("model_state_1234567#" + i, "quantiles", "categorizer_state#" + i);
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobStateIndexWriteAlias(), "doc", docId);
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).id(docId);
indexRequest.source(Collections.emptyMap());
bulkRequestBuilder.add(indexRequest);
}
@ -145,7 +145,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
// Update snapshot timestamp to force it out of snapshot retention window
String snapshotUpdate = "{ \"timestamp\": " + oneDayAgo + "}";
UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), "doc", snapshotDocId);
UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), snapshotDocId);
updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON);
client().execute(UpdateAction.INSTANCE, updateSnapshotRequest).get();

6
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java
View File

@ -28,6 +28,7 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
@ -36,12 +37,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
public class ModelPlotsIT extends MlNativeAutodetectIntegTestCase {
private static final String DATA_INDEX = "model-plots-test-data";
private static final String DATA_TYPE = "doc";
@Before
public void setUpData() {
client().admin().indices().prepareCreate(DATA_INDEX)
.addMapping(DATA_TYPE, "time", "type=date,format=epoch_millis", "user", "type=keyword")
.addMapping(SINGLE_MAPPING_NAME, "time", "type=date,format=epoch_millis", "user", "type=keyword")
.get();
List<String> users = Arrays.asList("user_1", "user_2", "user_3");
@ -53,7 +53,7 @@ public class ModelPlotsIT extends MlNativeAutodetectIntegTestCase {
for (int bucket = 0; bucket < totalBuckets; bucket++) {
long timestamp = nowMillis - TimeValue.timeValueHours(totalBuckets - bucket).getMillis();
for (String user : users) {
IndexRequest indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
IndexRequest indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", timestamp, "user", user);
bulkRequestBuilder.add(indexRequest);
}

15
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -111,7 +111,6 @@ import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import org.elasticsearch.xpack.ml.action.TransportCloseJobAction;
@ -250,6 +249,7 @@ import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import static java.util.Collections.emptyList;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlugin, PersistentTaskPlugin {
public static final String NAME = "ml";
@ -650,7 +650,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
try (XContentBuilder auditMapping = ElasticsearchMappings.auditMessageMapping()) {
IndexTemplateMetaData notificationMessageTemplate = IndexTemplateMetaData.builder(AuditorField.NOTIFICATIONS_INDEX)
.putMapping(AuditMessage.TYPE.getPreferredName(), Strings.toString(auditMapping))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(auditMapping))
.patterns(Collections.singletonList(AuditorField.NOTIFICATIONS_INDEX))
.version(Version.CURRENT.id)
.settings(Settings.builder()
@ -675,7 +675,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting))
.version(Version.CURRENT.id)
.putMapping(MlMetaIndex.TYPE, Strings.toString(docMapping))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(docMapping))
.build();
templates.put(MlMetaIndex.INDEX_NAME, metaTemplate);
} catch (IOException e) {
@ -694,7 +694,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW))
.version(Version.CURRENT.id)
.putMapping(ElasticsearchMappings.DOC_TYPE, Strings.toString(configMapping))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(configMapping))
.build();
templates.put(AnomalyDetectorsIndex.configIndexName(), configTemplate);
} catch (IOException e) {
@ -708,15 +708,16 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting))
.putMapping(ElasticsearchMappings.DOC_TYPE, Strings.toString(stateMapping))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(stateMapping))
.version(Version.CURRENT.id)
.build();
templates.put(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, stateTemplate);
} catch (IOException e) {
logger.error("Error loading the template for the " + AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + " index", e);
}
try (XContentBuilder docMapping = ElasticsearchMappings.resultsMapping()) {
try (XContentBuilder docMapping = ElasticsearchMappings.resultsMapping(SINGLE_MAPPING_NAME)) {
IndexTemplateMetaData jobResultsTemplate = IndexTemplateMetaData.builder(AnomalyDetectorsIndex.jobResultsIndexPrefix())
.patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"))
.settings(Settings.builder()
@ -728,7 +729,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async")
// set the default all search field
.put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), ElasticsearchMappings.ALL_FIELD_VALUES))
.putMapping(ElasticsearchMappings.DOC_TYPE, Strings.toString(docMapping))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(docMapping))
.version(Version.CURRENT.id)
.build();
templates.put(AnomalyDetectorsIndex.jobResultsIndexPrefix(), jobResultsTemplate);

16
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java
View File

@ -17,7 +17,6 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.Client;
@ -66,6 +65,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -414,7 +414,7 @@ public class MlConfigMigrator {
}
private IndexRequest indexRequest(ToXContentObject source, String documentId, ToXContent.Params params) {
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings.DOC_TYPE, documentId);
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName()).id(documentId);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
indexRequest.source(source.toXContent(builder, params));
@ -439,9 +439,9 @@ public class MlConfigMigrator {
logger.debug("taking a snapshot of ml_metadata");
String documentId = "ml-config";
IndexRequestBuilder indexRequest = client.prepareIndex(AnomalyDetectorsIndex.jobStateIndexWriteAlias(),
ElasticsearchMappings.DOC_TYPE, documentId)
.setOpType(DocWriteRequest.OpType.CREATE);
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobStateIndexWriteAlias())
.id(documentId)
.opType(DocWriteRequest.OpType.CREATE);
ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
@ -449,7 +449,7 @@ public class MlConfigMigrator {
mlMetadata.toXContent(builder, params);
builder.endObject();
indexRequest.setSource(builder);
indexRequest.source(builder);
} catch (IOException e) {
logger.error("failed to serialise ml_metadata", e);
listener.onFailure(e);
@ -458,7 +458,7 @@ public class MlConfigMigrator {
AnomalyDetectorsIndex.createStateIndexAndAliasIfNecessary(client, clusterService.state(), ActionListener.wrap(
r -> {
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest.request(),
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest,
ActionListener.<IndexResponse>wrap(
indexResponse -> {
listener.onResponse(indexResponse.getResult() == DocWriteResponse.Result.CREATED);
@ -489,7 +489,7 @@ public class MlConfigMigrator {
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
);
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, ElasticsearchMappings.configMapping());
createIndexRequest.mapping(SINGLE_MAPPING_NAME, ElasticsearchMappings.configMapping());
} catch (Exception e) {
logger.error("error writing the .ml-config mappings", e);
listener.onFailure(e);

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
View File

@ -55,7 +55,7 @@ public class TransportDeleteCalendarEventAction extends HandledTransportAction<D
ActionListener<Calendar> calendarListener = ActionListener.wrap(
calendar -> {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, eventId);
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, eventId);
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, ActionListener.wrap(
getResponse -> {
if (getResponse.isExists() == false) {
@ -89,7 +89,7 @@ public class TransportDeleteCalendarEventAction extends HandledTransportAction<D
}
private void deleteEvent(String eventId, Calendar calendar, ActionListener<AcknowledgedResponse> listener) {
DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, eventId);
DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, eventId);
deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, deleteRequest,

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java
View File

@ -84,8 +84,7 @@ public class TransportDeleteFilterAction extends HandledTransportAction<DeleteFi
}
private void deleteFilter(String filterId, ActionListener<AcknowledgedResponse> listener) {
DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE,
MlFilter.documentId(filterId));
DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlFilter.documentId(filterId));
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
bulkRequestBuilder.add(deleteRequest);
bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
View File

@ -24,7 +24,6 @@ import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.utils.VoidChainTaskExecutor;
@ -71,8 +70,7 @@ public class TransportFinalizeJobExecutionAction extends TransportMasterNodeActi
Map<String, Object> update = Collections.singletonMap(Job.FINISHED_TIME.getPreferredName(), new Date());
for (String jobId: request.getJobIds()) {
UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
updateRequest.retryOnConflict(3);
updateRequest.doc(update);
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java
View File

@ -68,7 +68,7 @@ public class TransportGetFiltersAction extends HandledTransportAction<GetFilters
}
private void getFilter(String filterId, ActionListener<GetFiltersAction.Response> listener) {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, MlFilter.documentId(filterId));
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlFilter.documentId(filterId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
public void onResponse(GetResponse getDocResponse) {

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
View File

@ -62,7 +62,7 @@ public class TransportPostCalendarEventsAction extends HandledTransportAction<Po
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (ScheduledEvent event: events) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE);
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
indexRequest.source(event.toXContent(builder,
new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE,

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java
View File

@ -49,7 +49,7 @@ public class TransportPutCalendarAction extends HandledTransportAction<PutCalend
protected void doExecute(Task task, PutCalendarAction.Request request, ActionListener<PutCalendarAction.Response> listener) {
Calendar calendar = request.getCalendar();
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId());
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME).id(calendar.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
indexRequest.source(calendar.toXContent(builder,
new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true"))));

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java
View File

@ -48,7 +48,7 @@ public class TransportPutFilterAction extends HandledTransportAction<PutFilterAc
@Override
protected void doExecute(Task task, PutFilterAction.Request request, ActionListener<PutFilterAction.Response> listener) {
MlFilter filter = request.getFilter();
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId());
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME).id(filter.documentId());
indexRequest.opType(DocWriteRequest.OpType.CREATE);
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java
View File

@ -103,7 +103,7 @@ public class TransportUpdateFilterAction extends HandledTransportAction<UpdateFi
private void indexUpdatedFilter(MlFilter filter, final long seqNo, final long primaryTerm,
UpdateFilterAction.Request request,
ActionListener<PutFilterAction.Response> listener) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId());
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME).id(filter.documentId());
indexRequest.setIfSeqNo(seqNo);
indexRequest.setIfPrimaryTerm(primaryTerm);
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
@ -139,7 +139,7 @@ public class TransportUpdateFilterAction extends HandledTransportAction<UpdateFi
}
private void getFilterWithVersion(String filterId, ActionListener<FilterWithSeqNo> listener) {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, MlFilter.documentId(filterId));
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlFilter.documentId(filterId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
public void onResponse(GetResponse getDocResponse) {

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
View File

@ -23,7 +23,6 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.core.ml.job.results.Result;
import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
@ -80,8 +79,7 @@ public class TransportUpdateModelSnapshotAction extends HandledTransportAction<U
}
private void indexModelSnapshot(Result<ModelSnapshot> modelSnapshot, Consumer<Boolean> handler, Consumer<Exception> errorHandler) {
IndexRequest indexRequest = new IndexRequest(modelSnapshot.index, ElasticsearchMappings.DOC_TYPE,
ModelSnapshot.documentId(modelSnapshot.result));
IndexRequest indexRequest = new IndexRequest(modelSnapshot.index).id(ModelSnapshot.documentId(modelSnapshot.result));
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS);
indexRequest.source(builder);

30
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
View File

@ -19,7 +19,6 @@ import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
@ -50,7 +49,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
import org.elasticsearch.xpack.ml.job.persistence.ExpandedIdsMatcher;
@ -126,12 +124,11 @@ public class DatafeedConfigProvider {
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
XContentBuilder source = config.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS));
IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId))
.setSource(source)
.setOpType(DocWriteRequest.OpType.CREATE)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.request();
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName())
.id(DatafeedConfig.documentId(datafeedId))
.source(source)
.opType(DocWriteRequest.OpType.CREATE)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap(
listener::onResponse,
@ -162,8 +159,7 @@ public class DatafeedConfigProvider {
* @param datafeedConfigListener The config listener
*/
public void getDatafeedConfig(String datafeedId, ActionListener<DatafeedConfig.Builder> datafeedConfigListener) {
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId));
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), DatafeedConfig.documentId(datafeedId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
public void onResponse(GetResponse getResponse) {
@ -230,8 +226,7 @@ public class DatafeedConfigProvider {
* @param actionListener Deleted datafeed listener
*/
public void deleteDatafeedConfig(String datafeedId, ActionListener<DeleteResponse> actionListener) {
DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId));
DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), DatafeedConfig.documentId(datafeedId));
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener<DeleteResponse>() {
@Override
@ -268,8 +263,7 @@ public class DatafeedConfigProvider {
public void updateDatefeedConfig(String datafeedId, DatafeedUpdate update, Map<String, String> headers,
BiConsumer<DatafeedConfig, ActionListener<Boolean>> validator,
ActionListener<DatafeedConfig> updatedConfigListener) {
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(datafeedId));
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), DatafeedConfig.documentId(datafeedId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
@ -325,15 +319,15 @@ public class DatafeedConfigProvider {
ActionListener<IndexResponse> listener) {
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
XContentBuilder updatedSource = updatedConfig.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS));
IndexRequestBuilder indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, DatafeedConfig.documentId(updatedConfig.getId()))
.setSource(updatedSource)
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName())
.id(DatafeedConfig.documentId(updatedConfig.getId()))
.source(updatedSource)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
indexRequest.setIfSeqNo(seqNo);
indexRequest.setIfPrimaryTerm(primaryTerm);
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest.request(), listener);
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener);
} catch (IOException e) {
listener.onFailure(

39
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
View File

@ -21,7 +21,6 @@ import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
@ -62,7 +61,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Detector;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
@ -120,12 +118,11 @@ public class JobConfigProvider {
public void putJob(Job job, ActionListener<IndexResponse> listener) {
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
XContentBuilder source = job.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS));
IndexRequest indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(job.getId()))
.setSource(source)
.setOpType(DocWriteRequest.OpType.CREATE)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.request();
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName())
.id(Job.documentId(job.getId()))
.source(source)
.opType(DocWriteRequest.OpType.CREATE)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap(
listener::onResponse,
@ -155,8 +152,7 @@ public class JobConfigProvider {
* @param jobListener Job listener
*/
public void getJob(String jobId, ActionListener<Job.Builder> jobListener) {
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getRequest, new ActionListener<GetResponse>() {
@Override
@ -193,8 +189,7 @@ public class JobConfigProvider {
* @param actionListener Deleted job listener
*/
public void deleteJob(String jobId, boolean errorIfMissing, ActionListener<DeleteResponse> actionListener) {
DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
DeleteRequest request = new DeleteRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, request, new ActionListener<DeleteResponse>() {
@ -230,8 +225,7 @@ public class JobConfigProvider {
*/
public void updateJob(String jobId, JobUpdate update, ByteSizeValue maxModelMemoryLimit,
ActionListener<Job> updatedJobListener) {
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
@ -295,8 +289,7 @@ public class JobConfigProvider {
*/
public void updateJobWithValidation(String jobId, JobUpdate update, ByteSizeValue maxModelMemoryLimit,
UpdateValidator validator, ActionListener<Job> updatedJobListener) {
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
@ -347,14 +340,14 @@ public class JobConfigProvider {
ActionListener<Job> updatedJobListener) {
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
XContentBuilder updatedSource = updatedJob.toXContent(builder, ToXContent.EMPTY_PARAMS);
IndexRequestBuilder indexRequest = client.prepareIndex(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(updatedJob.getId()))
.setSource(updatedSource)
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName())
.id(Job.documentId(updatedJob.getId()))
.source(updatedSource)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
indexRequest.setIfSeqNo(seqNo);
indexRequest.setIfPrimaryTerm(primaryTerm);
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest.request(), ActionListener.wrap(
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap(
indexResponse -> {
assert indexResponse.getResult() == DocWriteResponse.Result.UPDATED;
updatedJobListener.onResponse(updatedJob);
@ -383,8 +376,7 @@ public class JobConfigProvider {
* @param listener Exists listener
*/
public void jobExists(String jobId, boolean errorIfMissing, ActionListener<Boolean> listener) {
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
GetRequest getRequest = new GetRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
getRequest.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE);
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@ -458,8 +450,7 @@ public class JobConfigProvider {
* @param listener Responds with true if successful else an error
*/
public void markJobAsDeleting(String jobId, ActionListener<Boolean> listener) {
UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings.DOC_TYPE, Job.documentId(jobId));
UpdateRequest updateRequest = new UpdateRequest(AnomalyDetectorsIndex.configIndexName(), Job.documentId(jobId));
updateRequest.retryOnConflict(3);
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
updateRequest.doc(Collections.singletonMap(Job.DELETING.getPreferredName(), Boolean.TRUE));

8
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
View File

@ -17,7 +17,6 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
import java.io.IOException;
@ -54,10 +53,9 @@ public class JobDataCountsPersister {
*/
public void persistDataCounts(String jobId, DataCounts counts, ActionListener<Boolean> listener) {
try (XContentBuilder content = serialiseCounts(counts)) {
final IndexRequest request = client.prepareIndex(AnomalyDetectorsIndex.resultsWriteAlias(jobId), ElasticsearchMappings.DOC_TYPE,
DataCounts.documentId(jobId))
.setSource(content)
.request();
final IndexRequest request = new IndexRequest(AnomalyDetectorsIndex.resultsWriteAlias(jobId))
.id(DataCounts.documentId(jobId))
.source(content);
executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, request, new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse indexResponse) {

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
View File

@ -26,7 +26,6 @@ import java.util.List;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.DOC_TYPE;
/**
@ -78,7 +77,7 @@ public class JobRenormalizedResultsPersister {
public void updateResult(String id, String index, ToXContent resultDoc) {
try (XContentBuilder content = toXContentBuilder(resultDoc)) {
bulkRequest.add(new IndexRequest(index, DOC_TYPE, id).source(content));
bulkRequest.add(new IndexRequest(index).id(id).source(content));
} catch (IOException e) {
logger.error(new ParameterizedMessage("[{}] Error serialising result", jobId), e);
}

5
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
View File

@ -46,7 +46,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.DOC_TYPE;
/**
* Persists result types, Quantiles etc to Elasticsearch<br>
@ -175,7 +174,7 @@ public class JobResultsPersister {
private void indexResult(String id, ToXContent resultDoc, String resultType) {
try (XContentBuilder content = toXContentBuilder(resultDoc)) {
bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, id).source(content));
bulkRequest.add(new IndexRequest(indexName).id(id).source(content));
} catch (IOException e) {
logger.error(new ParameterizedMessage("[{}] Error serialising {}", jobId, resultType), e);
}
@ -349,7 +348,7 @@ public class JobResultsPersister {
logCall(indexName);
try (XContentBuilder content = toXContentBuilder(object)) {
IndexRequest indexRequest = new IndexRequest(indexName, DOC_TYPE, id).source(content).setRefreshPolicy(refreshPolicy);
IndexRequest indexRequest = new IndexRequest(indexName).id(id).source(content).setRefreshPolicy(refreshPolicy);
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest, listener, client::index);
} catch (IOException e) {
logger.error(new ParameterizedMessage("[{}] Error writing [{}]", jobId, (id == null) ? "auto-generated ID" : id), e);

42
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
View File

@ -124,6 +124,7 @@ import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.clientWithOrigin;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -289,8 +290,8 @@ public class JobResultsProvider {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
// This assumes the requested mapping will be merged with mappings from the template,
// and may need to be revisited if template merging is ever refactored
try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(ElasticsearchMappings.DOC_TYPE, termFields)) {
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, termFieldsMapping);
try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(termFields)) {
createIndexRequest.mapping(SINGLE_MAPPING_NAME, termFieldsMapping);
}
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(
@ -309,26 +310,22 @@ public class JobResultsProvider {
), client.admin().indices()::create);
} else {
long fieldCountLimit = MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.get(settings);
if (violatedFieldCountLimit(indexName, termFields.size(), fieldCountLimit, state)) {
IndexMetaData indexMetaData = state.metaData().index(indexName);
if (violatedFieldCountLimit(termFields.size(), fieldCountLimit, indexMetaData)) {
String message = "Cannot create job in index '" + indexName + "' as the " +
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey() + " setting will be violated";
finalListener.onFailure(new IllegalArgumentException(message));
} else {
updateIndexMappingWithTermFields(indexName, termFields,
updateIndexMappingWithTermFields(indexName, indexMetaData.mapping().type(), termFields,
ActionListener.wrap(createAliasListener::onResponse, finalListener::onFailure));
}
}
}
public static boolean violatedFieldCountLimit(
String indexName, long additionalFieldCount, long fieldCountLimit, ClusterState clusterState) {
long numFields = 0;
IndexMetaData indexMetaData = clusterState.metaData().index(indexName);
Iterator<MappingMetaData> mappings = indexMetaData.getMappings().valuesIt();
while (mappings.hasNext()) {
MappingMetaData mapping = mappings.next();
numFields += countFields(mapping.sourceAsMap());
}
public static boolean violatedFieldCountLimit(long additionalFieldCount, long fieldCountLimit, IndexMetaData indexMetaData) {
MappingMetaData mapping = indexMetaData.mapping();
long numFields = countFields(mapping.sourceAsMap());
return numFields + additionalFieldCount > fieldCountLimit;
}
@ -353,10 +350,12 @@ public class JobResultsProvider {
return count;
}
private void updateIndexMappingWithTermFields(String indexName, Collection<String> termFields, ActionListener<Boolean> listener) {
// Put the whole "doc" mapping, not just the term fields, otherwise we'll wipe the _meta section of the mapping
try (XContentBuilder termFieldsMapping = ElasticsearchMappings.resultsMapping(termFields)) {
final PutMappingRequest request = client.admin().indices().preparePutMapping(indexName).setType(ElasticsearchMappings.DOC_TYPE)
private void updateIndexMappingWithTermFields(String indexName, String mappingType, Collection<String> termFields,
ActionListener<Boolean> listener) {
// Put the whole mapping, not just the term fields, otherwise we'll wipe the _meta section of the mapping
try (XContentBuilder termFieldsMapping = ElasticsearchMappings.resultsMapping(mappingType, termFields)) {
final PutMappingRequest request = client.admin().indices().preparePutMapping(indexName)
.setType(mappingType)
.setSource(termFieldsMapping).request();
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request, new ActionListener<AcknowledgedResponse>() {
@Override
@ -504,7 +503,7 @@ public class JobResultsProvider {
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
return objectParser.apply(parser, null);
} catch (IOException e) {
errorHandler.accept(new ElasticsearchParseException("failed to parse " + hit.getType(), e));
errorHandler.accept(new ElasticsearchParseException("failed to parse " + hit.getId(), e));
return null;
}
}
@ -1184,7 +1183,7 @@ public class JobResultsProvider {
currentJobs.removeAll(jobIdsToRemove);
Calendar updatedCalendar = new Calendar(calendar.getId(), new ArrayList<>(currentJobs), calendar.getDescription());
UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, updatedCalendar.documentId());
UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, updatedCalendar.documentId());
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
@ -1252,8 +1251,7 @@ public class JobResultsProvider {
ids.remove(jobId);
return new Calendar(c.getId(), new ArrayList<>(ids), c.getDescription());
}).forEach(c -> {
UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE,
c.documentId());
UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, c.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
updateRequest.doc(c.toXContent(builder, ToXContent.EMPTY_PARAMS));
} catch (IOException e) {
@ -1276,7 +1274,7 @@ public class JobResultsProvider {
}
public void calendar(String calendarId, ActionListener<Calendar> listener) {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId));
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, Calendar.documentId(calendarId));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getRequest, new ActionListener<GetResponse>() {
@Override
public void onResponse(GetResponse getDocResponse) {

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamer.java
View File

@ -15,7 +15,6 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizerState;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
@ -75,7 +74,6 @@ public class StateStreamer {
try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN)) {
SearchResponse stateResponse = client.prepareSearch(indexName)
.setTypes(ElasticsearchMappings.DOC_TYPE)
.setSize(1)
.setQuery(QueryBuilders.idsQuery().addIds(stateDocId)).get();
if (stateResponse.getHits().getHits().length == 0) {
@ -102,7 +100,6 @@ public class StateStreamer {
try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN)) {
SearchResponse stateResponse = client.prepareSearch(indexName)
.setTypes(ElasticsearchMappings.DOC_TYPE)
.setSize(1)
.setQuery(QueryBuilders.idsQuery().addIds(docId)).get();
if (stateResponse.getHits().getHits().length == 0) {

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectStateProcessor.java
View File

@ -15,7 +15,6 @@ import org.elasticsearch.common.bytes.CompositeBytesReference;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.ml.process.StateProcessor;
import java.io.IOException;
@ -98,7 +97,7 @@ public class AutodetectStateProcessor implements StateProcessor {
void persist(BytesReference bytes) throws IOException {
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(bytes, AnomalyDetectorsIndex.jobStateIndexWriteAlias(), ElasticsearchMappings.DOC_TYPE, XContentType.JSON);
bulkRequest.add(bytes, AnomalyDetectorsIndex.jobStateIndexWriteAlias(), XContentType.JSON);
if (bulkRequest.numberOfActions() > 0) {
LOGGER.trace("[{}] Persisting job state document", jobId);
try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN)) {

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
View File

@ -18,7 +18,6 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizerState;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelState;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
@ -105,7 +104,6 @@ public class UnusedStateRemover implements MlDataRemover {
LOGGER.info("Found [{}] unused state documents; attempting to delete",
unusedDocIds.size());
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(AnomalyDetectorsIndex.jobStateIndexPattern())
.types(ElasticsearchMappings.DOC_TYPE)
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.setQuery(QueryBuilders.idsQuery().addIds(unusedDocIds.toArray(new String[0])));
client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, ActionListener.wrap(

15
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.notifications;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
@ -38,30 +37,30 @@ public class Auditor {
}
public void info(String jobId, String message) {
indexDoc(AuditMessage.TYPE.getPreferredName(), AuditMessage.newInfo(jobId, message, nodeName));
indexDoc(AuditMessage.newInfo(jobId, message, nodeName));
}
public void warning(String jobId, String message) {
indexDoc(AuditMessage.TYPE.getPreferredName(), AuditMessage.newWarning(jobId, message, nodeName));
indexDoc(AuditMessage.newWarning(jobId, message, nodeName));
}
public void error(String jobId, String message) {
indexDoc(AuditMessage.TYPE.getPreferredName(), AuditMessage.newError(jobId, message, nodeName));
indexDoc(AuditMessage.newError(jobId, message, nodeName));
}
private void indexDoc(String type, ToXContent toXContent) {
IndexRequest indexRequest = new IndexRequest(AuditorField.NOTIFICATIONS_INDEX, type);
private void indexDoc(ToXContent toXContent) {
IndexRequest indexRequest = new IndexRequest(AuditorField.NOTIFICATIONS_INDEX);
indexRequest.source(toXContentBuilder(toXContent));
indexRequest.timeout(TimeValue.timeValueSeconds(5));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest, new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse indexResponse) {
LOGGER.trace("Successfully persisted {}", type);
LOGGER.trace("Successfully persisted audit message");
}
@Override
public void onFailure(Exception e) {
LOGGER.debug(new ParameterizedMessage("Error writing {}", new Object[]{type}, e));
LOGGER.debug("Error writing audit message", e);
}
}, client::index);
}

14
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java
View File

@ -20,7 +20,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
import org.elasticsearch.test.ESTestCase;
@ -34,12 +33,9 @@ import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
import org.elasticsearch.xpack.core.ml.job.config.Detector;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.config.JobState;
import org.elasticsearch.xpack.core.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction.DatafeedTask;
import org.elasticsearch.xpack.ml.action.TransportStartDatafeedActionTests;
import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder;
import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.junit.Before;
@ -98,12 +94,6 @@ public class DatafeedManagerTests extends ESTestCase {
clusterService = mock(ClusterService.class);
when(clusterService.state()).thenReturn(cs.build());
ArgumentCaptor<XContentBuilder> argumentCaptor = ArgumentCaptor.forClass(XContentBuilder.class);
Client client = new MockClientBuilder("foo")
.prepareIndex(AuditorField.NOTIFICATIONS_INDEX, AuditMessage.TYPE.getPreferredName(), "responseId", argumentCaptor)
.build();
DiscoveryNode dNode = mock(DiscoveryNode.class);
when(dNode.getName()).thenReturn("this_node_has_a_name");
when(clusterService.localNode()).thenReturn(dNode);
@ -136,8 +126,8 @@ public class DatafeedManagerTests extends ESTestCase {
AutodetectProcessManager autodetectProcessManager = mock(AutodetectProcessManager.class);
doAnswer(invocation -> hasOpenAutodetectCommunicator.get()).when(autodetectProcessManager).hasOpenAutodetectCommunicator(anyLong());
datafeedManager = new DatafeedManager(threadPool, client, clusterService, datafeedJobBuilder, () -> currentTime, auditor,
autodetectProcessManager);
datafeedManager = new DatafeedManager(threadPool, mock(Client.class), clusterService, datafeedJobBuilder,
() -> currentTime, auditor, autodetectProcessManager);
verify(clusterService).addListener(capturedClusterStateListener.capture());
}

6
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
View File

@ -504,7 +504,7 @@ public class JobResultsProviderIT extends MlSingleNodeTestCase {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for (ScheduledEvent event : events) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE);
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true"));
indexRequest.source(event.toXContent(builder, params));
@ -547,7 +547,7 @@ public class JobResultsProviderIT extends MlSingleNodeTestCase {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for (MlFilter filter : filters) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId());
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME).id(filter.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true"));
indexRequest.source(filter.toXContent(builder, params));
@ -577,7 +577,7 @@ public class JobResultsProviderIT extends MlSingleNodeTestCase {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for (Calendar calendar: calendars) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId());
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME).id(calendar.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_TYPE, "true"));
indexRequest.source(calendar.toXContent(builder, params));

13
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java
View File

@ -7,7 +7,7 @@ package org.elasticsearch.xpack.ml.integration;
import org.elasticsearch.Version;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.PlainActionFuture;
@ -39,7 +39,6 @@ import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck;
import org.elasticsearch.xpack.ml.MlConfigMigrator;
import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
@ -213,13 +212,12 @@ public class MlConfigMigratorIT extends MlSingleNodeTestCase {
AnomalyDetectorsIndex.createStateIndexAndAliasIfNecessary(client(), clusterService.state(), future);
future.actionGet();
IndexRequestBuilder indexRequest = client().prepareIndex(AnomalyDetectorsIndex.jobStateIndexWriteAlias(),
ElasticsearchMappings.DOC_TYPE, "ml-config")
.setSource(Collections.singletonMap("a_field", "a_value"))
.setOpType(DocWriteRequest.OpType.CREATE)
IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).id("ml-config")
.source(Collections.singletonMap("a_field", "a_value"))
.opType(DocWriteRequest.OpType.CREATE)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
indexRequest.execute().actionGet();
client().index(indexRequest).actionGet();
doAnswer(invocation -> {
ClusterStateUpdateTask listener = (ClusterStateUpdateTask) invocation.getArguments()[1];
@ -386,7 +384,6 @@ public class MlConfigMigratorIT extends MlSingleNodeTestCase {
client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.jobStateIndexPattern()).get();
SearchResponse searchResponse = client()
.prepareSearch(AnomalyDetectorsIndex.jobStateIndexPattern())
.setTypes(ElasticsearchMappings.DOC_TYPE)
.setSize(1)
.setQuery(QueryBuilders.idsQuery().addIds("ml-config"))
.get();

47
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
View File

@ -770,39 +770,38 @@ public class JobResultsProviderTests extends ESTestCase {
public void testViolatedFieldCountLimit() throws Exception {
Map<String, Object> mapping = new HashMap<>();
for (int i = 0; i < 10; i++) {
int i = 0;
for (; i < 10; i++) {
mapping.put("field" + i, Collections.singletonMap("type", "string"));
}
IndexMetaData.Builder indexMetaData1 = new IndexMetaData.Builder("index1")
.settings(Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
.putMapping(new MappingMetaData("type1", Collections.singletonMap("properties", mapping)));
MetaData metaData = MetaData.builder()
.put(indexMetaData1)
.build();
boolean result = JobResultsProvider.violatedFieldCountLimit("index1", 0, 10,
ClusterState.builder(new ClusterName("_name")).metaData(metaData).build());
assertFalse(result);
result = JobResultsProvider.violatedFieldCountLimit("index1", 1, 10,
ClusterState.builder(new ClusterName("_name")).metaData(metaData).build());
assertTrue(result);
IndexMetaData.Builder indexMetaData2 = new IndexMetaData.Builder("index1")
IndexMetaData indexMetaData1 = new IndexMetaData.Builder("index1")
.settings(Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
.putMapping(new MappingMetaData("type1", Collections.singletonMap("properties", mapping)))
.putMapping(new MappingMetaData("type2", Collections.singletonMap("properties", mapping)));
metaData = MetaData.builder()
.put(indexMetaData2)
.build();
result = JobResultsProvider.violatedFieldCountLimit("index1", 0, 19,
ClusterState.builder(new ClusterName("_name")).metaData(metaData).build());
boolean result = JobResultsProvider.violatedFieldCountLimit(0, 10, indexMetaData1);
assertFalse(result);
result = JobResultsProvider.violatedFieldCountLimit(1, 10, indexMetaData1);
assertTrue(result);
for (; i < 20; i++) {
mapping.put("field" + i, Collections.singletonMap("type", "string"));
}
IndexMetaData indexMetaData2 = new IndexMetaData.Builder("index1")
.settings(Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
.putMapping(new MappingMetaData("type1", Collections.singletonMap("properties", mapping)))
.build();
result = JobResultsProvider.violatedFieldCountLimit(0, 19, indexMetaData2);
assertTrue(result);
}

36
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java
View File

@ -28,14 +28,11 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
@ -355,39 +352,6 @@ public class MockClientBuilder {
return this;
}
public MockClientBuilder prepareSearchAnySize(String index, String type, SearchResponse response, ArgumentCaptor<QueryBuilder> filter) {
SearchRequestBuilder builder = mock(SearchRequestBuilder.class);
when(builder.setTypes(eq(type))).thenReturn(builder);
when(builder.addSort(any(SortBuilder.class))).thenReturn(builder);
when(builder.setQuery(filter.capture())).thenReturn(builder);
when(builder.setPostFilter(filter.capture())).thenReturn(builder);
when(builder.setFrom(any(Integer.class))).thenReturn(builder);
when(builder.setSize(any(Integer.class))).thenReturn(builder);
when(builder.setFetchSource(eq(true))).thenReturn(builder);
when(builder.addDocValueField(any(String.class))).thenReturn(builder);
when(builder.addDocValueField(any(String.class), any(String.class))).thenReturn(builder);
when(builder.addSort(any(String.class), any(SortOrder.class))).thenReturn(builder);
when(builder.get()).thenReturn(response);
when(client.prepareSearch(eq(index))).thenReturn(builder);
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareIndex(String index, String type, String responseId, ArgumentCaptor<XContentBuilder> getSource) {
IndexRequestBuilder builder = mock(IndexRequestBuilder.class);
PlainActionFuture<IndexResponse> actionFuture = mock(PlainActionFuture.class);
IndexResponse response = mock(IndexResponse.class);
when(response.getId()).thenReturn(responseId);
when(client.prepareIndex(eq(index), eq(type))).thenReturn(builder);
when(client.prepareIndex(eq(index), eq(type), any(String.class))).thenReturn(builder);
when(builder.setSource(getSource.capture())).thenReturn(builder);
when(builder.setRefreshPolicy(eq(RefreshPolicy.IMMEDIATE))).thenReturn(builder);
when(builder.execute()).thenReturn(actionFuture);
when(actionFuture.actionGet()).thenReturn(response);
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareAlias(String indexName, String alias, QueryBuilder filter) {
when(aliasesRequestBuilder.addAlias(eq(indexName), eq(alias), eq(filter))).thenReturn(aliasesRequestBuilder);

3
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/notifications/AuditorTests.java
View File

@ -51,7 +51,6 @@ public class AuditorTests extends ESTestCase {
verify(client).index(indexRequestCaptor.capture(), any());
IndexRequest indexRequest = indexRequestCaptor.getValue();
assertArrayEquals(new String[] {".ml-notifications"}, indexRequest.indices());
assertEquals("audit_message", indexRequest.type());
assertEquals(TimeValue.timeValueSeconds(5), indexRequest.timeout());
AuditMessage auditMessage = parseAuditMessage(indexRequest.source());
assertEquals("foo", auditMessage.getJobId());
@ -66,7 +65,6 @@ public class AuditorTests extends ESTestCase {
verify(client).index(indexRequestCaptor.capture(), any());
IndexRequest indexRequest = indexRequestCaptor.getValue();
assertArrayEquals(new String[] {".ml-notifications"}, indexRequest.indices());
assertEquals("audit_message", indexRequest.type());
assertEquals(TimeValue.timeValueSeconds(5), indexRequest.timeout());
AuditMessage auditMessage = parseAuditMessage(indexRequest.source());
assertEquals("bar", auditMessage.getJobId());
@ -81,7 +79,6 @@ public class AuditorTests extends ESTestCase {
verify(client).index(indexRequestCaptor.capture(), any());
IndexRequest indexRequest = indexRequestCaptor.getValue();
assertArrayEquals(new String[] {".ml-notifications"}, indexRequest.indices());
assertEquals("audit_message", indexRequest.type());
assertEquals(TimeValue.timeValueSeconds(5), indexRequest.timeout());
AuditMessage auditMessage = parseAuditMessage(indexRequest.source());
assertEquals("foobar", auditMessage.getJobId());

2
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
View File

@ -35,7 +35,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-custom-all-test-1
type: doc
id: custom_all_1464739200000_1_1
body:
{
@ -62,7 +61,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-custom-all-test-2
type: doc
id: custom_all_1464739200000_1_2
body:
{

6
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml
View File

@ -34,7 +34,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "delete-forecast-job_model_forecast_someforecastid_1486591200000_1800_0_961_0"
body:
{
@ -56,7 +55,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "delete-forecast-job_model_forecast_someforecastid_1486591300000_1800_0_961_0"
body:
{
@ -78,7 +76,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "delete-forecast-job_model_forecast_request_stats_someforecastid"
body:
{
@ -112,19 +109,16 @@ setup:
get:
id: delete-forecast-job_model_forecast_request_stats_someforecastid
index: .ml-anomalies-shared
type: doc
- do:
catch: missing
get:
id: delete-forecast-job_model_forecast_someforecastid_1486591300000_1800_0_961_0
index: .ml-anomalies-shared
type: doc
- do:
catch: missing
get:
id: delete-forecast-job_model_forecast_someforecastid_1486591200000_1800_0_961_0
index: .ml-anomalies-shared
type: doc
---
"Test delete on _all forecasts not allow no forecasts":

6
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
View File

@ -39,7 +39,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-delete-model-snapshot
type: doc
id: "delete-model-snapshot_model_snapshot_inactive-snapshot"
body: >
{
@ -57,7 +56,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "delete-model-snapshot_model_state_inactive-snapshot#1"
body: >
{
@ -69,7 +67,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "delete-model-snapshot_model_state_inactive-snapshot#2"
body: >
{
@ -82,7 +79,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-delete-model-snapshot
type: doc
id: "delete-model-snapshot_model_snapshot_active-snapshot"
body: >
{
@ -158,7 +154,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
count:
index: .ml-state
type: doc
- match: { count: 3 }
@ -191,7 +186,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
count:
index: .ml-state
type: doc
- match: { count: 1 }

1
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
View File

@ -8,7 +8,6 @@ setup:
Content-Type: application/json
index:
index: .ml-meta
type: doc
id: filter_imposter-filter
body: >
{

5
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
View File

@ -23,7 +23,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-get-model-snapshots
type: doc
id: "get-model-snapshots-1"
body: >
{
@ -39,7 +38,6 @@ setup:
Content-Type: application/json
index:
index: .ml-state
type: doc
id: "get-model-snapshots_model_state_1#1"
body: >
{
@ -51,7 +49,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-get-model-snapshots
type: doc
id: "get-model-snapshots-2"
body: >
{
@ -66,7 +63,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "get-model-snapshots_model_state_2#1"
body: >
{
@ -77,7 +73,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "get-model-snapshots_model_state_2#2"
body: >
{

14
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
View File

@ -186,7 +186,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-job2_categorizer_state#1
body:
key: value
@ -196,7 +195,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-job2_categorizer_state#2
body:
key: value
@ -299,7 +297,6 @@ setup:
headers:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
count:
type: doc
index: .ml-state
- match: {count: 0}
@ -307,7 +304,6 @@ setup:
headers:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
count:
type: doc
index: .ml-state
- match: {count: 0}
@ -315,7 +311,6 @@ setup:
headers:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
count:
type: doc
index: .ml-state
- match: {count: 0}
@ -387,7 +382,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: foo
type: doc
body:
key: value
@ -396,7 +390,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-anomalies-foo
type: doc
body:
key: value
@ -405,7 +398,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-anomalies-foo
type: doc
body:
key: value
job_id: foo
@ -512,7 +504,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-quantiles-job_quantiles
body:
state: quantile-state
@ -563,7 +554,6 @@ setup:
index:
index: .ml-anomalies-shared
type: doc
id: "index-layout-state-job_model_snapshot_123"
body: >
{
@ -579,7 +569,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-state-job_model_state_123#1
body:
state: new-model-state
@ -589,7 +578,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-state-job_model_state_123#2
body:
state: more-new-model-state
@ -599,7 +587,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-state-job_categorizer_state#1
body:
state: new-categorizer-state
@ -609,7 +596,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: index-layout-state-job_categorizer_state#2
body:
state: more-new-categorizer-state

3
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml
View File

@ -23,7 +23,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-buckets
type: doc
id: "jobs-get-result-buckets_1464739200000_1"
body:
{
@ -40,7 +39,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-buckets
type: doc
id: "jobs-get-result-buckets_1470009600000_2"
body:
{
@ -57,7 +55,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-buckets
type: doc
id: "jobs-get-result-buckets_1470096000000_3"
body:
{

3
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
View File

@ -23,7 +23,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-categories
type: doc
id: jobs-get-result-categories-1
body: { "job_id": "jobs-get-result-categories", "category_id": 1 }
- do:
@ -32,7 +31,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-categories
type: doc
id: jobs-get-result-categories-2
body: { "job_id": "jobs-get-result-categories", "category_id": 2 }
- do:
@ -41,7 +39,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-unrelated
type: doc
id: jobs-get-result-categories-3
body: { "job_id": "unrelated", "category_id": 1 }

3
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml
View File

@ -23,7 +23,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-get-influencers-test
type: doc
id: get-influencers-test_1464739200000_1_1
body:
{
@ -42,7 +41,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-get-influencers-test
type: doc
id: get-influencers-test_1464825600000_1_2
body:
{
@ -62,7 +60,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-get-influencers-test
type: doc
id: get-influencers-test_1464912000000_1_3
body:
{

10
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml
View File

@ -64,7 +64,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-60_1"
body:
{
@ -81,7 +80,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-60_2"
body:
{
@ -98,7 +96,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-60_3"
body:
{
@ -114,7 +111,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-30_1"
body:
{
@ -131,7 +127,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-30_2"
body:
{
@ -148,7 +143,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-30_3"
body:
{
@ -165,7 +159,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-17_1"
body:
{
@ -182,7 +175,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-17_2"
body:
{
@ -199,7 +191,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-17_3"
body:
{
@ -216,7 +207,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "jobs-get-result-overall-buckets-17_4"
body:
{

2
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml
View File

@ -23,7 +23,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-records
type: doc
id: jobs-get-result-records_1464739200000_1_1
body:
{
@ -40,7 +39,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-jobs-get-result-records
type: doc
id: jobs-get-result-records_1464825600000_1_2
body:
{

2
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
View File

@ -234,7 +234,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: job-stats-v54-bwc-test-data-counts
body:
{
@ -259,7 +258,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: job-stats-v54-bwc-test-model_size_stats
body:
{

1
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
View File

@ -25,7 +25,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-shared
type: doc
id: "new_doc"
body: >
{

1
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
View File

@ -89,7 +89,6 @@ setup:
- do:
get:
index: .ml-anomalies-post-data-job
type: doc
id: post-data-job_data_counts
- match: { _source.processed_record_count: 2 }

9
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
View File

@ -39,7 +39,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_model_snapshot_first"
body: >
{
@ -67,7 +66,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_model_snapshot_second"
body: >
{
@ -95,7 +93,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1464825600000_1"
body: >
{
@ -111,7 +108,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1464782400000_1"
body: >
{
@ -127,7 +123,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1462060800000_1"
body: >
{
@ -143,7 +138,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1464825600000_1_1"
body: >
{
@ -159,7 +153,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1462060800000_1_2"
body: >
{
@ -175,7 +168,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1464825600000_1_3"
body: {
"job_id": "revert-model-snapshot",
@ -193,7 +185,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-revert-model-snapshot
type: doc
id: "revert-model-snapshot_1462060800000_1_4"
body:
{

7
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
View File

@ -23,7 +23,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-update-model-snapshot
type: doc
id: "update-model-snapshot_model_snapshot_snapshot-1"
body: >
{
@ -39,7 +38,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "update-model-snapshot_model_state_1#1"
body: >
{
@ -50,7 +48,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "update-model-snapshot_model_state_1#2"
body: >
{
@ -61,7 +58,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "update-model-snapshot_model_state_1#3"
body: >
{
@ -73,7 +69,6 @@ setup:
Content-Type: application/json
index:
index: .ml-anomalies-update-model-snapshot
type: doc
id: "update-model-snapshot_model_snapshot_snapshot-2"
body: >
{
@ -90,7 +85,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "update-model-snapshot_model_state_2#1"
body: >
{
@ -101,7 +95,6 @@ setup:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
index:
index: .ml-state
type: doc
id: "update-model-snapshot_model_state_2#2"
body: >
{