honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Datafeed deprecation checks (#38026) 

Deprecation checks for the ML datafeed
query and aggregations.
This commit is contained in:
David Roberts 2019-01-30 20:12:20 +00:00 committed by GitHub
parent 81c443c9de
commit be788160ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 239 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.core.deprecation;
import org.elasticsearch.Version;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse;
@ -23,9 +24,12 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -67,16 +71,19 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
private List<DeprecationIssue> clusterSettingsIssues;
private List<DeprecationIssue> nodeSettingsIssues;
private Map<String, List<DeprecationIssue>> indexSettingsIssues;
private List<DeprecationIssue> mlSettingsIssues;
public Response() {
}
public Response(List<DeprecationIssue> clusterSettingsIssues,
List<DeprecationIssue> nodeSettingsIssues,
Map<String, List<DeprecationIssue>> indexSettingsIssues) {
Map<String, List<DeprecationIssue>> indexSettingsIssues,
List<DeprecationIssue> mlSettingsIssues) {
this.clusterSettingsIssues = clusterSettingsIssues;
this.nodeSettingsIssues = nodeSettingsIssues;
this.indexSettingsIssues = indexSettingsIssues;
this.mlSettingsIssues = mlSettingsIssues;
}
public List<DeprecationIssue> getClusterSettingsIssues() {
@ -91,12 +98,21 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
return indexSettingsIssues;
}
public List<DeprecationIssue> getMlSettingsIssues() {
return mlSettingsIssues;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
clusterSettingsIssues = in.readList(DeprecationIssue::new);
nodeSettingsIssues = in.readList(DeprecationIssue::new);
indexSettingsIssues = in.readMapOfLists(StreamInput::readString, DeprecationIssue::new);
if (in.getVersion().onOrAfter(Version.V_6_7_0)) {
mlSettingsIssues = in.readList(DeprecationIssue::new);
} else {
mlSettingsIssues = Collections.emptyList();
}
}
@Override
@ -105,6 +121,9 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
out.writeList(clusterSettingsIssues);
out.writeList(nodeSettingsIssues);
out.writeMapOfLists(indexSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o));
if (out.getVersion().onOrAfter(Version.V_6_7_0)) {
out.writeList(mlSettingsIssues);
}
}
@Override
@ -114,10 +133,10 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
.array("node_settings", nodeSettingsIssues.toArray())
.field("index_settings")
.map(indexSettingsIssues)
.array("ml_settings", mlSettingsIssues.toArray())
.endObject();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
@ -125,12 +144,13 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
Response response = (Response) o;
return Objects.equals(clusterSettingsIssues, response.clusterSettingsIssues) &&
Objects.equals(nodeSettingsIssues, response.nodeSettingsIssues) &&
Objects.equals(indexSettingsIssues, response.indexSettingsIssues);
Objects.equals(indexSettingsIssues, response.indexSettingsIssues) &&
Objects.equals(mlSettingsIssues, response.mlSettingsIssues);
}
@Override
public int hashCode() {
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
}
/**
@ -145,22 +165,30 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
* @param indexNameExpressionResolver Used to resolve indices into their concrete names
* @param indices The list of index expressions to evaluate using `indexNameExpressionResolver`
* @param indicesOptions The options to use when resolving and filtering which indices to check
* @param datafeeds The ml datafeed configurations
* @param clusterSettingsChecks The list of cluster-level checks
* @param nodeSettingsChecks The list of node-level checks
* @param indexSettingsChecks The list of index-level checks that will be run across all specified
* concrete indices
* @param mlSettingsCheck The list of ml checks
* @return The list of deprecation issues found in the cluster
*/
public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List<NodeStats> nodesStats, ClusterState state,
IndexNameExpressionResolver indexNameExpressionResolver,
String[] indices, IndicesOptions indicesOptions,
List<DatafeedConfig> datafeeds,
List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks) {
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
List<DeprecationIssue> clusterSettingsIssues = filterChecks(clusterSettingsChecks,
(c) -> c.apply(state));
List<DeprecationIssue> nodeSettingsIssues = filterChecks(nodeSettingsChecks,
(c) -> c.apply(nodesInfo, nodesStats));
List<DeprecationIssue> mlSettingsIssues = new ArrayList<>();
for (DatafeedConfig config : datafeeds) {
mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config)));
}
String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, indices);
@ -174,7 +202,7 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
}
}
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
}
}

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
View File

@ -665,7 +665,7 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
}
}
void setQuery(Map<String, Object> query) {
public void setQuery(Map<String, Object> query) {
this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName());
}

22
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java
View File

@ -22,6 +22,8 @@ import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.AbstractStreamableTestCase;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests;
import java.io.IOException;
import java.util.Arrays;
@ -45,13 +47,15 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
List<DeprecationIssue> nodeIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
List<DeprecationIssue> mlIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
Map<String, List<DeprecationIssue>> indexIssues = new HashMap<>();
for (int i = 0; i < randomIntBetween(0, 10); i++) {
List<DeprecationIssue> perIndexIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
indexIssues.put(randomAlphaOfLength(10), perIndexIssues);
}
return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues);
return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues, mlIssues);
}
@Override
@ -80,12 +84,14 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
List<NodeStats> nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null,
null, null, null, null, null, null, null, null,
null, null, null, null));
List<DatafeedConfig> datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo"));
IndexNameExpressionResolver resolver = new IndexNameExpressionResolver();
IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false,
true, true);
boolean clusterIssueFound = randomBoolean();
boolean nodeIssueFound = randomBoolean();
boolean indexIssueFound = randomBoolean();
boolean mlIssueFound = randomBoolean();
DeprecationIssue foundIssue = DeprecationIssueTests.createTestInstance();
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks =
Collections.unmodifiableList(Arrays.asList(
@ -100,10 +106,14 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
Collections.unmodifiableList(Arrays.asList(
(idx) -> indexIssueFound ? foundIssue : null
));
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsChecks =
Collections.unmodifiableList(Arrays.asList(
(idx) -> mlIssueFound ? foundIssue : null
));
DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state,
resolver, Strings.EMPTY_ARRAY, indicesOptions,
clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks);
resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds,
clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks);
if (clusterIssueFound) {
assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
@ -123,5 +133,11 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
} else {
assertTrue(response.getIndexSettingsIssues().isEmpty());
}
if (mlIssueFound) {
assertThat(response.getMlSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
} else {
assertTrue(response.getMlSettingsIssues().isEmpty());
}
}
}

7
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import java.util.Arrays;
import java.util.Collections;
@ -41,6 +42,12 @@ public class DeprecationChecks {
Collections.unmodifiableList(Arrays.asList(
IndexDeprecationChecks::oldIndicesCheck));
static List<Function<DatafeedConfig, DeprecationIssue>> ML_SETTINGS_CHECKS =
Collections.unmodifiableList(Arrays.asList(
MlDeprecationChecks::checkDataFeedAggregations,
MlDeprecationChecks::checkDataFeedQuery
));
/**
* helper utility function to reduce repeat of running a specific {@link List} of checks.
*

44
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.deprecation;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import java.util.List;
/**
* Check the {@link DatafeedConfig} query and aggregations for deprecated usages.
*/
final class MlDeprecationChecks {
private MlDeprecationChecks() {
}
static DeprecationIssue checkDataFeedQuery(DatafeedConfig datafeedConfig) {
List<String> deprecations = datafeedConfig.getQueryDeprecations();
if (deprecations.isEmpty()) {
return null;
} else {
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
"Datafeed [" + datafeedConfig.getId() + "] uses deprecated query options",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes",
deprecations.toString());
}
}
static DeprecationIssue checkDataFeedAggregations(DatafeedConfig datafeedConfig) {
List<String> deprecations = datafeedConfig.getAggDeprecations();
if (deprecations.isEmpty()) {
return null;
} else {
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
"Datafeed [" + datafeedConfig.getId() + "] uses deprecated aggregation options",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
"#breaking_70_aggregations_changes", deprecations.toString());
}
}
}

40
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.license.XPackLicenseState;
@ -26,7 +27,13 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import java.util.Collections;
import java.util.List;
public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction<DeprecationInfoAction.Request,
DeprecationInfoAction.Response> {
@ -34,9 +41,10 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio
private final XPackLicenseState licenseState;
private final NodeClient client;
private final IndexNameExpressionResolver indexNameExpressionResolver;
private final Settings settings;
@Inject
public TransportDeprecationInfoAction(TransportService transportService, ClusterService clusterService,
public TransportDeprecationInfoAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
XPackLicenseState licenseState, NodeClient client) {
@ -45,6 +53,7 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio
this.licenseState = licenseState;
this.client = client;
this.indexNameExpressionResolver = indexNameExpressionResolver;
this.settings = settings;
}
@Override
@ -83,11 +92,20 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio
if (nodesStatsResponse.hasFailures()) {
throw nodesStatsResponse.failures().get(0);
}
listener.onResponse(DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
getDatafeedConfigs(ActionListener.wrap(
datafeeds -> {
listener.onResponse(
DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
nodesStatsResponse.getNodes(), state, indexNameExpressionResolver,
request.indices(), request.indicesOptions(),
DeprecationChecks.CLUSTER_SETTINGS_CHECKS, DeprecationChecks.NODE_SETTINGS_CHECKS,
DeprecationChecks.INDEX_SETTINGS_CHECKS));
request.indices(), request.indicesOptions(), datafeeds,
DeprecationChecks.CLUSTER_SETTINGS_CHECKS,
DeprecationChecks.NODE_SETTINGS_CHECKS,
DeprecationChecks.INDEX_SETTINGS_CHECKS,
DeprecationChecks.ML_SETTINGS_CHECKS));
},
listener::onFailure
));
}, listener::onFailure),
client.admin().cluster()::nodesStats);
}, listener::onFailure), client.admin().cluster()::nodesInfo);
@ -95,4 +113,16 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio
listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION));
}
}
private void getDatafeedConfigs(ActionListener<List<DatafeedConfig>> listener) {
if (XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) == false) {
listener.onResponse(Collections.emptyList());
} else {
ClientHelper.executeAsyncWithOrigin(client, ClientHelper.DEPRECATION_ORIGIN, GetDatafeedsAction.INSTANCE,
new GetDatafeedsAction.Request(GetDatafeedsAction.ALL), ActionListener.wrap(
datafeedsResponse -> listener.onResponse(datafeedsResponse.getResponse().results()),
listener::onFailure
));
}
}
}

48
x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.deprecation;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import java.util.Collections;
public class MlDeprecationChecksTests extends ESTestCase {
@Override
protected boolean enableWarningsCheck() {
return false;
}
public void testCheckDataFeedQuery() {
DatafeedConfig.Builder goodDatafeed = new DatafeedConfig.Builder("good-df", "job-id");
goodDatafeed.setIndices(Collections.singletonList("some-index"));
goodDatafeed.setParsedQuery(new TermQueryBuilder("foo", "bar"));
assertNull(MlDeprecationChecks.checkDataFeedQuery(goodDatafeed.build()));
DatafeedConfig.Builder deprecatedDatafeed = new DatafeedConfig.Builder("df-with-deprecated-query", "job-id");
deprecatedDatafeed.setIndices(Collections.singletonList("some-index"));
// TODO: once some query syntax has been removed from 8.0 and deprecated in 7.x reinstate this test
// to check that particular query syntax causes a deprecation warning
/*
Map<String, Object> qs = new HashMap<>();
qs.put("query", "foo");
qs.put("use_dis_max", true);
Map<String, Object> query = Collections.singletonMap("query_string", qs);
deprecatedDatafeed.setQuery(query);
DeprecationIssue issue = MlDeprecationChecks.checkDataFeedQuery(deprecatedDatafeed.build());
assertNotNull(issue);
assertThat(issue.getDetails(), equalTo("[Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]]"));
assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.WARNING));
assertThat(issue.getMessage(), equalTo("Datafeed [df-with-deprecated-query] uses deprecated query options"));
assertThat(issue.getUrl(), equalTo("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
"#breaking_80_search_changes"));
*/
}
}

46
x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml
View File

@ -12,4 +12,50 @@ setup:
- length: { cluster_settings: 0 }
- length: { node_settings: 0 }
- length: { index_settings: 0 }
- length: { ml_settings: 0 }
---
"Test ml":
- skip:
version: "7.0.0 - "
features: ["headers", "warnings"]
reason: this test needs adjusting to contain syntax deprecated in 7.x and removed in 8.0
# Index the config directly to prevent the deprecated
# use_dis_max field being rewritten by the parser. This
# simulates the config being created in an older version
# of elasticsearch
- do:
headers:
Content-Type: application/json
index:
index: .ml-config
type: doc
id: deprecation-datafeed-datafeed
body: >
{
"datafeed_id" : "deprecation-datafeed",
"config_type" : "datafeed",
"job_id" : "deprecation-job",
"indices" : ["index-foo"],
"query" : {
"query_string" : {
"query" : "foo",
"use_dis_max" : true
}
}
}
- do:
indices.refresh:
index: [.ml-config]
# TODO: change the query and expected warnings to one that makes sense for 7.x
- do:
warnings:
- Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]
xpack.migration.deprecations:
index: "*"
- length: { ml_settings: 1 }
- match: { ml_settings.0.level : warning }
- match: { ml_settings.0.message : "Datafeed [deprecation-datafeed] uses deprecated query options" }