From c89917c799151ef555d7a3d9e387e5ddccaeaa21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Fri, 24 Apr 2020 10:59:32 +0200 Subject: [PATCH] Register DFA jobs on putAnalytics rather than via a separate method (#55458) (#55708) --- .../xpack/ml/integration/ClassificationIT.java | 14 -------------- .../xpack/ml/integration/DatafeedJobsRestIT.java | 6 +++++- .../xpack/ml/integration/MlJobIT.java | 4 +++- .../MlNativeDataFrameAnalyticsIntegTestCase.java | 11 +++++------ .../OutlierDetectionWithMissingFieldsIT.java | 1 - .../xpack/ml/integration/RegressionIT.java | 8 -------- .../ml/integration/RunDataFrameAnalyticsIT.java | 10 ---------- 7 files changed, 13 insertions(+), 41 deletions(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java index a60253412d6..33186ab8522 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java @@ -100,7 +100,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { null, null, null)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -148,7 +147,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 300, 0, KEYWORD_FIELD); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -209,7 +207,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { destIndex, null, new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, null, numTopClasses, 50.0, null)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -305,7 +302,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 350, 0, KEYWORD_FIELD); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -374,7 +370,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { } DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> startAnalytics(jobId)); @@ -393,7 +388,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { QueryBuilder query = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(KEYWORD_FIELD, KEYWORD_FIELD_VALUES)); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD), query); - registerAnalytics(config); putAnalytics(config); // Should not throw @@ -409,7 +403,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 100, 0, NESTED_FIELD); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(NESTED_FIELD)); - registerAnalytics(config); putAnalytics(config); startAnalytics(jobId); waitUntilAnalyticsIsStopped(jobId); @@ -428,7 +421,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 100, 0, KEYWORD_FIELD); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(ALIAS_TO_KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); startAnalytics(jobId); waitUntilAnalyticsIsStopped(jobId); @@ -447,7 +439,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 100, 0, NESTED_FIELD); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(ALIAS_TO_NESTED_FIELD)); - registerAnalytics(config); putAnalytics(config); startAnalytics(jobId); waitUntilAnalyticsIsStopped(jobId); @@ -482,7 +473,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null, new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, null)); - registerAnalytics(firstJob); putAnalytics(firstJob); String secondJobId = "classification_two_jobs_with_same_randomize_seed_2"; @@ -492,7 +482,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { DataFrameAnalyticsConfig secondJob = buildAnalytics(secondJobId, sourceIndex, secondJobDestIndex, null, new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, randomizeSeed)); - registerAnalytics(secondJob); putAnalytics(secondJob); // Let's run both jobs in parallel and wait until they are finished @@ -515,7 +504,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { assertThat(upgradeMode(), is(false)); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); startAnalytics(jobId); assertThat(analyticsTaskList(), hasSize(1)); @@ -561,7 +549,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { assertThat(upgradeMode(), is(false)); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); setUpgradeModeTo(true); @@ -581,7 +568,6 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 100, 0, KEYWORD_FIELD); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD)); - registerAnalytics(config); putAnalytics(config); startAnalytics(jobId); waitUntilAnalyticsIsStopped(jobId); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 373654a9c48..a3d0cff23d5 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.rollup.job.RollupJob; @@ -1210,7 +1211,10 @@ public class DatafeedJobsRestIT extends ESRestTestCase { public void clearMlState() throws Exception { new MlRestTestStateCleaner(logger, adminClient()).clearMlMetadata(); // Don't check rollup jobs because we clear them in the superclass. - waitForPendingTasks(adminClient(), taskName -> taskName.startsWith(RollupJob.NAME)); + // Don't check analytics jobs as they are independent of anomaly detection jobs and should not be created by this test. + waitForPendingTasks( + adminClient(), + taskName -> taskName.startsWith(RollupJob.NAME) || taskName.contains(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME)); } private static class DatafeedBuilder { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 82e6083f598..365149c1239 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -827,6 +828,7 @@ public class MlJobIT extends ESRestTestCase { @After public void clearMlState() throws Exception { new MlRestTestStateCleaner(logger, adminClient()).clearMlMetadata(); - ESRestTestCase.waitForPendingTasks(adminClient()); + // Don't check analytics jobs as they are independent of anomaly detection jobs and should not be created by this test. + waitForPendingTasks(adminClient(), taskName -> taskName.contains(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME)); } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java index 1cee2ecd75a..d257239901a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; @@ -76,7 +77,7 @@ import static org.hamcrest.Matchers.nullValue; */ abstract class MlNativeDataFrameAnalyticsIntegTestCase extends MlNativeIntegTestCase { - private List analytics = new ArrayList<>(); + private final List analytics = new ArrayList<>(); @Override protected void cleanUpResources() { @@ -91,7 +92,8 @@ abstract class MlNativeDataFrameAnalyticsIntegTestCase extends MlNativeIntegTest assertThat(deleteAnalytics(config.getId()).isAcknowledged(), is(true)); assertThat(searchStoredProgress(config.getId()).getHits().getTotalHits().value, equalTo(0L)); } catch (Exception e) { - // ignore + // just log and ignore + logger.error(new ParameterizedMessage("[{}] Could not clean up analytics job config", config.getId()), e); } } } @@ -110,13 +112,10 @@ abstract class MlNativeDataFrameAnalyticsIntegTestCase extends MlNativeIntegTest } } - protected void registerAnalytics(DataFrameAnalyticsConfig config) { + protected PutDataFrameAnalyticsAction.Response putAnalytics(DataFrameAnalyticsConfig config) { if (analytics.add(config) == false) { throw new IllegalArgumentException("analytics config [" + config.getId() + "] is already registered"); } - } - - protected PutDataFrameAnalyticsAction.Response putAnalytics(DataFrameAnalyticsConfig config) { PutDataFrameAnalyticsAction.Request request = new PutDataFrameAnalyticsAction.Request(config); return client().execute(PutDataFrameAnalyticsAction.INSTANCE, request).actionGet(); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java index c30f1c1a983..faeb43c7ab0 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java @@ -71,7 +71,6 @@ public class OutlierDetectionWithMissingFieldsIT extends MlNativeDataFrameAnalyt String id = "test_outlier_detection_with_missing_fields"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, sourceIndex + "-results", null, new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java index 9a9ed8c5aa2..fdd329118db 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java @@ -69,7 +69,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { null, null) ); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -127,7 +126,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 350, 0); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Regression(DEPENDENT_VARIABLE_FIELD)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -183,7 +181,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { destIndex, null, new Regression(DEPENDENT_VARIABLE_FIELD, BoostedTreeParams.builder().build(), null, 50.0, null)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -243,7 +240,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 350, 0); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Regression(DEPENDENT_VARIABLE_FIELD)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); @@ -312,7 +308,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null, new Regression(DEPENDENT_VARIABLE_FIELD, boostedTreeParams, null, 50.0, null)); - registerAnalytics(firstJob); putAnalytics(firstJob); String secondJobId = "regression_two_jobs_with_same_randomize_seed_2"; @@ -322,7 +317,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { DataFrameAnalyticsConfig secondJob = buildAnalytics(secondJobId, sourceIndex, secondJobDestIndex, null, new Regression(DEPENDENT_VARIABLE_FIELD, boostedTreeParams, null, 50.0, randomizeSeed)); - registerAnalytics(secondJob); putAnalytics(secondJob); // Let's run both jobs in parallel and wait until they are finished @@ -344,7 +338,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { indexData(sourceIndex, 100, 0); DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Regression(DEPENDENT_VARIABLE_FIELD)); - registerAnalytics(config); putAnalytics(config); startAnalytics(jobId); waitUntilAnalyticsIsStopped(jobId); @@ -384,7 +377,6 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase { destIndex, null, new Regression(DISCRETE_NUMERICAL_FEATURE_FIELD, BoostedTreeParams.builder().build(), null, null, null)); - registerAnalytics(config); putAnalytics(config); assertIsStopped(jobId); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java index 667dfba2e48..cdc5ffc881a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java @@ -98,7 +98,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest String id = "test_outlier_detection_with_few_docs"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, sourceIndex + "-results", null, new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -184,7 +183,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest String id = "test_outlier_detection_with_enough_docs_to_scroll"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, sourceIndex + "-results", "custom_ml", new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -259,7 +257,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest String id = "test_outlier_detection_with_more_fields_than_docvalue_limit"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, sourceIndex + "-results", null, new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -328,7 +325,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest String id = "test_stop_outlier_detection_with_enough_docs_to_scroll"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, sourceIndex + "-results", "custom_ml", new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -398,7 +394,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest .setDest(new DataFrameAnalyticsDest(destIndex, null)) .setAnalysis(new OutlierDetection.Builder().build()) .build(); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -460,7 +455,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest String id = "test_outlier_detection_with_pre_existing_dest_index"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, destIndex, null, new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -524,7 +518,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest .setModelMemoryLimit(modelMemoryLimit) .build(); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -569,7 +562,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest .setAllowLazyStart(true) .build(); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -619,7 +611,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest String id = "test_outlier_detection_stop_and_restart"; DataFrameAnalyticsConfig config = buildAnalytics(id, sourceIndex, sourceIndex + "-results", "custom_ml", new OutlierDetection.Builder().build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id); @@ -697,7 +688,6 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest .setOutlierFraction(0.04) .setStandardizationEnabled(true) .build()); - registerAnalytics(config); putAnalytics(config); assertIsStopped(id);