From 8f49d01113500b1f8e405fbe2142c0bb85547bed Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Sun, 30 Jun 2019 19:37:00 +0300 Subject: [PATCH] [7.x][ML] Rename df-analytics `_id_copy` to `ml__id_copy` (#43754) (#43783) Renames `_id_copy` to `ml__id_copy` as field names starting with underscore are deprecated. The new field name `ml__id_copy` was chosen as an obscure enough field that users won't have in their data. Otherwise, this field is only intented to be used by df-analytics. --- .../integration/RunDataFrameAnalyticsIT.java | 6 +++++ .../dataframe/DataFrameAnalyticsFields.java | 20 -------------- .../ml/dataframe/DataFrameAnalyticsIndex.java | 27 ++++++++++++------- .../dataframe/DataFrameAnalyticsManager.java | 2 +- .../extractor/DataFrameDataExtractor.java | 4 +-- .../DataFrameAnalyticsIndexTests.java | 4 +-- .../DataFrameDataExtractorTests.java | 2 +- 7 files changed, 30 insertions(+), 35 deletions(-) delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java index 5696edcf646..c7295ce24db 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java @@ -91,7 +91,10 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest assertThat(destDoc.get(field), equalTo(sourceDoc.get(field))); } assertThat(destDoc.containsKey("ml"), is(true)); + + @SuppressWarnings("unchecked") Map resultsObject = (Map) destDoc.get("ml"); + assertThat(resultsObject.containsKey("outlier_score"), is(true)); double outlierScore = (double) resultsObject.get("outlier_score"); assertThat(outlierScore, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0))); @@ -209,7 +212,10 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest assertThat(destDoc.get(field), equalTo(sourceDoc.get(field))); } assertThat(destDoc.containsKey("ml"), is(true)); + + @SuppressWarnings("unchecked") Map resultsObject = (Map) destDoc.get("ml"); + assertThat(resultsObject.containsKey("outlier_score"), is(true)); double outlierScore = (double) resultsObject.get("outlier_score"); assertThat(outlierScore, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0))); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java deleted file mode 100644 index 4ade30ae68b..00000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.dataframe; - -public final class DataFrameAnalyticsFields { - - public static final String ID = "_id_copy"; - - // Metadata fields - static final String CREATION_DATE_MILLIS = "creation_date_in_millis"; - static final String VERSION = "version"; - static final String CREATED = "created"; - static final String CREATED_BY = "created_by"; - static final String ANALYTICS = "analytics"; - - private DataFrameAnalyticsFields() {} -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java index e07eb99a3f5..8ae7db0a9fb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java @@ -42,7 +42,16 @@ import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; /** * {@link DataFrameAnalyticsIndex} class encapsulates logic for creating destination index based on source index metadata. */ -final class DataFrameAnalyticsIndex { +public final class DataFrameAnalyticsIndex { + + public static final String ID_COPY = "ml__id_copy"; + + // Metadata fields + static final String CREATION_DATE_MILLIS = "creation_date_in_millis"; + static final String VERSION = "version"; + static final String CREATED = "created"; + static final String CREATED_BY = "created_by"; + static final String ANALYTICS = "analytics"; private static final String PROPERTIES = "properties"; private static final String META = "_meta"; @@ -122,7 +131,7 @@ final class DataFrameAnalyticsIndex { Integer maxNumberOfReplicas = findMaxSettingValue(settingsResponse, IndexMetaData.SETTING_NUMBER_OF_REPLICAS); Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataFrameAnalyticsFields.ID); + settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), ID_COPY); settingsBuilder.put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), SortOrder.ASC); if (maxNumberOfShards != null) { settingsBuilder.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, maxNumberOfShards); @@ -151,17 +160,17 @@ final class DataFrameAnalyticsIndex { Map properties = getOrPutDefault(mappingsAsMap, PROPERTIES, HashMap::new); Map idCopyMapping = new HashMap<>(); idCopyMapping.put("type", "keyword"); - properties.put(DataFrameAnalyticsFields.ID, idCopyMapping); + properties.put(ID_COPY, idCopyMapping); } private static void addMetaData(Map mappingsAsMap, String analyticsId, Clock clock) { Map metadata = getOrPutDefault(mappingsAsMap, META, HashMap::new); - metadata.put(DataFrameAnalyticsFields.CREATION_DATE_MILLIS, clock.millis()); - metadata.put(DataFrameAnalyticsFields.CREATED_BY, "data-frame-analytics"); + metadata.put(CREATION_DATE_MILLIS, clock.millis()); + metadata.put(CREATED_BY, "data-frame-analytics"); Map versionMapping = new HashMap<>(); - versionMapping.put(DataFrameAnalyticsFields.CREATED, Version.CURRENT); - metadata.put(DataFrameAnalyticsFields.VERSION, versionMapping); - metadata.put(DataFrameAnalyticsFields.ANALYTICS, analyticsId); + versionMapping.put(CREATED, Version.CURRENT); + metadata.put(VERSION, versionMapping); + metadata.put(ANALYTICS, analyticsId); } private static V getOrPutDefault(Map map, K key, Supplier valueSupplier) { @@ -182,7 +191,7 @@ final class DataFrameAnalyticsIndex { String type = mappings.keysIt().next(); Map addedMappings = Collections.singletonMap(PROPERTIES, - Collections.singletonMap(DataFrameAnalyticsFields.ID, Collections.singletonMap("type", "keyword"))); + Collections.singletonMap(ID_COPY, Collections.singletonMap("type", "keyword"))); PutMappingRequest putMappingRequest = new PutMappingRequest(getIndexResponse.indices()); putMappingRequest.type(type); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index c7cfe2b6253..9132e0f8192 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -150,7 +150,7 @@ public class DataFrameAnalyticsManager { reindexRequest.setSourceIndices(config.getSource().getIndex()); reindexRequest.setSourceQuery(config.getSource().getParsedQuery()); reindexRequest.setDestIndex(config.getDest().getIndex()); - reindexRequest.setScript(new Script("ctx._source." + DataFrameAnalyticsFields.ID + " = ctx._id")); + reindexRequest.setScript(new Script("ctx._source." + DataFrameAnalyticsIndex.ID_COPY + " = ctx._id")); final ThreadContext threadContext = client.threadPool().getThreadContext(); final Supplier supplier = threadContext.newRestorableContext(false); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java index 59cd78b4cc6..fa18f3bb25b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java @@ -23,7 +23,7 @@ import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField; -import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsFields; +import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex; import java.io.IOException; import java.util.ArrayList; @@ -126,7 +126,7 @@ public class DataFrameDataExtractor { .setScroll(SCROLL_TIMEOUT) // This ensures the search throws if there are failures and the scroll context gets cleared automatically .setAllowPartialSearchResults(false) - .addSort(DataFrameAnalyticsFields.ID, SortOrder.ASC) + .addSort(DataFrameAnalyticsIndex.ID_COPY, SortOrder.ASC) .setIndices(context.indices) .setSize(context.scrollSize) .setQuery(context.query); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java index 80391de519e..7079a3295bd 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java @@ -167,12 +167,12 @@ public class DataFrameAnalyticsIndexTests extends ESTestCase { containsInAnyOrder("index.number_of_shards", "index.number_of_replicas", "index.sort.field", "index.sort.order")); assertThat(createIndexRequest.settings().getAsInt("index.number_of_shards", -1), equalTo(5)); assertThat(createIndexRequest.settings().getAsInt("index.number_of_replicas", -1), equalTo(1)); - assertThat(createIndexRequest.settings().get("index.sort.field"), equalTo("_id_copy")); + assertThat(createIndexRequest.settings().get("index.sort.field"), equalTo("ml__id_copy")); assertThat(createIndexRequest.settings().get("index.sort.order"), equalTo("asc")); try (XContentParser parser = createParser(JsonXContent.jsonXContent, createIndexRequest.mappings().get("_doc"))) { Map map = parser.map(); - assertThat(extractValue("_doc.properties._id_copy.type", map), equalTo("keyword")); + assertThat(extractValue("_doc.properties.ml__id_copy.type", map), equalTo("keyword")); assertThat(extractValue("_doc.properties.field_1", map), equalTo("field_1_mappings")); assertThat(extractValue("_doc.properties.field_2", map), equalTo("field_2_mappings")); assertThat(extractValue("_doc._meta.analytics", map), equalTo(ANALYTICS_ID)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java index 778b2826a72..71baa0bb94f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java @@ -127,7 +127,7 @@ public class DataFrameDataExtractorTests extends ESTestCase { assertThat(searchRequest, containsString("\"query\":{\"match_all\":{\"boost\":1.0}}")); assertThat(searchRequest, containsString("\"docvalue_fields\":[{\"field\":\"field_1\"},{\"field\":\"field_2\"}]")); assertThat(searchRequest, containsString("\"_source\":{\"includes\":[],\"excludes\":[]}")); - assertThat(searchRequest, containsString("\"sort\":[{\"_id_copy\":{\"order\":\"asc\"}}]")); + assertThat(searchRequest, containsString("\"sort\":[{\"ml__id_copy\":{\"order\":\"asc\"}}]")); // Check continue scroll requests had correct ids assertThat(dataExtractor.capturedContinueScrollIds.size(), equalTo(2));