honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x][ML] Rename df-analytics `_id_copy` to `ml__id_copy` (#43754) (#43783) 

Renames `_id_copy` to `ml__id_copy` as field names starting with
underscore are deprecated. The new field name `ml__id_copy` was
chosen as an obscure enough field that users won't have in their data.
Otherwise, this field is only intented to be used by df-analytics.
This commit is contained in:
Dimitris Athanasiou 2019-06-30 19:37:00 +03:00 committed by GitHub
parent fca7a19713
commit 8f49d01113
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 30 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
View File

@ -91,7 +91,10 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest
assertThat(destDoc.get(field), equalTo(sourceDoc.get(field)));
}
assertThat(destDoc.containsKey("ml"), is(true));
@SuppressWarnings("unchecked")
Map<String, Object> resultsObject = (Map<String, Object>) destDoc.get("ml");
assertThat(resultsObject.containsKey("outlier_score"), is(true));
double outlierScore = (double) resultsObject.get("outlier_score");
assertThat(outlierScore, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)));
@ -209,7 +212,10 @@ public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTest
assertThat(destDoc.get(field), equalTo(sourceDoc.get(field)));
}
assertThat(destDoc.containsKey("ml"), is(true));
@SuppressWarnings("unchecked")
Map<String, Object> resultsObject = (Map<String, Object>) destDoc.get("ml");
assertThat(resultsObject.containsKey("outlier_score"), is(true));
double outlierScore = (double) resultsObject.get("outlier_score");
assertThat(outlierScore, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)));

20
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java
View File

@ -1,20 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.dataframe;
public final class DataFrameAnalyticsFields {
public static final String ID = "_id_copy";
// Metadata fields
static final String CREATION_DATE_MILLIS = "creation_date_in_millis";
static final String VERSION = "version";
static final String CREATED = "created";
static final String CREATED_BY = "created_by";
static final String ANALYTICS = "analytics";
private DataFrameAnalyticsFields() {}
}

27
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java
View File

@ -42,7 +42,16 @@ import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
/**
* {@link DataFrameAnalyticsIndex} class encapsulates logic for creating destination index based on source index metadata.
*/
final class DataFrameAnalyticsIndex {
public final class DataFrameAnalyticsIndex {
public static final String ID_COPY = "ml__id_copy";
// Metadata fields
static final String CREATION_DATE_MILLIS = "creation_date_in_millis";
static final String VERSION = "version";
static final String CREATED = "created";
static final String CREATED_BY = "created_by";
static final String ANALYTICS = "analytics";
private static final String PROPERTIES = "properties";
private static final String META = "_meta";
@ -122,7 +131,7 @@ final class DataFrameAnalyticsIndex {
Integer maxNumberOfReplicas = findMaxSettingValue(settingsResponse, IndexMetaData.SETTING_NUMBER_OF_REPLICAS);
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataFrameAnalyticsFields.ID);
settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), ID_COPY);
settingsBuilder.put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), SortOrder.ASC);
if (maxNumberOfShards != null) {
settingsBuilder.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, maxNumberOfShards);
@ -151,17 +160,17 @@ final class DataFrameAnalyticsIndex {
Map<String, Object> properties = getOrPutDefault(mappingsAsMap, PROPERTIES, HashMap::new);
Map<String, String> idCopyMapping = new HashMap<>();
idCopyMapping.put("type", "keyword");
properties.put(DataFrameAnalyticsFields.ID, idCopyMapping);
properties.put(ID_COPY, idCopyMapping);
}
private static void addMetaData(Map<String, Object> mappingsAsMap, String analyticsId, Clock clock) {
Map<String, Object> metadata = getOrPutDefault(mappingsAsMap, META, HashMap::new);
metadata.put(DataFrameAnalyticsFields.CREATION_DATE_MILLIS, clock.millis());
metadata.put(DataFrameAnalyticsFields.CREATED_BY, "data-frame-analytics");
metadata.put(CREATION_DATE_MILLIS, clock.millis());
metadata.put(CREATED_BY, "data-frame-analytics");
Map<String, Version> versionMapping = new HashMap<>();
versionMapping.put(DataFrameAnalyticsFields.CREATED, Version.CURRENT);
metadata.put(DataFrameAnalyticsFields.VERSION, versionMapping);
metadata.put(DataFrameAnalyticsFields.ANALYTICS, analyticsId);
versionMapping.put(CREATED, Version.CURRENT);
metadata.put(VERSION, versionMapping);
metadata.put(ANALYTICS, analyticsId);
}
private static <K, V> V getOrPutDefault(Map<K, Object> map, K key, Supplier<V> valueSupplier) {
@ -182,7 +191,7 @@ final class DataFrameAnalyticsIndex {
String type = mappings.keysIt().next();
Map<String, Object> addedMappings = Collections.singletonMap(PROPERTIES,
Collections.singletonMap(DataFrameAnalyticsFields.ID, Collections.singletonMap("type", "keyword")));
Collections.singletonMap(ID_COPY, Collections.singletonMap("type", "keyword")));
PutMappingRequest putMappingRequest = new PutMappingRequest(getIndexResponse.indices());
putMappingRequest.type(type);

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
View File

@ -150,7 +150,7 @@ public class DataFrameAnalyticsManager {
reindexRequest.setSourceIndices(config.getSource().getIndex());
reindexRequest.setSourceQuery(config.getSource().getParsedQuery());
reindexRequest.setDestIndex(config.getDest().getIndex());
reindexRequest.setScript(new Script("ctx._source." + DataFrameAnalyticsFields.ID + " = ctx._id"));
reindexRequest.setScript(new Script("ctx._source." + DataFrameAnalyticsIndex.ID_COPY + " = ctx._id"));
final ThreadContext threadContext = client.threadPool().getThreadContext();
final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(false);

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
View File

@ -23,7 +23,7 @@ import org.elasticsearch.search.fetch.StoredFieldsContext;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsFields;
import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex;
import java.io.IOException;
import java.util.ArrayList;
@ -126,7 +126,7 @@ public class DataFrameDataExtractor {
.setScroll(SCROLL_TIMEOUT)
// This ensures the search throws if there are failures and the scroll context gets cleared automatically
.setAllowPartialSearchResults(false)
.addSort(DataFrameAnalyticsFields.ID, SortOrder.ASC)
.addSort(DataFrameAnalyticsIndex.ID_COPY, SortOrder.ASC)
.setIndices(context.indices)
.setSize(context.scrollSize)
.setQuery(context.query);

4
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java
View File

@ -167,12 +167,12 @@ public class DataFrameAnalyticsIndexTests extends ESTestCase {
containsInAnyOrder("index.number_of_shards", "index.number_of_replicas", "index.sort.field", "index.sort.order"));
assertThat(createIndexRequest.settings().getAsInt("index.number_of_shards", -1), equalTo(5));
assertThat(createIndexRequest.settings().getAsInt("index.number_of_replicas", -1), equalTo(1));
assertThat(createIndexRequest.settings().get("index.sort.field"), equalTo("_id_copy"));
assertThat(createIndexRequest.settings().get("index.sort.field"), equalTo("ml__id_copy"));
assertThat(createIndexRequest.settings().get("index.sort.order"), equalTo("asc"));
try (XContentParser parser = createParser(JsonXContent.jsonXContent, createIndexRequest.mappings().get("_doc"))) {
Map<String, Object> map = parser.map();
assertThat(extractValue("_doc.properties._id_copy.type", map), equalTo("keyword"));
assertThat(extractValue("_doc.properties.ml__id_copy.type", map), equalTo("keyword"));
assertThat(extractValue("_doc.properties.field_1", map), equalTo("field_1_mappings"));
assertThat(extractValue("_doc.properties.field_2", map), equalTo("field_2_mappings"));
assertThat(extractValue("_doc._meta.analytics", map), equalTo(ANALYTICS_ID));

2
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
View File

@ -127,7 +127,7 @@ public class DataFrameDataExtractorTests extends ESTestCase {
assertThat(searchRequest, containsString("\"query\":{\"match_all\":{\"boost\":1.0}}"));
assertThat(searchRequest, containsString("\"docvalue_fields\":[{\"field\":\"field_1\"},{\"field\":\"field_2\"}]"));
assertThat(searchRequest, containsString("\"_source\":{\"includes\":[],\"excludes\":[]}"));
assertThat(searchRequest, containsString("\"sort\":[{\"_id_copy\":{\"order\":\"asc\"}}]"));
assertThat(searchRequest, containsString("\"sort\":[{\"ml__id_copy\":{\"order\":\"asc\"}}]"));
// Check continue scroll requests had correct ids
assertThat(dataExtractor.capturedContinueScrollIds.size(), equalTo(2));