honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] renaming inference processor field field_mappings to new name field_map (#53433) (#53502) 

This renames the `inference` processor configuration field `field_mappings` to `field_map`.

`field_mappings` is now deprecated.
This commit is contained in:
Benjamin Trent 2020-03-13 15:40:57 -04:00 committed by GitHub
parent b6c94fd73e
commit 4e43ede735
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 156 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
View File

@ -2297,7 +2297,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
" \"target_field\": \"regression_value\",\n" +
" \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
" \"inference_config\": {\"regression\": {}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +

4
docs/reference/ingest/processors/inference.asciidoc
View File

@ -14,7 +14,7 @@ ingested in the pipeline.
| Name | Required | Default | Description
| `model_id` | yes | - | (String) The ID of the model to load and infer against.
| `target_field` | no | `ml.inference.<processor_tag>` | (String) Field added to incoming documents to contain results objects.
| `field_mappings` | yes | - | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
| `field_map` | yes | - | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
| `inference_config` | yes | - | (Object) Contains the inference type and its options. There are two types: <<inference-processor-regression-opt,`regression`>> and <<inference-processor-classification-opt,`classification`>>.
include::common-options.asciidoc[]
|======
@ -26,7 +26,7 @@ include::common-options.asciidoc[]
"inference": {
"model_id": "flight_delay_regression-1571767128603",
"target_field": "FlightDelayMin_prediction_infer",
"field_mappings": {},
"field_map": {},
"inference_config": { "regression": {} }
}
}

3
x-pack/plugin/ml/qa/ml-with-security/build.gradle
View File

@ -139,6 +139,9 @@ integTest.runner {
'ml/inference_crud/Test put ensemble with empty models',
'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index',
'ml/inference_crud/Test put model with empty input.field_names',
'ml/inference_processor/Test create processor with missing mandatory fields',
'ml/inference_processor/Test create and delete pipeline with inference processor',
'ml/inference_processor/Test create processor with deprecated fields',
'ml/inference_stats_crud/Test get stats given missing trained model',
'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false',
'ml/jobs_crud/Test cannot create job with existing categorizer state document',

14
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
View File

@ -120,7 +120,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"num_top_feature_importance_values\": 2" +
" }},\n" +
" \"model_id\": \"test_classification\",\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
@ -133,7 +133,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"target_field\": \"ml.regression\",\n" +
" \"model_id\": \"test_regression\",\n" +
" \"inference_config\": {\"regression\":{}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
@ -166,7 +166,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"inference\": {\n" +
" \"model_id\": \"test_classification_missing\",\n" +
" \"inference_config\": {\"classification\":{}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
@ -204,7 +204,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"num_top_feature_importance_values\": 2" +
" }},\n" +
" \"model_id\": \"test_classification\",\n" +
" \"field_mappings\": {}\n" +
" \"field_map\": {}\n" +
" }\n" +
" }\n"+
" ]\n" +
@ -233,7 +233,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"inference\": {\n" +
" \"inference_config\": {\"classification\":{}},\n" +
" \"model_id\": \"lang_ident_model_1\",\n" +
" \"field_mappings\": {}\n" +
" \"field_map\": {}\n" +
" }\n" +
" }\n" +
" ]\n" +
@ -570,7 +570,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"model_id\": \"test_classification\",\n" +
" \"tag\": \"classification\",\n" +
" \"inference_config\": {\"classification\": {}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
@ -586,7 +586,7 @@ public class InferenceIngestIT extends ESRestTestCase {
" \"model_id\": \"test_regression\",\n" +
" \"tag\": \"regression\",\n" +
" \"inference_config\": {\"regression\": {}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +

21
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
View File

@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -63,6 +64,7 @@ public class InferenceProcessor extends AbstractProcessor {
public static final String INFERENCE_CONFIG = "inference_config";
public static final String TARGET_FIELD = "target_field";
public static final String FIELD_MAPPINGS = "field_mappings";
public static final String FIELD_MAP = "field_map";
private static final String DEFAULT_TARGET_FIELD = "ml.inference";
private final Client client;
@ -70,7 +72,7 @@ public class InferenceProcessor extends AbstractProcessor {
private final String targetField;
private final InferenceConfig inferenceConfig;
private final Map<String, String> fieldMapping;
private final Map<String, String> fieldMap;
private final InferenceAuditor auditor;
private volatile boolean previouslyLicensed;
private final AtomicBoolean shouldAudit = new AtomicBoolean(true);
@ -81,14 +83,14 @@ public class InferenceProcessor extends AbstractProcessor {
String targetField,
String modelId,
InferenceConfig inferenceConfig,
Map<String, String> fieldMapping) {
Map<String, String> fieldMap) {
super(tag);
this.client = ExceptionsHelper.requireNonNull(client, "client");
this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD);
this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG);
this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS);
this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP);
}
public String getModelId() {
@ -126,7 +128,7 @@ public class InferenceProcessor extends AbstractProcessor {
InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) {
Map<String, Object> fields = new HashMap<>(ingestDocument.getSourceAndMetadata());
Model.mapFieldsIfNecessary(fields, fieldMapping);
Model.mapFieldsIfNecessary(fields, fieldMap);
return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
}
@ -235,7 +237,14 @@ public class InferenceProcessor extends AbstractProcessor {
// If multiple inference processors are in the same pipeline, it is wise to tag them
// The tag will keep default value entries from stepping on each other
String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField);
Map<String, String> fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
Map<String, String> fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP);
if (fieldMap == null) {
fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
//TODO Remove in 8.x
if (fieldMap != null) {
LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
}
}
InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
return new InferenceProcessor(client,
@ -244,7 +253,7 @@ public class InferenceProcessor extends AbstractProcessor {
targetField,
modelId,
inferenceConfig,
fieldMapping);
fieldMap);
}
// Package private for testing

2
x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
View File

@ -564,7 +564,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
" \"target_field\": \"regression_value\",\n" +
" \"model_id\": \"modelprocessorlicensetest\",\n" +
" \"inference_config\": {\"regression\": {}},\n" +
" \"field_mappings\": {}\n" +
" \"field_map\": {}\n" +
" }\n" +
" }]}\n";
// Creating a pipeline should work

2
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
View File

@ -269,7 +269,7 @@ public class TransportGetTrainedModelsStatsActionTests extends ESTestCase {
new HashMap<String, Object>() {{
put(InferenceProcessor.MODEL_ID, modelId);
put("inference_config", Collections.singletonMap("regression", Collections.emptyMap()));
put("field_mappings", Collections.emptyMap());
put("field_map", Collections.emptyMap());
put("target_field", randomAlphaOfLength(10));
}}))))) {
return new PipelineConfiguration("pipeline_with_model_" + modelId + num,

18
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
View File

@ -133,7 +133,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
ingestService);
Map<String, Object> config = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap()));
@ -145,7 +145,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]"));
Map<String, Object> config2 = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom"));
@ -156,7 +156,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
equalTo("inference_config must be an object with one inference type mapped to an object."));
Map<String, Object> config3 = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap());
@ -175,7 +175,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1"));
Map<String, Object> regression = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG,
@ -193,7 +193,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
}
Map<String, Object> classification = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@ -218,7 +218,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
ingestService);
Map<String, Object> regression = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG,
@ -232,7 +232,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
}
Map<String, Object> classification = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@ -253,7 +253,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
ingestService);
Map<String, Object> regression = new HashMap<String, Object>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "ml");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(),
@ -307,7 +307,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
put(InferenceProcessor.INFERENCE_CONFIG,
Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap()));
put(InferenceProcessor.TARGET_FIELD, "new_field");
put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest"));
put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
}}))))) {
return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON);
}

2
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
View File

@ -313,7 +313,7 @@ setup:
"model_id" : "a-regression-model-0",
"inference_config": {"regression": {}},
"target_field": "regression_field",
"field_mappings": {}
"field_map": {}
}
}
]

114
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml Normal file
View File

@ -0,0 +1,114 @@
# These tests are not part of the ml-with-security tests
# These are more like ingest processor tests than actual ML focused tests.
setup:
- skip:
features: headers
- do:
headers:
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
ml.put_trained_model:
model_id: a-perfect-regression-model
body: >
{
"description": "empty model for tests",
"tags": ["regression", "tag1"],
"input": {"field_names": ["field1", "field2"]},
"definition": {
"preprocessors": [],
"trained_model": {
"tree": {
"feature_names": ["field1", "field2"],
"tree_structure": [
{"node_index": 0, "leaf_value": 42}
],
"target_type": "regression"
}
}
}
}
---
"Test create and delete pipeline with inference processor":
- do:
ingest.put_pipeline:
id: "regression-model-pipeline"
body: >
{
"processors": [
{
"inference" : {
"model_id" : "a-perfect-regression-model",
"inference_config": {"regression": {}},
"target_field": "regression_field",
"field_map": {}
}
}
]
}
- match: { acknowledged: true }
- do:
ingest.delete_pipeline:
id: "regression-model-pipeline"
---
"Test create processor with missing mandatory fields":
- do:
catch: /\[model_id\] required property is missing/
ingest.put_pipeline:
id: "regression-model-pipeline"
body: >
{
"processors": [
{
"inference" : {
"inference_config": {"regression": {}},
"target_field": "regression_field",
"field_map": {}
}
}
]
}
- do:
catch: /\[inference_config\] required property is missing/
ingest.put_pipeline:
id: "regression-model-pipeline"
body: >
{
"processors": [
{
"inference" : {
"model_id" : "a-perfect-regression-model",
"target_field": "regression_field",
"field_map": {}
}
}
]
}
---
"Test create processor with deprecated fields":
- skip:
features:
- "warnings"
- "allowed_warnings"
- do:
warnings:
- 'Deprecated field [field_mappings] used, expected [field_map] instead'
ingest.put_pipeline:
id: "regression-model-pipeline"
body: >
{
"processors": [
{
"inference" : {
"model_id" : "a-perfect-regression-model",
"inference_config": {"regression": {}},
"field_mappings": {}
}
}
]
}
- do:
allowed_warnings:
- 'Deprecated field [field_mappings] used, expected [field_map] instead'
ingest.delete_pipeline:
id: "regression-model-pipeline"

4
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
View File

@ -66,7 +66,7 @@ setup:
"model_id" : "a-used-regression-model",
"inference_config": {"regression": {}},
"target_field": "regression_field",
"field_mappings": {}
"field_map": {}
}
}
]
@ -84,7 +84,7 @@ setup:
"model_id" : "a-used-regression-model",
"inference_config": {"regression": {}},
"target_field": "regression_field",
"field_mappings": {}
"field_map": {}
}
}
]