honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Add description to DF analytics (#45774) (#46019)

This commit is contained in:
Dimitris Athanasiou 2019-08-27 15:48:59 +03:00 committed by GitHub
parent 267183998e
commit dd6c13fdf9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 110 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java
View File

@ -48,6 +48,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
}
private static final ParseField ID = new ParseField("id");
private static final ParseField DESCRIPTION = new ParseField("description");
private static final ParseField SOURCE = new ParseField("source");
private static final ParseField DEST = new ParseField("dest");
private static final ParseField ANALYSIS = new ParseField("analysis");
@ -60,6 +61,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
static {
PARSER.declareString(Builder::setId, ID);
PARSER.declareString(Builder::setDescription, DESCRIPTION);
PARSER.declareObject(Builder::setSource, (p, c) -> DataFrameAnalyticsSource.fromXContent(p), SOURCE);
PARSER.declareObject(Builder::setDest, (p, c) -> DataFrameAnalyticsDest.fromXContent(p), DEST);
PARSER.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p), ANALYSIS);
@ -95,6 +97,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
}
private final String id;
private final String description;
private final DataFrameAnalyticsSource source;
private final DataFrameAnalyticsDest dest;
private final DataFrameAnalysis analysis;
@ -103,10 +106,12 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
private final Instant createTime;
private final Version version;
private DataFrameAnalyticsConfig(@Nullable String id, @Nullable DataFrameAnalyticsSource source, @Nullable DataFrameAnalyticsDest dest,
@Nullable DataFrameAnalysis analysis, @Nullable FetchSourceContext analyzedFields,
@Nullable ByteSizeValue modelMemoryLimit, @Nullable Instant createTime, @Nullable Version version) {
private DataFrameAnalyticsConfig(@Nullable String id, @Nullable String description, @Nullable DataFrameAnalyticsSource source,
@Nullable DataFrameAnalyticsDest dest, @Nullable DataFrameAnalysis analysis,
@Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit,
@Nullable Instant createTime, @Nullable Version version) {
this.id = id;
this.description = description;
this.source = source;
this.dest = dest;
this.analysis = analysis;
@ -120,6 +125,10 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
return id;
}
public String getDescription() {
return description;
}
public DataFrameAnalyticsSource getSource() {
return source;
}
@ -154,6 +163,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
if (id != null) {
builder.field(ID.getPreferredName(), id);
}
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
if (source != null) {
builder.field(SOURCE.getPreferredName(), source);
}
@ -189,6 +201,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
return Objects.equals(id, other.id)
&& Objects.equals(description, other.description)
&& Objects.equals(source, other.source)
&& Objects.equals(dest, other.dest)
&& Objects.equals(analysis, other.analysis)
@ -200,7 +213,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
@Override
public int hashCode() {
return Objects.hash(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
return Objects.hash(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
}
@Override
@ -211,6 +224,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
public static class Builder {
private String id;
private String description;
private DataFrameAnalyticsSource source;
private DataFrameAnalyticsDest dest;
private DataFrameAnalysis analysis;
@ -226,6 +240,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
return this;
}
public Builder setDescription(String description) {
this.description = description;
return this;
}
public Builder setSource(DataFrameAnalyticsSource source) {
this.source = Objects.requireNonNull(source);
return this;
@ -262,7 +281,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
}
public DataFrameAnalyticsConfig build() {
return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime,
version);
}
}
}

2
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
View File

@ -1257,6 +1257,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
.setIndex("put-test-dest-index")
.build())
.setAnalysis(OutlierDetection.createDefault())
.setDescription("some description")
.build();
createIndex("put-test-source-index", defaultMappingForTest());
@ -1273,6 +1274,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
assertThat(createdConfig.getAnalysis(), equalTo(config.getAnalysis()));
assertThat(createdConfig.getAnalyzedFields(), equalTo(config.getAnalyzedFields()));
assertThat(createdConfig.getModelMemoryLimit(), equalTo(ByteSizeValue.parseBytesSizeValue("1gb", ""))); // default value
assertThat(createdConfig.getDescription(), equalTo("some description"));
}
public void testGetDataFrameAnalyticsConfig_SingleConfig() throws Exception {

1
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
View File

@ -2950,6 +2950,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
.setAnalysis(outlierDetection) // <4>
.setAnalyzedFields(analyzedFields) // <5>
.setModelMemoryLimit(new ByteSizeValue(5, ByteSizeUnit.MB)) // <6>
.setDescription("this is an example description") // <7>
.build();
// end::put-data-frame-analytics-config

3
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java
View File

@ -49,6 +49,9 @@ public class DataFrameAnalyticsConfigTests extends AbstractXContentTestCase<Data
.setSource(randomSourceConfig())
.setDest(randomDestConfig())
.setAnalysis(randomOutlierDetection());
if (randomBoolean()) {
builder.setDescription(randomAlphaOfLength(20));
}
if (randomBoolean()) {
builder.setAnalyzedFields(new FetchSourceContext(true,
generateRandomStringArray(10, 10, false, false),

1
docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
View File

@ -36,6 +36,7 @@ include-tagged::{doc-tests-file}[{api}-config]
<4> The analysis to be performed
<5> The fields to be included in / excluded from the analysis
<6> The memory limit for the model created as part of the analysis process
<7> Optionally, a human-readable description
[id="{upid}-{api}-query-config"]

3
docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc
View File

@ -42,6 +42,9 @@ PUT _ml/data_frame/analytics/loganalytics
// CONSOLE
// TEST[setup:setup_logdata]
`description`::
(Optional, string) A description of the job.
`dest`::
(object) The destination configuration of the analysis. The `index` property
(string) is the name of the index in which to store the results of the

5
docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
View File

@ -66,6 +66,9 @@ and mappings.
`analyzed_fields` is not set, only the relevant fields will be included. For
example, all the numeric fields for {oldetection}.
`description`::
(Optional, string) A description of the job.
`dest`::
(Required, object) The destination configuration, consisting of `index` and
optionally `results_field` (`ml` by default). See
@ -94,6 +97,7 @@ type is `outlier_detection`:
--------------------------------------------------
PUT _ml/data_frame/analytics/loganalytics
{
"description": "Outlier detection on log data",
"source": {
"index": "logdata"
},
@ -115,6 +119,7 @@ The API returns the following result:
----
{
"id" : "loganalytics",
"description": "Outlier detection on log data",
"source" : {
"index" : [
"logdata"

35
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
View File

@ -44,6 +44,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(20, ByteSizeUnit.MB);
public static final ParseField ID = new ParseField("id");
public static final ParseField DESCRIPTION = new ParseField("description");
public static final ParseField SOURCE = new ParseField("source");
public static final ParseField DEST = new ParseField("dest");
public static final ParseField ANALYSIS = new ParseField("analysis");
@ -62,6 +63,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
parser.declareString((c, s) -> {}, CONFIG_TYPE);
parser.declareString(Builder::setId, ID);
parser.declareString(Builder::setDescription, DESCRIPTION);
parser.declareObject(Builder::setSource, DataFrameAnalyticsSource.createParser(ignoreUnknownFields), SOURCE);
parser.declareObject(Builder::setDest, DataFrameAnalyticsDest.createParser(ignoreUnknownFields), DEST);
parser.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p, ignoreUnknownFields), ANALYSIS);
@ -100,6 +102,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
}
private final String id;
private final String description;
private final DataFrameAnalyticsSource source;
private final DataFrameAnalyticsDest dest;
private final DataFrameAnalysis analysis;
@ -117,10 +120,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
private final Instant createTime;
private final Version version;
public DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest,
public DataFrameAnalyticsConfig(String id, String description, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest,
DataFrameAnalysis analysis, Map<String, String> headers, ByteSizeValue modelMemoryLimit,
FetchSourceContext analyzedFields, Instant createTime, Version version) {
this.id = ExceptionsHelper.requireNonNull(id, ID);
this.description = description;
this.source = ExceptionsHelper.requireNonNull(source, SOURCE);
this.dest = ExceptionsHelper.requireNonNull(dest, DEST);
this.analysis = ExceptionsHelper.requireNonNull(analysis, ANALYSIS);
@ -133,6 +137,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
public DataFrameAnalyticsConfig(StreamInput in) throws IOException {
this.id = in.readString();
if (in.getVersion().onOrAfter(Version.V_7_4_0)) {
description = in.readOptionalString();
} else {
description = null;
}
this.source = new DataFrameAnalyticsSource(in);
this.dest = new DataFrameAnalyticsDest(in);
this.analysis = in.readNamedWriteable(DataFrameAnalysis.class);
@ -152,6 +161,10 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
return id;
}
public String getDescription() {
return description;
}
public DataFrameAnalyticsSource getSource() {
return source;
}
@ -188,6 +201,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(ID.getPreferredName(), id);
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
builder.field(SOURCE.getPreferredName(), source);
builder.field(DEST.getPreferredName(), dest);
@ -218,6 +234,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
out.writeOptionalString(description);
}
source.writeTo(out);
dest.writeTo(out);
out.writeNamedWriteable(analysis);
@ -242,6 +261,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
return Objects.equals(id, other.id)
&& Objects.equals(description, other.description)
&& Objects.equals(source, other.source)
&& Objects.equals(dest, other.dest)
&& Objects.equals(analysis, other.analysis)
@ -254,7 +274,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
@Override
public int hashCode() {
return Objects.hash(id, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version);
return Objects.hash(id, description, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version);
}
@Override
@ -269,6 +289,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
public static class Builder {
private String id;
private String description;
private DataFrameAnalyticsSource source;
private DataFrameAnalyticsDest dest;
private DataFrameAnalysis analysis;
@ -287,6 +308,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
public Builder(DataFrameAnalyticsConfig config, ByteSizeValue maxModelMemoryLimit) {
this.id = config.id;
this.description = config.description;
this.source = new DataFrameAnalyticsSource(config.source);
this.dest = new DataFrameAnalyticsDest(config.dest);
this.analysis = config.analysis;
@ -304,6 +326,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
return id;
}
public Builder setDescription(String description) {
this.description = description;
return this;
}
public Builder setId(String id) {
this.id = ExceptionsHelper.requireNonNull(id, ID);
return this;
@ -354,7 +381,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
*/
public DataFrameAnalyticsConfig build() {
applyMaxModelMemoryLimit();
return new DataFrameAnalyticsConfig(id, source, dest, analysis, headers, modelMemoryLimit, analyzedFields, createTime, version);
return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, headers, modelMemoryLimit, analyzedFields,
createTime, version);
}
/**
@ -365,6 +393,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
public DataFrameAnalyticsConfig buildForMemoryEstimation() {
return new DataFrameAnalyticsConfig(
id != null ? id : "dummy",
description,
source,
dest != null ? dest : new DataFrameAnalyticsDest("dummy", null),
analysis,

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
View File

@ -286,6 +286,7 @@ public final class ReservedFieldNames {
ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(),
DataFrameAnalyticsConfig.ID.getPreferredName(),
DataFrameAnalyticsConfig.DESCRIPTION.getPreferredName(),
DataFrameAnalyticsConfig.SOURCE.getPreferredName(),
DataFrameAnalyticsConfig.DEST.getPreferredName(),
DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(),

3
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
View File

@ -113,6 +113,9 @@ public class DataFrameAnalyticsConfigTests extends AbstractSerializingTestCase<D
if (randomBoolean()) {
builder.setModelMemoryLimit(new ByteSizeValue(randomIntBetween(1, 16), randomFrom(ByteSizeUnit.MB, ByteSizeUnit.GB)));
}
if (randomBoolean()) {
builder.setDescription(randomAlphaOfLength(20));
}
if (withGeneratedFields) {
if (randomBoolean()) {
builder.setCreateTime(Instant.now());

33
x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
View File

@ -1182,3 +1182,36 @@ setup:
}}
- is_true: create_time
- is_true: version
---
"Test put with description":
- do:
ml.put_data_frame_analytics:
id: "with-description"
body: >
{
"description": "This is a described config",
"source": {
"index": "index-source"
},
"dest": {
"index": "index-dest"
},
"analysis": {
"regression": {
"dependent_variable": "foo"
}
}
}
- match: { id: "with-description" }
- match: { description: "This is a described config" }
- match: { source.index: ["index-source"] }
- match: { dest.index: "index-dest" }
- match: { analysis: {
"regression":{
"dependent_variable": "foo"
}
}}
- is_true: create_time
- is_true: version