honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-24 17:09:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ML] cleanup + adding description field to transforms (#41554) (#41605)

Browse Source 
* [ML] cleanup + adding description field to transforms

* making description length have a max of 1k
This commit is contained in:
Benjamin Trent 2019-04-26 16:50:59 -05:00 committed by GitHub
parent 858e7f4a62
commit a0990ca239
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 290 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.client.dataframe.transforms;
import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@ -38,6 +39,7 @@ public class DataFrameTransformConfig implements ToXContentObject {
public static final ParseField ID = new ParseField("id");
public static final ParseField SOURCE = new ParseField("source");
public static final ParseField DEST = new ParseField("dest");
public static final ParseField DESCRIPTION = new ParseField("description");
// types of transforms
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
@ -45,6 +47,7 @@ public class DataFrameTransformConfig implements ToXContentObject {
private final SourceConfig source;
private final DestConfig dest;
private final PivotConfig pivotConfig;
private final String description;
public static final ConstructingObjectParser<DataFrameTransformConfig, Void> PARSER =
new ConstructingObjectParser<>("data_frame_transform", true,
@ -53,7 +56,8 @@ public class DataFrameTransformConfig implements ToXContentObject {
SourceConfig source = (SourceConfig) args[1];
DestConfig dest = (DestConfig) args[2];
PivotConfig pivotConfig = (PivotConfig) args[3];
return new DataFrameTransformConfig(id, source, dest, pivotConfig);
String description = (String)args[4];
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
});
static {
@ -61,21 +65,38 @@ public class DataFrameTransformConfig implements ToXContentObject {
PARSER.declareObject(constructorArg(), (p, c) -> SourceConfig.PARSER.apply(p, null), SOURCE);
PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST);
PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
}
public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
return PARSER.apply(parser, null);
}
/**
* Helper method for previewing a data frame transform configuration
*
* The DataFrameTransformConfig returned from this method should only be used for previewing the resulting data.
*
* A new, valid, DataFrameTransformConfig with an appropriate destination and ID will have to be constructed to create
* the transform.
* @param source Source configuration for gathering the data
* @param pivotConfig Pivot config to preview
* @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
*/
public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
return new DataFrameTransformConfig(null, source, null, pivotConfig, null);
}
public DataFrameTransformConfig(final String id,
final SourceConfig source,
final DestConfig dest,
final PivotConfig pivotConfig) {
final PivotConfig pivotConfig,
final String description) {
this.id = id;
this.source = source;
this.dest = dest;
this.pivotConfig = pivotConfig;
this.description = description;
}
public String getId() {
@ -94,6 +115,11 @@ public class DataFrameTransformConfig implements ToXContentObject {
return pivotConfig;
}
@Nullable
public String getDescription() {
return description;
}
@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
builder.startObject();
@ -109,6 +135,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
if (pivotConfig != null) {
builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
}
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
builder.endObject();
return builder;
}
@ -128,12 +157,13 @@ public class DataFrameTransformConfig implements ToXContentObject {
return Objects.equals(this.id, that.id)
&& Objects.equals(this.source, that.source)
&& Objects.equals(this.dest, that.dest)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.pivotConfig, that.pivotConfig);
}
@Override
public int hashCode() {
return Objects.hash(id, source, dest, pivotConfig);
return Objects.hash(id, source, dest, pivotConfig, description);
}
@Override

8
client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java
View File

@ -312,7 +312,8 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase {
return new DataFrameTransformConfig(id,
new SourceConfig(new String[]{source}, queryConfig),
destConfig,
pivotConfig);
pivotConfig,
"this is a test transform");
}
public void testGetStats() throws Exception {
@ -329,7 +330,10 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase {
String id = "test-get-stats";
DataFrameTransformConfig transform = new DataFrameTransformConfig(id,
new SourceConfig(new String[]{sourceIndex}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
new SourceConfig(new String[]{sourceIndex}, queryConfig),
new DestConfig("pivot-dest"),
pivotConfig,
"transform for testing stats");
DataFrameClient client = highLevelClient().dataFrame();
AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,

5
client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java
View File

@ -65,13 +65,12 @@ public class PreviewDataFrameTransformRequestTests extends AbstractXContentTestC
containsString("preview requires a non-null data frame config"));
// null id and destination is valid
DataFrameTransformConfig config = new DataFrameTransformConfig(null, randomSourceConfig(), null,
PivotConfigTests.randomPivotConfig());
DataFrameTransformConfig config = DataFrameTransformConfig.forPreview(randomSourceConfig(), PivotConfigTests.randomPivotConfig());
assertFalse(new PreviewDataFrameTransformRequest(config).validate().isPresent());
// null source is not valid
config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig());
config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig(), null);
Optional<ValidationException> error = new PreviewDataFrameTransformRequest(config).validate();
assertTrue(error.isPresent());

2
client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java
View File

@ -40,7 +40,7 @@ public class PutDataFrameTransformRequestTests extends AbstractXContentTestCase<
public void testValidate() {
assertFalse(createTestInstance().validate().isPresent());
DataFrameTransformConfig config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig());
DataFrameTransformConfig config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig(), null);
Optional<ValidationException> error = new PutDataFrameTransformRequest(config).validate();
assertTrue(error.isPresent());

2
client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java
View File

@ -37,7 +37,7 @@ public class DataFrameTransformConfigTests extends AbstractXContentTestCase<Data
public static DataFrameTransformConfig randomDataFrameTransformConfig() {
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
randomDestConfig(), PivotConfigTests.randomPivotConfig());
randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
}
@Override

23
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java
View File

@ -141,7 +141,8 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
new DataFrameTransformConfig("reviewer-avg-rating", // <1>
sourceConfig, // <2>
new DestConfig("pivot-destination"), // <3>
pivotConfig); // <4>
pivotConfig, // <4>
"This is my test transform"); // <5>
// end::put-data-frame-transform-config
{
@ -161,7 +162,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
{
DataFrameTransformConfig configWithDifferentId = new DataFrameTransformConfig("reviewer-avg-rating2",
transformConfig.getSource(), transformConfig.getDestination(),
transformConfig.getPivotConfig());
transformConfig.getPivotConfig(), null);
PutDataFrameTransformRequest request = new PutDataFrameTransformRequest(configWithDifferentId);
// tag::put-data-frame-transform-execute-listener
@ -205,7 +206,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
DataFrameTransformConfig transformConfig = new DataFrameTransformConfig("mega-transform",
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig, null);
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
transformsToClean.add(transformConfig.getId());
@ -320,9 +321,9 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
DataFrameTransformConfig transformConfig1 = new DataFrameTransformConfig("mega-transform",
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig, null);
DataFrameTransformConfig transformConfig2 = new DataFrameTransformConfig("mega-transform2",
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest2"), pivotConfig);
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest2"), pivotConfig, null);
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig1), RequestOptions.DEFAULT);
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig2), RequestOptions.DEFAULT);
@ -386,11 +387,9 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
// tag::preview-data-frame-transform-request
DataFrameTransformConfig transformConfig =
new DataFrameTransformConfig(null, // <1>
new SourceConfig(new String[]{"source-data"}, queryConfig),
null, // <2>
pivotConfig);
DataFrameTransformConfig.forPreview(
new SourceConfig(new String[]{"source-data"}, queryConfig), // <1>
pivotConfig); // <2>
PreviewDataFrameTransformRequest request =
new PreviewDataFrameTransformRequest(transformConfig); // <3>
// end::preview-data-frame-transform-request
@ -447,7 +446,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
String id = "statisitcal-transform";
DataFrameTransformConfig transformConfig = new DataFrameTransformConfig(id,
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("dest"), pivotConfig);
new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("dest"), pivotConfig, null);
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
// tag::get-data-frame-transform-stats-request
@ -526,7 +525,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
DataFrameTransformConfig putTransformConfig = new DataFrameTransformConfig("mega-transform",
new SourceConfig(new String[]{"source-data"}, queryConfig),
new DestConfig("pivot-dest"), pivotConfig);
new DestConfig("pivot-dest"), pivotConfig, null);
RestHighLevelClient client = highLevelClient();
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(putTransformConfig), RequestOptions.DEFAULT);

4
docs/java-rest/high-level/dataframe/preview_data_frame.asciidoc
View File

@ -20,8 +20,8 @@ A +{request}+ takes a single argument: a valid data frame transform config.
--------------------------------------------------
include-tagged::{doc-tests-file}[{api}-request]
--------------------------------------------------
<1> The transform Id may be null for the preview
<2> The destination may be null for the preview
<1> The source config from which the data should be gathered
<2> The pivot config used to transform the data
<3> The configuration of the {dataframe-job} to preview
include::../execution.asciidoc[]

1
docs/java-rest/high-level/dataframe/put_data_frame.asciidoc
View File

@ -35,6 +35,7 @@ include-tagged::{doc-tests-file}[{api}-config]
<2> The source indices and query from which to gather data
<3> The destination index
<4> The PivotConfig
<5> Optional free text description of the transform
[id="{upid}-{api}-query-config"]

5
docs/reference/data-frames/apis/put-transform.asciidoc
View File

@ -33,6 +33,8 @@ a `query`.
`pivot`:: Defines the pivot function `group by` fields and the aggregation to
reduce the data.
`description`:: Optional free text description of the data frame transform
//==== Authorization
@ -73,7 +75,8 @@ PUT _data_frame/transforms/ecommerce_transform
}
}
}
}
},
"description": "Maximum priced ecommerce data by customer_id in Asia"
}
--------------------------------------------------
// CONSOLE

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java
View File

@ -18,7 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Collections;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java
View File

@ -16,7 +16,7 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Collections;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java
View File

@ -17,7 +17,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Collections;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java
View File

@ -18,7 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Arrays;

15
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java
View File

@ -12,7 +12,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
import org.elasticsearch.xpack.core.common.notifications.Level;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils;
import java.util.Date;
@ -36,15 +36,10 @@ public class DataFrameAuditMessage extends AbstractAuditMessage {
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, LEVEL, ObjectParser.ValueType.STRING);
PARSER.declareField(constructorArg(), parser -> {
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
return new Date(parser.longValue());
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
return new Date(TimeUtils.dateStringToEpoch(parser.text()));
}
throw new IllegalArgumentException(
"unexpected token [" + parser.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]");
}, TIMESTAMP, ObjectParser.ValueType.VALUE);
PARSER.declareField(constructorArg(),
p -> TimeUtils.parseTimeField(p, TIMESTAMP.getPreferredName()),
TIMESTAMP,
ObjectParser.ValueType.VALUE);
PARSER.declareString(optionalConstructorArg(), NODE_NAME);
}

33
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java
View File

@ -20,7 +20,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Collections;
@ -41,13 +41,15 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
// types of transforms
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
public static final ParseField DESCRIPTION = new ParseField("description");
private static final ConstructingObjectParser<DataFrameTransformConfig, String> STRICT_PARSER = createParser(false);
private static final ConstructingObjectParser<DataFrameTransformConfig, String> LENIENT_PARSER = createParser(true);
private static final int MAX_DESCRIPTION_LENGTH = 1_000;
private final String id;
private final SourceConfig source;
private final DestConfig dest;
private final String description;
// headers store the user context from the creating user, which allows us to run the transform as this user
// the header only contains name, groups and other context but no authorization keys
private Map<String, String> headers;
@ -81,7 +83,8 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
Map<String, String> headers = (Map<String, String>) args[4];
PivotConfig pivotConfig = (PivotConfig) args[5];
return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig);
String description = (String)args[6];
return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig, description);
});
parser.declareString(optionalConstructorArg(), DataFrameField.ID);
@ -91,6 +94,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
parser.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE);
parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS);
parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM);
parser.declareString(optionalConstructorArg(), DESCRIPTION);
return parser;
}
@ -103,17 +107,22 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
final SourceConfig source,
final DestConfig dest,
final Map<String, String> headers,
final PivotConfig pivotConfig) {
final PivotConfig pivotConfig,
final String description) {
this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName());
this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName());
this.setHeaders(headers == null ? Collections.emptyMap() : headers);
this.pivotConfig = pivotConfig;
this.description = description;
// at least one function must be defined
if (this.pivotConfig == null) {
throw new IllegalArgumentException(DataFrameMessages.DATA_FRAME_TRANSFORM_CONFIGURATION_NO_TRANSFORM);
}
if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) {
throw new IllegalArgumentException("[description] must be less than 1000 characters in length.");
}
}
public DataFrameTransformConfig(final StreamInput in) throws IOException {
@ -122,6 +131,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
dest = new DestConfig(in);
setHeaders(in.readMap(StreamInput::readString, StreamInput::readString));
pivotConfig = in.readOptionalWriteable(PivotConfig::new);
description = in.readOptionalString();
}
public String getId() {
@ -148,6 +158,11 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
return pivotConfig;
}
@Nullable
public String getDescription() {
return description;
}
public boolean isValid() {
if (pivotConfig != null && pivotConfig.isValid() == false) {
return false;
@ -163,6 +178,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
dest.writeTo(out);
out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
out.writeOptionalWriteable(pivotConfig);
out.writeOptionalString(description);
}
@Override
@ -180,7 +196,9 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
if (headers.isEmpty() == false && params.paramAsBoolean(DataFrameField.FOR_INTERNAL_STORAGE, false) == true) {
builder.field(HEADERS.getPreferredName(), headers);
}
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
builder.endObject();
return builder;
}
@ -201,12 +219,13 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
&& Objects.equals(this.source, that.source)
&& Objects.equals(this.dest, that.dest)
&& Objects.equals(this.headers, that.headers)
&& Objects.equals(this.pivotConfig, that.pivotConfig);
&& Objects.equals(this.pivotConfig, that.pivotConfig)
&& Objects.equals(this.description, that.description);
}
@Override
public int hashCode(){
return Objects.hash(id, source, dest, headers, pivotConfig);
return Objects.hash(id, source, dest, headers, pivotConfig, description);
}
@Override

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfig.java
View File

@ -14,7 +14,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Objects;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SourceConfig.java
View File

@ -15,7 +15,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Arrays;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/GroupConfig.java
View File

@ -22,7 +22,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.LinkedHashMap;

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/PivotConfig.java
View File

@ -15,7 +15,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Map.Entry;

129
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.dataframe.utils;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.DateFieldMapper;
import java.io.IOException;
import java.util.Date;
import java.util.concurrent.TimeUnit;
public final class TimeUtils {
private TimeUtils() {
// Do nothing
}
public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
return new Date(parser.longValue());
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
return new Date(TimeUtils.dateStringToEpoch(parser.text()));
}
throw new IllegalArgumentException(
"unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
}
/**
* First tries to parse the date first as a Long and convert that to an
* epoch time. If the long number has more than 10 digits it is considered a
* time in milliseconds else if 10 or less digits it is in seconds. If that
* fails it tries to parse the string using
* {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER}
*
* If the date string cannot be parsed -1 is returned.
*
* @return The epoch time in milliseconds or -1 if the date cannot be
* parsed.
*/
public static long dateStringToEpoch(String date) {
try {
long epoch = Long.parseLong(date);
if (date.trim().length() <= 10) { // seconds
return epoch * 1000;
} else {
return epoch;
}
} catch (NumberFormatException nfe) {
// not a number
}
try {
return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(date);
} catch (ElasticsearchParseException | IllegalArgumentException e) {
}
// Could not do the conversion
return -1;
}
/**
* Checks that the given {@code timeValue} is a non-negative multiple value of the {@code baseUnit}.
*
* <ul>
* <li>400ms is valid for base unit of seconds</li>
* <li>450ms is invalid for base unit of seconds but valid for base unit of milliseconds</li>
* </ul>
*/
public static void checkNonNegativeMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
checkNonNegative(timeValue, field);
checkMultiple(timeValue, baseUnit, field);
}
/**
* Checks that the given {@code timeValue} is a positive multiple value of the {@code baseUnit}.
*
* <ul>
* <li>400ms is valid for base unit of seconds</li>
* <li>450ms is invalid for base unit of seconds but valid for base unit of milliseconds</li>
* </ul>
*/
public static void checkPositiveMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
checkPositive(timeValue, field);
checkMultiple(timeValue, baseUnit, field);
}
/**
* Checks that the given {@code timeValue} is positive.
*
* <ul>
* <li>1s is valid</li>
* <li>-1s is invalid</li>
* </ul>
*/
public static void checkPositive(TimeValue timeValue, ParseField field) {
long nanos = timeValue.getNanos();
if (nanos <= 0) {
throw new IllegalArgumentException(field.getPreferredName() + " cannot be less or equal than 0. Value = "
+ timeValue.toString());
}
}
private static void checkNonNegative(TimeValue timeValue, ParseField field) {
long nanos = timeValue.getNanos();
if (nanos < 0) {
throw new IllegalArgumentException(field.getPreferredName() + " cannot be less than 0. Value = " + timeValue.toString());
}
}
/**
* Check the given {@code timeValue} is a multiple of the {@code baseUnit}
*/
public static void checkMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
long nanos = timeValue.getNanos();
TimeValue base = new TimeValue(1, baseUnit);
long baseNanos = base.getNanos();
if (nanos % baseNanos != 0) {
throw new IllegalArgumentException(field.getPreferredName() + " has to be a multiple of " + base.toString() + "; actual was '"
+ timeValue.toString() + "'");
}
}
}

2
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java
View File

@ -68,7 +68,7 @@ public class PreviewDataFrameTransformActionRequestTests extends AbstractSeriali
@Override
protected Request createTestInstance() {
DataFrameTransformConfig config = new DataFrameTransformConfig("transform-preview", randomSourceConfig(),
new DestConfig("unused-transform-preview-index"), null, PivotConfigTests.randomPivotConfig());
new DestConfig("unused-transform-preview-index"), null, PivotConfigTests.randomPivotConfig(), null);
return new Request(config);
}

22
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java
View File

@ -25,6 +25,7 @@ import static org.elasticsearch.test.TestMatchers.matchesPattern;
import static org.elasticsearch.xpack.core.dataframe.transforms.DestConfigTests.randomDestConfig;
import static org.elasticsearch.xpack.core.dataframe.transforms.SourceConfigTests.randomInvalidSourceConfig;
import static org.elasticsearch.xpack.core.dataframe.transforms.SourceConfigTests.randomSourceConfig;
import static org.hamcrest.Matchers.equalTo;
public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameTestCase<DataFrameTransformConfig> {
@ -41,21 +42,23 @@ public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameT
public static DataFrameTransformConfig randomDataFrameTransformConfigWithoutHeaders(String id) {
return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), null,
PivotConfigTests.randomPivotConfig());
PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
}
public static DataFrameTransformConfig randomDataFrameTransformConfig(String id) {
return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomHeaders(),
PivotConfigTests.randomPivotConfig());
PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
}
public static DataFrameTransformConfig randomInvalidDataFrameTransformConfig() {
if (randomBoolean()) {
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(),
randomDestConfig(), randomHeaders(), PivotConfigTests.randomPivotConfig());
randomDestConfig(), randomHeaders(), PivotConfigTests.randomPivotConfig(),
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
} // else
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
randomDestConfig(), randomHeaders(), PivotConfigTests.randomInvalidPivotConfig());
randomDestConfig(), randomHeaders(), PivotConfigTests.randomInvalidPivotConfig(),
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
}
@Before
@ -162,6 +165,16 @@ public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameT
}
}
public void testMaxLengthDescription() {
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new DataFrameTransformConfig("id",
randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), randomAlphaOfLength(1001)));
assertThat(exception.getMessage(), equalTo("[description] must be less than 1000 characters in length."));
String description = randomAlphaOfLength(1000);
DataFrameTransformConfig config = new DataFrameTransformConfig("id",
randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), description);
assertThat(description, equalTo(config.getDescription()));
}
public void testSetIdInBody() throws IOException {
String pivotTransform = "{"
+ " \"id\" : \"body_id\","
@ -189,6 +202,7 @@ public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameT
ex.getCause().getMessage());
}
private DataFrameTransformConfig createDataFrameTransformConfigFromString(String json, String id) throws IOException {
final XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(),
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);

3
x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java
View File

@ -193,7 +193,8 @@ abstract class DataFrameIntegTestCase extends ESIntegTestCase {
new SourceConfig(sourceIndices, createQueryConfig(queryBuilder)),
new DestConfig(destinationIndex),
Collections.emptyMap(),
createPivotConfig(groups, aggregations));
createPivotConfig(groups, aggregations),
"Test data frame transform config id: " + id);
}
protected void createReviewsIndex() throws Exception {

1
x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java
View File

@ -57,6 +57,7 @@ public class DataFrameGetAndGetStatsIT extends DataFrameRestTestCase {
wipeDataFrameTransforms();
}
@SuppressWarnings("unchecked")
public void testGetAndGetStats() throws Exception {
createPivotReviewsTransform("pivot_1", "pivot_reviews_1", null);
createPivotReviewsTransform("pivot_2", "pivot_reviews_2", null);

6
x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java
View File

@ -135,7 +135,8 @@ public class DataFrameTransformProgressIT extends ESIntegTestCase {
sourceConfig,
destConfig,
null,
pivotConfig);
pivotConfig,
null);
PlainActionFuture<DataFrameTransformProgress> progressFuture = new PlainActionFuture<>();
TransformProgressGatherer.getInitialProgress(client(), config, progressFuture);
@ -154,7 +155,8 @@ public class DataFrameTransformProgressIT extends ESIntegTestCase {
sourceConfig,
destConfig,
null,
pivotConfig);
pivotConfig,
null);
progressFuture = new PlainActionFuture<>();

4
x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java
View File

@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig;
import org.elasticsearch.xpack.core.dataframe.transforms.SourceConfig;
@ -195,6 +196,9 @@ public final class DataFrameInternalIndex {
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.startObject(DataFrameTransformConfig.DESCRIPTION.getPreferredName())
.field(TYPE, TEXT)
.endObject();
}

31
x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformActionTests.java
View File

@ -6,16 +6,19 @@
package org.elasticsearch.xpack.dataframe.action;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.persistent.PersistentTaskParams;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.dataframe.DataFrameField;
import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
@ -34,8 +37,28 @@ public class TransportStopDataFrameTransformActionTests extends ESTestCase {
tasksBuilder.addTask(dataFrameIdBar,
DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdBar),
new PersistentTasksCustomMetaData.Assignment("node-2", "test assignment"));
tasksBuilder.addTask(MlTasks.jobTaskId("foo-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"),
new PersistentTasksCustomMetaData.Assignment("node-3", "test assignment"));
tasksBuilder.addTask("test-task1", "testTasks", new PersistentTaskParams() {
@Override
public String getWriteableName() {
return "testTasks";
}
@Override
public Version getMinimalSupportedVersion() {
return null;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return null;
}
},
new PersistentTasksCustomMetaData.Assignment("node-3", "test assignment"));
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()))

5
x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml
View File

@ -63,7 +63,8 @@ setup:
"pivot": {
"group_by": { "airline": {"terms": {"field": "airline"}}},
"aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
}
},
"description": "yaml test transform on airline-data"
}
- match: { acknowledged: true }
@ -91,6 +92,7 @@ setup:
- is_true: transforms.0.source.query.match_all
- match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
- match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
- match: { transforms.0.description: "yaml test transform on airline-data" }
- do:
data_frame.get_data_frame_transform:
@ -98,6 +100,7 @@ setup:
- match: { count: 2 }
- match: { transforms.0.id: "airline-transform" }
- match: { transforms.1.id: "airline-transform-dos" }
- is_false: transforms.1.description
- do:
data_frame.get_data_frame_transform: