honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] add detectorIndex to modelplot and forecast (elastic/x-pack-elasticsearch#2796) 

add detector_index to model plots and forecast

relates elastic/x-pack-elasticsearch#2547

corresponding ml-cpp change: elastic/machine-learning-cpp#361

Original commit: elastic/x-pack-elasticsearch@5927d8578e
This commit is contained in:
Hendrik Muhs 2017-10-27 12:54:42 +02:00 committed by GitHub
parent c7e94b3b4c
commit f74e680142
7 changed files with 90 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Forecast.java
View File

@ -34,8 +34,6 @@ public class Forecast implements ToXContentObject, Writeable {
public static final ParseField FORECAST_ID = new ParseField("forecast_id");
public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name");
public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value");
public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name");
public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value");
public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name");
public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value");
public static final ParseField MODEL_FEATURE = new ParseField("model_feature");
@ -43,9 +41,11 @@ public class Forecast implements ToXContentObject, Writeable {
public static final ParseField FORECAST_UPPER = new ParseField("forecast_upper");
public static final ParseField FORECAST_PREDICTION = new ParseField("forecast_prediction");
public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
public static final ConstructingObjectParser<Forecast, Void> PARSER =
new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new Forecast((String) a[0], (long) a[1], (Date) a[2], (long) a[3]));
new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a ->
new Forecast((String) a[0], (long) a[1], (Date) a[2], (long) a[3], (int) a[4]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
@ -60,11 +60,10 @@ public class Forecast implements ToXContentObject, Writeable {
+ Result.TIMESTAMP.getPreferredName() + "]");
}, Result.TIMESTAMP, ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
PARSER.declareInt(ConstructingObjectParser.constructorArg(), DETECTOR_INDEX);
PARSER.declareString((modelForecast, s) -> {}, Result.RESULT_TYPE);
PARSER.declareString(Forecast::setPartitionFieldName, PARTITION_FIELD_NAME);
PARSER.declareString(Forecast::setPartitionFieldValue, PARTITION_FIELD_VALUE);
PARSER.declareString(Forecast::setOverFieldName, OVER_FIELD_NAME);
PARSER.declareString(Forecast::setOverFieldValue, OVER_FIELD_VALUE);
PARSER.declareString(Forecast::setByFieldName, BY_FIELD_NAME);
PARSER.declareString(Forecast::setByFieldValue, BY_FIELD_VALUE);
PARSER.declareString(Forecast::setModelFeature, MODEL_FEATURE);
@ -77,10 +76,9 @@ public class Forecast implements ToXContentObject, Writeable {
private final long forecastId;
private final Date timestamp;
private final long bucketSpan;
private int detectorIndex;
private String partitionFieldName;
private String partitionFieldValue;
private String overFieldName;
private String overFieldValue;
private String byFieldName;
private String byFieldValue;
private String modelFeature;
@ -88,11 +86,12 @@ public class Forecast implements ToXContentObject, Writeable {
private double forecastUpper;
private double forecastPrediction;
public Forecast(String jobId, long forecastId, Date timestamp, long bucketSpan) {
public Forecast(String jobId, long forecastId, Date timestamp, long bucketSpan, int detectorIndex) {
this.jobId = jobId;
this.forecastId = forecastId;
this.timestamp = timestamp;
this.bucketSpan = bucketSpan;
this.detectorIndex = detectorIndex;
}
public Forecast(StreamInput in) throws IOException {
@ -101,8 +100,6 @@ public class Forecast implements ToXContentObject, Writeable {
timestamp = new Date(in.readLong());
partitionFieldName = in.readOptionalString();
partitionFieldValue = in.readOptionalString();
overFieldName = in.readOptionalString();
overFieldValue = in.readOptionalString();
byFieldName = in.readOptionalString();
byFieldValue = in.readOptionalString();
modelFeature = in.readOptionalString();
@ -110,6 +107,7 @@ public class Forecast implements ToXContentObject, Writeable {
forecastUpper = in.readDouble();
forecastPrediction = in.readDouble();
bucketSpan = in.readLong();
detectorIndex = in.readInt();
}
@Override
@ -119,8 +117,6 @@ public class Forecast implements ToXContentObject, Writeable {
out.writeLong(timestamp.getTime());
out.writeOptionalString(partitionFieldName);
out.writeOptionalString(partitionFieldValue);
out.writeOptionalString(overFieldName);
out.writeOptionalString(overFieldValue);
out.writeOptionalString(byFieldName);
out.writeOptionalString(byFieldValue);
out.writeOptionalString(modelFeature);
@ -128,6 +124,7 @@ public class Forecast implements ToXContentObject, Writeable {
out.writeDouble(forecastUpper);
out.writeDouble(forecastPrediction);
out.writeLong(bucketSpan);
out.writeInt(detectorIndex);
}
@Override
@ -137,6 +134,7 @@ public class Forecast implements ToXContentObject, Writeable {
builder.field(FORECAST_ID.getPreferredName(), forecastId);
builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
if (timestamp != null) {
builder.dateField(Result.TIMESTAMP.getPreferredName(),
Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
@ -147,12 +145,6 @@ public class Forecast implements ToXContentObject, Writeable {
if (partitionFieldValue != null) {
builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue);
}
if (overFieldName != null) {
builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName);
}
if (overFieldValue != null) {
builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue);
}
if (byFieldName != null) {
builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName);
}
@ -178,12 +170,12 @@ public class Forecast implements ToXContentObject, Writeable {
}
public String getId() {
int valuesHash = Objects.hash(byFieldValue, overFieldValue, partitionFieldValue);
int valuesHash = Objects.hash(byFieldValue, partitionFieldValue);
int length = (byFieldValue == null ? 0 : byFieldValue.length()) +
(overFieldValue == null ? 0 : overFieldValue.length()) +
(partitionFieldValue == null ? 0 : partitionFieldValue.length());
return jobId + "_model_forecast_" + forecastId + "_" + timestamp.getTime() + "_" + bucketSpan + "_"
+ (modelFeature == null ? "" : modelFeature) + "_" + valuesHash + "_" + length;
return jobId + "_model_forecast_" + forecastId + "_" + timestamp.getTime()
+ "_" + bucketSpan + "_" + detectorIndex + "_"
+ valuesHash + "_" + length;
}
public Date getTimestamp() {
@ -202,6 +194,10 @@ public class Forecast implements ToXContentObject, Writeable {
this.partitionFieldName = partitionFieldName;
}
public int getDetectorIndex() {
return detectorIndex;
}
public String getPartitionFieldValue() {
return partitionFieldValue;
}
@ -210,22 +206,6 @@ public class Forecast implements ToXContentObject, Writeable {
this.partitionFieldValue = partitionFieldValue;
}
public String getOverFieldName() {
return overFieldName;
}
public void setOverFieldName(String overFieldName) {
this.overFieldName = overFieldName;
}
public String getOverFieldValue() {
return overFieldValue;
}
public void setOverFieldValue(String overFieldValue) {
this.overFieldValue = overFieldValue;
}
public String getByFieldName() {
return byFieldName;
}
@ -288,21 +268,20 @@ public class Forecast implements ToXContentObject, Writeable {
Objects.equals(this.timestamp, that.timestamp) &&
Objects.equals(this.partitionFieldValue, that.partitionFieldValue) &&
Objects.equals(this.partitionFieldName, that.partitionFieldName) &&
Objects.equals(this.overFieldValue, that.overFieldValue) &&
Objects.equals(this.overFieldName, that.overFieldName) &&
Objects.equals(this.byFieldValue, that.byFieldValue) &&
Objects.equals(this.byFieldName, that.byFieldName) &&
Objects.equals(this.modelFeature, that.modelFeature) &&
this.forecastLower == that.forecastLower &&
this.forecastUpper == that.forecastUpper &&
this.forecastPrediction == that.forecastPrediction &&
this.bucketSpan == that.bucketSpan;
this.bucketSpan == that.bucketSpan &&
this.detectorIndex == that.detectorIndex;
}
@Override
public int hashCode() {
return Objects.hash(jobId, forecastId, timestamp, partitionFieldName, partitionFieldValue,
overFieldName, overFieldValue, byFieldName, byFieldValue,
modelFeature, forecastLower, forecastUpper, forecastPrediction, bucketSpan);
byFieldName, byFieldValue, modelFeature, forecastLower, forecastUpper,
forecastPrediction, bucketSpan, detectorIndex);
}
}

33
plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ModelPlot.java
View File

@ -44,9 +44,11 @@ public class ModelPlot implements ToXContentObject, Writeable {
public static final ParseField MODEL_MEDIAN = new ParseField("model_median");
public static final ParseField ACTUAL = new ParseField("actual");
public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
public static final ConstructingObjectParser<ModelPlot, Void> PARSER =
new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new ModelPlot((String) a[0], (Date) a[1], (long) a[2]));
new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a ->
new ModelPlot((String) a[0], (Date) a[1], (long) a[2], (int) a[3]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
@ -60,6 +62,7 @@ public class ModelPlot implements ToXContentObject, Writeable {
+ Result.TIMESTAMP.getPreferredName() + "]");
}, Result.TIMESTAMP, ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
PARSER.declareInt(ConstructingObjectParser.constructorArg(), DETECTOR_INDEX);
PARSER.declareString((modelPlot, s) -> {}, Result.RESULT_TYPE);
PARSER.declareString(ModelPlot::setPartitionFieldName, PARTITION_FIELD_NAME);
PARSER.declareString(ModelPlot::setPartitionFieldValue, PARTITION_FIELD_VALUE);
@ -77,6 +80,7 @@ public class ModelPlot implements ToXContentObject, Writeable {
private final String jobId;
private final Date timestamp;
private final long bucketSpan;
private int detectorIndex;
private String partitionFieldName;
private String partitionFieldValue;
private String overFieldName;
@ -92,10 +96,11 @@ public class ModelPlot implements ToXContentObject, Writeable {
*/
private Double actual;
public ModelPlot(String jobId, Date timestamp, long bucketSpan) {
public ModelPlot(String jobId, Date timestamp, long bucketSpan, int detectorIndex) {
this.jobId = jobId;
this.timestamp = timestamp;
this.bucketSpan = bucketSpan;
this.detectorIndex = detectorIndex;
}
public ModelPlot(StreamInput in) throws IOException {
@ -134,6 +139,12 @@ public class ModelPlot implements ToXContentObject, Writeable {
} else {
bucketSpan = 0;
}
if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
detectorIndex = in.readInt();
} else {
// default to -1 as marker for no detector index
detectorIndex = -1;
}
}
@Override
@ -177,6 +188,9 @@ public class ModelPlot implements ToXContentObject, Writeable {
if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
out.writeLong(bucketSpan);
}
if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
out.writeInt(detectorIndex);
}
}
@Override
@ -185,6 +199,8 @@ public class ModelPlot implements ToXContentObject, Writeable {
builder.field(Job.ID.getPreferredName(), jobId);
builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
if (timestamp != null) {
builder.dateField(Result.TIMESTAMP.getPreferredName(),
Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
@ -229,8 +245,8 @@ public class ModelPlot implements ToXContentObject, Writeable {
int length = (byFieldValue == null ? 0 : byFieldValue.length()) +
(overFieldValue == null ? 0 : overFieldValue.length()) +
(partitionFieldValue == null ? 0 : partitionFieldValue.length());
return jobId + "_model_plot_" + timestamp.getTime() + "_" + bucketSpan + "_" +
(modelFeature == null ? "" : modelFeature) + "_" + valuesHash + "_" + length;
return jobId + "_model_plot_" + timestamp.getTime() + "_" + bucketSpan
+ "_" + detectorIndex + "_" + valuesHash + "_" + length;
}
public Date getTimestamp() {
@ -241,6 +257,10 @@ public class ModelPlot implements ToXContentObject, Writeable {
return bucketSpan;
}
public int getDetectorIndex() {
return detectorIndex;
}
public String getPartitionFieldName() {
return partitionFieldName;
}
@ -351,13 +371,14 @@ public class ModelPlot implements ToXContentObject, Writeable {
this.modelUpper == that.modelUpper &&
this.modelMedian == that.modelMedian &&
Objects.equals(this.actual, that.actual) &&
this.bucketSpan == that.bucketSpan;
this.bucketSpan == that.bucketSpan &&
this.detectorIndex == that.detectorIndex;
}
@Override
public int hashCode() {
return Objects.hash(jobId, timestamp, partitionFieldName, partitionFieldValue,
overFieldName, overFieldValue, byFieldName, byFieldValue,
modelFeature, modelLower, modelUpper, modelMedian, actual, bucketSpan);
modelFeature, modelLower, modelUpper, modelMedian, actual, bucketSpan, detectorIndex);
}
}

2
plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java
View File

@ -47,7 +47,7 @@ public class JobRenormalizedResultsPersisterTests extends ESTestCase {
Client client = new MockClientBuilder("cluster").bulk(bulkResponse).build();
JobRenormalizedResultsPersister persister = new JobRenormalizedResultsPersister("foo", Settings.EMPTY, client);
ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456);
ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456, 0);
for (int i=0; i<=JobRenormalizedResultsPersister.BULK_LIMIT; i++) {
persister.updateResult("bar", "index-foo", modelPlot);
}

2
plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
View File

@ -183,7 +183,7 @@ public class JobResultsPersisterTests extends ESTestCase {
JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client);
JobResultsPersister.Builder bulkBuilder = persister.bulkPersisterBuilder("foo");
ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456);
ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456, 0);
for (int i=0; i<=JobRenormalizedResultsPersister.BULK_LIMIT; i++) {
bulkBuilder.persistModelPlot(modelPlot);
}

4
plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java
View File

@ -82,12 +82,12 @@ public class AutodetectResultTests extends AbstractSerializingTestCase<Autodetec
modelSizeStats = null;
}
if (randomBoolean()) {
modelPlot = new ModelPlot(jobId, new Date(randomLong()), randomNonNegativeLong());
modelPlot = new ModelPlot(jobId, new Date(randomLong()), randomNonNegativeLong(), randomInt());
} else {
modelPlot = null;
}
if (randomBoolean()) {
forecast = new Forecast(jobId, randomNonNegativeLong(), new Date(randomLong()), randomNonNegativeLong());
forecast = new Forecast(jobId, randomNonNegativeLong(), new Date(randomLong()), randomNonNegativeLong(), randomInt());
} else {
forecast = null;
}

28
plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.test.AbstractSerializingTestCase;
import java.io.IOException;
import java.util.Date;
import java.util.Objects;
public class ForecastTests extends AbstractSerializingTestCase<Forecast> {
@ -25,7 +26,9 @@ public class ForecastTests extends AbstractSerializingTestCase<Forecast> {
}
public Forecast createTestInstance(String jobId) {
Forecast forecast = new Forecast(jobId, randomNonNegativeLong(), new Date(randomLong()), randomNonNegativeLong());
Forecast forecast =
new Forecast(jobId, randomNonNegativeLong(), new Date(randomLong()),
randomNonNegativeLong(), randomInt());
if (randomBoolean()) {
forecast.setByFieldName(randomAlphaOfLengthBetween(1, 20));
@ -65,4 +68,27 @@ public class ForecastTests extends AbstractSerializingTestCase<Forecast> {
return Forecast.PARSER.apply(parser, null);
}
public void testId() {
Forecast forecast = new Forecast("job-foo", 222, new Date(100L), 60L, 2);
String byFieldValue = null;
String partitionFieldValue = null;
int valuesHash = Objects.hash(byFieldValue, partitionFieldValue);
assertEquals("job-foo_model_forecast_222_100_60_2_" + valuesHash + "_0", forecast.getId());
int length = 0;
if (randomBoolean()) {
byFieldValue = randomAlphaOfLength(10);
length += byFieldValue.length();
forecast.setByFieldValue(byFieldValue);
}
if (randomBoolean()) {
partitionFieldValue = randomAlphaOfLength(10);
length += partitionFieldValue.length();
forecast.setPartitionFieldValue(partitionFieldValue);
}
valuesHash = Objects.hash(byFieldValue, partitionFieldValue);
assertEquals("job-foo_model_forecast_222_100_60_2_" + valuesHash + "_" + length, forecast.getId());
}
}

20
plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java
View File

@ -27,7 +27,8 @@ public class ModelPlotTests extends AbstractSerializingTestCase<ModelPlot> {
}
public ModelPlot createTestInstance(String jobId) {
ModelPlot modelPlot = new ModelPlot(jobId, new Date(randomLong()), randomNonNegativeLong());
ModelPlot modelPlot =
new ModelPlot(jobId, new Date(randomLong()), randomNonNegativeLong(), randomInt());
if (randomBoolean()) {
modelPlot.setByFieldName(randomAlphaOfLengthBetween(1, 20));
}
@ -69,13 +70,15 @@ public class ModelPlotTests extends AbstractSerializingTestCase<ModelPlot> {
}
public void testEquals_GivenSameObject() {
ModelPlot modelPlot = new ModelPlot(randomAlphaOfLength(15), new Date(randomLong()), randomNonNegativeLong());
ModelPlot modelPlot =
new ModelPlot(randomAlphaOfLength(15), new Date(randomLong()), randomNonNegativeLong(), randomInt());
assertTrue(modelPlot.equals(modelPlot));
}
public void testEquals_GivenObjectOfDifferentClass() {
ModelPlot modelPlot = new ModelPlot(randomAlphaOfLength(15), new Date(randomLong()), randomNonNegativeLong());
ModelPlot modelPlot =
new ModelPlot(randomAlphaOfLength(15), new Date(randomLong()), randomNonNegativeLong(), randomInt());
assertFalse(modelPlot.equals("a string"));
}
@ -209,16 +212,13 @@ public class ModelPlotTests extends AbstractSerializingTestCase<ModelPlot> {
}
public void testId() {
ModelPlot plot = new ModelPlot("job-foo", new Date(100L), 60L);
ModelPlot plot = new ModelPlot("job-foo", new Date(100L), 60L, 33);
String byFieldValue = null;
String overFieldValue = null;
String partitionFieldValue = null;
int valuesHash = Objects.hash(byFieldValue, overFieldValue, partitionFieldValue);
assertEquals("job-foo_model_plot_100_60__" + valuesHash + "_0", plot.getId());
plot.setModelFeature("a-feature");
assertEquals("job-foo_model_plot_100_60_a-feature_" + valuesHash + "_0", plot.getId());
assertEquals("job-foo_model_plot_100_60_33_" + valuesHash + "_0", plot.getId());
int length = 0;
if (randomBoolean()) {
@ -238,11 +238,11 @@ public class ModelPlotTests extends AbstractSerializingTestCase<ModelPlot> {
}
valuesHash = Objects.hash(byFieldValue, overFieldValue, partitionFieldValue);
assertEquals("job-foo_model_plot_100_60_a-feature_" + valuesHash + "_" + length, plot.getId());
assertEquals("job-foo_model_plot_100_60_33_" + valuesHash + "_" + length, plot.getId());
}
private ModelPlot createFullyPopulated() {
ModelPlot modelPlot = new ModelPlot("foo", new Date(12345678L), 360L);
ModelPlot modelPlot = new ModelPlot("foo", new Date(12345678L), 360L, 22);
modelPlot.setByFieldName("by");
modelPlot.setByFieldValue("by_val");
modelPlot.setPartitionFieldName("part");