[ML] Invert interim result inclusion strategy (elastic/x-pack-elasticsearch#1139)

This changes the get-result actions to include interim
results by default. The former include_interim param is
now changed into an exclude_interim param.

relates elastic/x-pack-elasticsearch#1091

Original commit: elastic/x-pack-elasticsearch@c55a9a89d7
This commit is contained in:
Dimitris Athanasiou 2017-04-20 13:52:35 +01:00 committed by GitHub
parent bf543f0735
commit a2124b68e1
16 changed files with 131 additions and 57 deletions

View File

@ -64,7 +64,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
public static class Request extends ActionRequest implements ToXContent {
public static final ParseField EXPAND = new ParseField("expand");
public static final ParseField INCLUDE_INTERIM = new ParseField("include_interim");
public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim");
public static final ParseField START = new ParseField("start");
public static final ParseField END = new ParseField("end");
public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score");
@ -76,7 +76,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID);
PARSER.declareString(Request::setTimestamp, Result.TIMESTAMP);
PARSER.declareBoolean(Request::setExpand, EXPAND);
PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM);
PARSER.declareBoolean(Request::setExcludeInterim, EXCLUDE_INTERIM);
PARSER.declareStringOrNull(Request::setStart, START);
PARSER.declareStringOrNull(Request::setEnd, END);
PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE);
@ -94,7 +94,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
private String jobId;
private String timestamp;
private boolean expand = false;
private boolean includeInterim = false;
private boolean excludeInterim = false;
private String start;
private String end;
private PageParams pageParams;
@ -135,12 +135,12 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
this.expand = expand;
}
public boolean isIncludeInterim() {
return includeInterim;
public boolean isExcludeInterim() {
return excludeInterim;
}
public void setIncludeInterim(boolean includeInterim) {
this.includeInterim = includeInterim;
public void setExcludeInterim(boolean excludeInterim) {
this.excludeInterim = excludeInterim;
}
public String getStart() {
@ -202,7 +202,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
jobId = in.readString();
timestamp = in.readOptionalString();
expand = in.readBoolean();
includeInterim = in.readBoolean();
excludeInterim = in.readBoolean();
start = in.readOptionalString();
end = in.readOptionalString();
anomalyScore = in.readOptionalDouble();
@ -215,7 +215,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
out.writeString(jobId);
out.writeOptionalString(timestamp);
out.writeBoolean(expand);
out.writeBoolean(includeInterim);
out.writeBoolean(excludeInterim);
out.writeOptionalString(start);
out.writeOptionalString(end);
out.writeOptionalDouble(anomalyScore);
@ -230,7 +230,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
builder.field(Result.TIMESTAMP.getPreferredName(), timestamp);
}
builder.field(EXPAND.getPreferredName(), expand);
builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim);
builder.field(EXCLUDE_INTERIM.getPreferredName(), excludeInterim);
if (start != null) {
builder.field(START.getPreferredName(), start);
}
@ -249,7 +249,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
@Override
public int hashCode() {
return Objects.hash(jobId, timestamp, expand, includeInterim, anomalyScore, pageParams, start, end);
return Objects.hash(jobId, timestamp, expand, excludeInterim, anomalyScore, pageParams, start, end);
}
@Override
@ -264,7 +264,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
return Objects.equals(jobId, other.jobId) &&
Objects.equals(timestamp, other.timestamp) &&
Objects.equals(expand, other.expand) &&
Objects.equals(includeInterim, other.includeInterim) &&
Objects.equals(excludeInterim, other.excludeInterim) &&
Objects.equals(anomalyScore, other.anomalyScore) &&
Objects.equals(pageParams, other.pageParams) &&
Objects.equals(start, other.start) &&
@ -359,7 +359,7 @@ public class GetBucketsAction extends Action<GetBucketsAction.Request, GetBucket
BucketsQueryBuilder query =
new BucketsQueryBuilder().expand(request.expand)
.includeInterim(request.includeInterim)
.includeInterim(request.excludeInterim == false)
.start(request.start)
.end(request.end)
.anomalyScoreThreshold(request.anomalyScore);

View File

@ -65,7 +65,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
public static final ParseField START = new ParseField("start");
public static final ParseField END = new ParseField("end");
public static final ParseField INCLUDE_INTERIM = new ParseField("include_interim");
public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim");
public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score");
public static final ParseField SORT_FIELD = new ParseField("sort");
public static final ParseField DESCENDING_SORT = new ParseField("desc");
@ -76,7 +76,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID);
PARSER.declareStringOrNull(Request::setStart, START);
PARSER.declareStringOrNull(Request::setEnd, END);
PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM);
PARSER.declareBoolean(Request::setExcludeInterim, EXCLUDE_INTERIM);
PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE);
PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE);
PARSER.declareString(Request::setSort, SORT_FIELD);
@ -94,7 +94,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
private String jobId;
private String start;
private String end;
private boolean includeInterim = false;
private boolean excludeInterim = false;
private PageParams pageParams = new PageParams();
private double anomalyScoreFilter = 0.0;
private String sort = Influencer.INFLUENCER_SCORE.getPreferredName();
@ -135,12 +135,12 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
this.descending = descending;
}
public boolean isIncludeInterim() {
return includeInterim;
public boolean isExcludeInterim() {
return excludeInterim;
}
public void setIncludeInterim(boolean includeInterim) {
this.includeInterim = includeInterim;
public void setExcludeInterim(boolean excludeInterim) {
this.excludeInterim = excludeInterim;
}
public void setPageParams(PageParams pageParams) {
@ -176,7 +176,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
jobId = in.readString();
includeInterim = in.readBoolean();
excludeInterim = in.readBoolean();
pageParams = new PageParams(in);
start = in.readOptionalString();
end = in.readOptionalString();
@ -189,7 +189,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(jobId);
out.writeBoolean(includeInterim);
out.writeBoolean(excludeInterim);
pageParams.writeTo(out);
out.writeOptionalString(start);
out.writeOptionalString(end);
@ -202,7 +202,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(Job.ID.getPreferredName(), jobId);
builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim);
builder.field(EXCLUDE_INTERIM.getPreferredName(), excludeInterim);
builder.field(PageParams.PAGE.getPreferredName(), pageParams);
builder.field(START.getPreferredName(), start);
builder.field(END.getPreferredName(), end);
@ -215,7 +215,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
@Override
public int hashCode() {
return Objects.hash(jobId, includeInterim, pageParams, start, end, sort, descending, anomalyScoreFilter);
return Objects.hash(jobId, excludeInterim, pageParams, start, end, sort, descending, anomalyScoreFilter);
}
@Override
@ -229,7 +229,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
Request other = (Request) obj;
return Objects.equals(jobId, other.jobId) && Objects.equals(start, other.start)
&& Objects.equals(end, other.end)
&& Objects.equals(includeInterim, other.includeInterim)
&& Objects.equals(excludeInterim, other.excludeInterim)
&& Objects.equals(pageParams, other.pageParams)
&& Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter)
&& Objects.equals(descending, other.descending)
@ -321,7 +321,7 @@ extends Action<GetInfluencersAction.Request, GetInfluencersAction.Response, GetI
protected void doExecute(Request request, ActionListener<Response> listener) {
jobManager.getJobOrThrowIfUnknown(request.jobId);
InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder().includeInterim(request.includeInterim)
InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder().includeInterim(request.excludeInterim == false)
.start(request.start).end(request.end).from(request.pageParams.getFrom()).size(request.pageParams.getSize())
.anomalyScoreThreshold(request.anomalyScoreFilter).sortField(request.sort).sortDescending(request.descending).build();
jobProvider.influencers(request.jobId, query, page -> listener.onResponse(new Response(page)), listener::onFailure, client);

View File

@ -65,7 +65,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
public static final ParseField START = new ParseField("start");
public static final ParseField END = new ParseField("end");
public static final ParseField INCLUDE_INTERIM = new ParseField("include_interim");
public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim");
public static final ParseField RECORD_SCORE_FILTER = new ParseField("record_score");
public static final ParseField SORT = new ParseField("sort");
public static final ParseField DESCENDING = new ParseField("desc");
@ -78,7 +78,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
PARSER.declareStringOrNull(Request::setEnd, END);
PARSER.declareString(Request::setSort, SORT);
PARSER.declareBoolean(Request::setDescending, DESCENDING);
PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM);
PARSER.declareBoolean(Request::setExcludeInterim, EXCLUDE_INTERIM);
PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE);
PARSER.declareDouble(Request::setRecordScore, RECORD_SCORE_FILTER);
}
@ -94,7 +94,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
private String jobId;
private String start;
private String end;
private boolean includeInterim = false;
private boolean excludeInterim = false;
private PageParams pageParams = new PageParams();
private double recordScoreFilter = 0.0;
private String sort = Influencer.INFLUENCER_SCORE.getPreferredName();
@ -135,12 +135,12 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
this.descending = descending;
}
public boolean isIncludeInterim() {
return includeInterim;
public boolean isExcludeInterim() {
return excludeInterim;
}
public void setIncludeInterim(boolean includeInterim) {
this.includeInterim = includeInterim;
public void setExcludeInterim(boolean excludeInterim) {
this.excludeInterim = excludeInterim;
}
public void setPageParams(PageParams pageParams) {
@ -175,7 +175,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
jobId = in.readString();
includeInterim = in.readBoolean();
excludeInterim = in.readBoolean();
pageParams = new PageParams(in);
start = in.readOptionalString();
end = in.readOptionalString();
@ -188,7 +188,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(jobId);
out.writeBoolean(includeInterim);
out.writeBoolean(excludeInterim);
pageParams.writeTo(out);
out.writeOptionalString(start);
out.writeOptionalString(end);
@ -206,7 +206,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
builder.field(SORT.getPreferredName(), sort);
builder.field(DESCENDING.getPreferredName(), descending);
builder.field(RECORD_SCORE_FILTER.getPreferredName(), recordScoreFilter);
builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim);
builder.field(EXCLUDE_INTERIM.getPreferredName(), excludeInterim);
builder.field(PageParams.PAGE.getPreferredName(), pageParams);
builder.endObject();
return builder;
@ -214,7 +214,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
@Override
public int hashCode() {
return Objects.hash(jobId, start, end, sort, descending, recordScoreFilter, includeInterim, pageParams);
return Objects.hash(jobId, start, end, sort, descending, recordScoreFilter, excludeInterim, pageParams);
}
@Override
@ -232,7 +232,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
Objects.equals(sort, other.sort) &&
Objects.equals(descending, other.descending) &&
Objects.equals(recordScoreFilter, other.recordScoreFilter) &&
Objects.equals(includeInterim, other.includeInterim) &&
Objects.equals(excludeInterim, other.excludeInterim) &&
Objects.equals(pageParams, other.pageParams);
}
}
@ -324,7 +324,7 @@ public class GetRecordsAction extends Action<GetRecordsAction.Request, GetRecord
jobManager.getJobOrThrowIfUnknown(request.getJobId());
RecordsQueryBuilder.RecordsQuery query = new RecordsQueryBuilder()
.includeInterim(request.includeInterim)
.includeInterim(request.excludeInterim == false)
.epochStart(request.start)
.epochEnd(request.end)
.from(request.pageParams.getFrom())

View File

@ -74,7 +74,7 @@ public class RestGetBucketsAction extends BaseRestHandler {
// single and multiple bucket options
request.setExpand(restRequest.paramAsBoolean(GetBucketsAction.Request.EXPAND.getPreferredName(), false));
request.setIncludeInterim(restRequest.paramAsBoolean(GetBucketsAction.Request.INCLUDE_INTERIM.getPreferredName(), false));
request.setExcludeInterim(restRequest.paramAsBoolean(GetBucketsAction.Request.EXCLUDE_INTERIM.getPreferredName(), false));
}
return channel -> client.execute(GetBucketsAction.INSTANCE, request, new RestToXContentListener<>(channel));

View File

@ -44,7 +44,7 @@ public class RestGetInfluencersAction extends BaseRestHandler {
request = new GetInfluencersAction.Request(jobId);
request.setStart(start);
request.setEnd(end);
request.setIncludeInterim(restRequest.paramAsBoolean(GetInfluencersAction.Request.INCLUDE_INTERIM.getPreferredName(), false));
request.setExcludeInterim(restRequest.paramAsBoolean(GetInfluencersAction.Request.EXCLUDE_INTERIM.getPreferredName(), false));
request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE)));
request.setAnomalyScore(

View File

@ -42,7 +42,7 @@ public class RestGetRecordsAction extends BaseRestHandler {
request = new GetRecordsAction.Request(jobId);
request.setStart(restRequest.param(GetRecordsAction.Request.START.getPreferredName()));
request.setEnd(restRequest.param(GetRecordsAction.Request.END.getPreferredName()));
request.setIncludeInterim(restRequest.paramAsBoolean(GetRecordsAction.Request.INCLUDE_INTERIM.getPreferredName(), false));
request.setExcludeInterim(restRequest.paramAsBoolean(GetRecordsAction.Request.EXCLUDE_INTERIM.getPreferredName(), false));
request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE)));
request.setRecordScore(

View File

@ -26,7 +26,7 @@ public class GetBucketActionRequestTests extends AbstractStreamableXContentTestC
request.setEnd(String.valueOf(randomLong()));
}
if (randomBoolean()) {
request.setIncludeInterim(randomBoolean());
request.setExcludeInterim(randomBoolean());
}
if (randomBoolean()) {
request.setAnomalyScore(randomDouble());
@ -42,7 +42,7 @@ public class GetBucketActionRequestTests extends AbstractStreamableXContentTestC
request.setExpand(randomBoolean());
}
if (randomBoolean()) {
request.setIncludeInterim(randomBoolean());
request.setExcludeInterim(randomBoolean());
}
return request;
}

View File

@ -32,7 +32,7 @@ public class GetInfluencersActionRequestTests extends AbstractStreamableXContent
request.setAnomalyScore(randomDouble());
}
if (randomBoolean()) {
request.setIncludeInterim(randomBoolean());
request.setExcludeInterim(randomBoolean());
}
if (randomBoolean()) {
request.setSort(randomAlphaOfLengthBetween(1, 20));

View File

@ -38,7 +38,7 @@ public class GetRecordsActionRequestTests extends AbstractStreamableXContentTest
request.setRecordScore(randomDouble());
}
if (randomBoolean()) {
request.setIncludeInterim(randomBoolean());
request.setExcludeInterim(randomBoolean());
}
if (randomBoolean()) {
int from = randomInt(PageParams.MAX_FROM_SIZE_SUM);

View File

@ -123,7 +123,6 @@ public class UpdateInterimResultsIT extends MlNativeAutodetectIntegTestCase {
private List<Bucket> getInterimResults(String jobId) throws Exception {
GetBucketsAction.Request request = new GetBucketsAction.Request(jobId);
request.setIncludeInterim(true);
request.setExpand(true);
request.setPageParams(new PageParams(0, 1500));
GetBucketsAction.Response response = client().execute(GetBucketsAction.INSTANCE, request).get();

View File

@ -23,9 +23,9 @@
"type": "boolean",
"description" : "Include anomaly records"
},
"include_interim": {
"exclude_interim": {
"type": "boolean",
"description" : "Include interim results"
"description" : "Exclude interim results"
},
"from": {
"type": "int",

View File

@ -11,9 +11,9 @@
}
},
"params": {
"include_interim": {
"exclude_interim": {
"type": "boolean",
"description" : "Include interim results"
"description" : "Exclude interim results"
},
"from": {
"type": "int",

View File

@ -13,9 +13,9 @@
}
},
"params": {
"include_interim": {
"exclude_interim": {
"type": "boolean",
"description": "Include interim results"
"description": "Exclude interim results"
},
"from": {
"type": "int",

View File

@ -18,14 +18,27 @@ setup:
index: .ml-anomalies-farequote
type: result
id: "farequote_1464739200000_1"
body: { "job_id": "farequote", "result_type": "bucket", "timestamp": "2016-06-01T00:00:00Z", "bucket_span":1 }
body:
{
"job_id": "farequote",
"result_type": "bucket",
"timestamp": "2016-06-01T00:00:00Z",
"bucket_span":1
}
- do:
index:
index: .ml-anomalies-farequote
type: result
id: "farequote_1464739200000_2"
body: { "job_id": "farequote", "result_type": "bucket", "timestamp": "2016-08-01T00:00:00Z", "bucket_span":1 }
body:
{
"job_id": "farequote",
"result_type": "bucket",
"timestamp": "2016-08-01T00:00:00Z",
"bucket_span":1,
"is_interim": true
}
- do:
indices.refresh:
@ -58,6 +71,26 @@ setup:
- match: { buckets.1.job_id: farequote}
- match: { buckets.1.result_type: bucket}
---
"Test get buckets given exclude_interim is false":
- do:
xpack.ml.get_buckets:
job_id: "farequote"
exclude_interim: false
- match: { count: 2 }
---
"Test get buckets given exclude_interim is true":
- do:
xpack.ml.get_buckets:
job_id: "farequote"
exclude_interim: true
- match: { count: 1 }
- match: { buckets.0.timestamp: 1464739200000 }
- match: { buckets.0.is_interim: false }
---
"Test result single bucket api":
- do:

View File

@ -44,7 +44,8 @@ setup:
"influencer_score": 50.0,
"result_type" : "influencer",
"bucket_span" : 1,
"sequence_num" : 2
"sequence_num" : 2,
"is_interim": true
}
- do:
indices.refresh:
@ -60,6 +61,26 @@ setup:
- match: { influencers.0.timestamp: 1464739200000 }
- match: { influencers.1.timestamp: 1464825600000 }
---
"Test get influencers given exclude_interim false":
- do:
xpack.ml.get_influencers:
job_id: "farequote"
exclude_interim: false
- match: { count: 2 }
---
"Test get influencers given exclude_interim true":
- do:
xpack.ml.get_influencers:
job_id: "farequote"
exclude_interim: true
- match: { count: 1 }
- match: { influencers.0.timestamp: 1464739200000 }
- match: { influencers.0.is_interim: false }
---
"Test result influencers api with time range":
- do:

View File

@ -40,7 +40,8 @@ setup:
"timestamp": "2016-06-02T00:00:00Z",
"record_score": 80.0,
"bucket_span": 1,
"sequence_num": 2
"sequence_num": 2,
"is_interim": true
}
- do:
@ -61,6 +62,26 @@ setup:
- match: { records.1.job_id: farequote}
- match: { records.1.result_type: record}
---
"Test get records given exclude_interim is false":
- do:
xpack.ml.get_records:
job_id: "farequote"
exclude_interim: false
- match: { count: 2 }
---
"Test get records given exclude_interim is true":
- do:
xpack.ml.get_records:
job_id: "farequote"
exclude_interim: true
- match: { count: 1 }
- match: { records.0.timestamp: 1464739200000 }
- match: { records.0.is_interim: false }
---
"Test result records api with time range":
- do: