honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Change result index searches to not use _type (elastic/x-pack-elasticsearch#1509) 

Adjusts the searches for

- buckets
- categories
- model snapshots

to not use _type.

Relates elastic/x-pack-elasticsearch#668

Original commit: elastic/x-pack-elasticsearch@8269609705
This commit is contained in:
Dimitris Athanasiou 2017-05-22 14:44:39 +01:00 committed by GitHub
parent edc299a532
commit aff8258398
11 changed files with 50 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCategoriesAction.java
View File

@ -28,12 +28,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.action.util.PageParams;
import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.job.JobManager;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.ml.action.util.PageParams;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException;
@ -71,7 +71,7 @@ Action<GetCategoriesAction.Request, GetCategoriesAction.Response, GetCategoriesA
static {
PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID);
PARSER.declareString(Request::setCategoryId, CATEGORY_ID);
PARSER.declareLong(Request::setCategoryId, CATEGORY_ID);
PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE);
}
@ -84,7 +84,7 @@ Action<GetCategoriesAction.Request, GetCategoriesAction.Response, GetCategoriesA
}
private String jobId;
private String categoryId;
private Long categoryId;
private PageParams pageParams;
public Request(String jobId) {
@ -94,11 +94,7 @@ Action<GetCategoriesAction.Request, GetCategoriesAction.Response, GetCategoriesA
Request() {
}
public String getCategoryId() {
return categoryId;
}
public void setCategoryId(String categoryId) {
public void setCategoryId(Long categoryId) {
if (pageParams != null) {
throw new IllegalArgumentException("Param [" + CATEGORY_ID.getPreferredName() + "] is incompatible with ["
+ PageParams.FROM.getPreferredName() + ", " + PageParams.SIZE.getPreferredName() + "].");
@ -106,10 +102,6 @@ Action<GetCategoriesAction.Request, GetCategoriesAction.Response, GetCategoriesA
this.categoryId = ExceptionsHelper.requireNonNull(categoryId, CATEGORY_ID.getPreferredName());
}
public PageParams getPageParams() {
return pageParams;
}
public void setPageParams(PageParams pageParams) {
if (categoryId != null) {
throw new IllegalArgumentException("Param [" + PageParams.FROM.getPreferredName() + ", "
@ -133,7 +125,7 @@ Action<GetCategoriesAction.Request, GetCategoriesAction.Response, GetCategoriesA
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
jobId = in.readString();
categoryId = in.readOptionalString();
categoryId = in.readOptionalLong();
pageParams = in.readOptionalWriteable(PageParams::new);
}
@ -141,7 +133,7 @@ Action<GetCategoriesAction.Request, GetCategoriesAction.Response, GetCategoriesA
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(jobId);
out.writeOptionalString(categoryId);
out.writeOptionalLong(categoryId);
out.writeOptionalWriteable(pageParams);
}

19
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java
View File

@ -38,8 +38,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@ -377,7 +375,6 @@ public class JobProvider {
.filter(QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), Bucket.RESULT_TYPE_VALUE));
String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.types(Result.TYPE.getPreferredName());
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.sort(sortBuilder);
searchSourceBuilder.query(boolQuery);
@ -569,7 +566,7 @@ public class JobProvider {
* @param from Skip the first N categories. This parameter is for paging
* @param size Take only this number of categories
*/
public void categoryDefinitions(String jobId, String categoryId, Integer from, Integer size,
public void categoryDefinitions(String jobId, Long categoryId, Integer from, Integer size,
Consumer<QueryPage<CategoryDefinition>> handler,
Consumer<Exception> errorHandler, Client client) {
if (categoryId != null && (from != null || size != null)) {
@ -584,13 +581,10 @@ public class JobProvider {
searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions()));
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
if (categoryId != null) {
String documentId = CategoryDefinition.documentId(jobId, categoryId);
String uid = Uid.createUid(CategoryDefinition.TYPE.getPreferredName(), documentId);
sourceBuilder.query(QueryBuilders.termQuery(UidFieldMapper.NAME, uid));
searchRequest.routing(documentId);
sourceBuilder.query(QueryBuilders.termQuery(CategoryDefinition.CATEGORY_ID.getPreferredName(), categoryId));
} else if (from != null && size != null) {
searchRequest.types(CategoryDefinition.TYPE.getPreferredName());
sourceBuilder.from(from).size(size)
.query(QueryBuilders.existsQuery(CategoryDefinition.CATEGORY_ID.getPreferredName()))
.sort(new FieldSortBuilder(CategoryDefinition.CATEGORY_ID.getPreferredName()).order(SortOrder.ASC));
} else {
throw new IllegalStateException("Both categoryId and pageParams are not specified");
@ -812,6 +806,10 @@ public class JobProvider {
sortField = ModelSnapshot.TIMESTAMP.getPreferredName();
}
QueryBuilder finalQuery = QueryBuilders.boolQuery()
.filter(QueryBuilders.existsQuery(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()))
.must(qb);
FieldSortBuilder sb = new FieldSortBuilder(sortField)
.order(sortDescending ? SortOrder.DESC : SortOrder.ASC);
@ -821,10 +819,9 @@ public class JobProvider {
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions()));
searchRequest.types(ModelSnapshot.TYPE.getPreferredName());
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.sort(sb);
sourceBuilder.query(qb);
sourceBuilder.query(finalQuery);
sourceBuilder.from(from);
sourceBuilder.size(size);
searchRequest.source(sourceBuilder);

12
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.ml.rest.results;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
@ -17,8 +16,8 @@ import org.elasticsearch.rest.action.RestToXContentListener;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.GetCategoriesAction;
import org.elasticsearch.xpack.ml.action.GetCategoriesAction.Request;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.action.util.PageParams;
import org.elasticsearch.xpack.ml.job.config.Job;
import java.io.IOException;
@ -43,23 +42,24 @@ public class RestGetCategoriesAction extends BaseRestHandler {
protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
Request request;
String jobId = restRequest.param(Job.ID.getPreferredName());
String categoryId = restRequest.param(Request.CATEGORY_ID.getPreferredName());
Long categoryId = restRequest.hasParam(Request.CATEGORY_ID.getPreferredName()) ? Long.parseLong(
restRequest.param(Request.CATEGORY_ID.getPreferredName())) : null;
BytesReference bodyBytes = restRequest.content();
if (bodyBytes != null && bodyBytes.length() > 0) {
XContentParser parser = restRequest.contentParser();
request = GetCategoriesAction.Request.parseRequest(jobId, parser);
if (!Strings.isNullOrEmpty(categoryId)) {
if (categoryId != null) {
request.setCategoryId(categoryId);
}
} else {
request = new Request(jobId);
if (!Strings.isNullOrEmpty(categoryId)) {
if (categoryId != null) {
request.setCategoryId(categoryId);
}
if (restRequest.hasParam(Request.FROM.getPreferredName())
|| restRequest.hasParam(Request.SIZE.getPreferredName())
|| Strings.isNullOrEmpty(categoryId)){
|| categoryId == null){
request.setPageParams(new PageParams(
restRequest.paramAsInt(Request.FROM.getPreferredName(), 0),

2
plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesRequestTests.java
View File

@ -16,7 +16,7 @@ public class GetCategoriesRequestTests extends AbstractStreamableXContentTestCas
String jobId = randomAlphaOfLength(10);
GetCategoriesAction.Request request = new GetCategoriesAction.Request(jobId);
if (randomBoolean()) {
request.setCategoryId(randomAlphaOfLength(10));
request.setCategoryId(randomNonNegativeLong());
} else {
int from = randomInt(PageParams.MAX_FROM_SIZE_SUM);
int maxSize = PageParams.MAX_FROM_SIZE_SUM - from;

5
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
View File

@ -146,8 +146,7 @@ public class AutodetectResultProcessorIT extends XPackSingleNodeTestCase {
QueryPage<Influencer> persistedInfluencers = getInfluencers();
assertResultsAreSame(influencers, persistedInfluencers);
QueryPage<CategoryDefinition> persistedDefinition =
getCategoryDefinition(Long.toString(categoryDefinition.getCategoryId()));
QueryPage<CategoryDefinition> persistedDefinition = getCategoryDefinition(categoryDefinition.getCategoryId());
assertEquals(1, persistedDefinition.count());
assertEquals(categoryDefinition, persistedDefinition.results().get(0));
@ -422,7 +421,7 @@ public class AutodetectResultProcessorIT extends XPackSingleNodeTestCase {
return resultHolder.get();
}
private QueryPage<CategoryDefinition> getCategoryDefinition(String categoryId) throws Exception {
private QueryPage<CategoryDefinition> getCategoryDefinition(long categoryId) throws Exception {
AtomicReference<Exception> errorHolder = new AtomicReference<>();
AtomicReference<QueryPage<CategoryDefinition>> resultHolder = new AtomicReference<>();
CountDownLatch latch = new CountDownLatch(1);

2
plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java
View File

@ -621,7 +621,7 @@ public class JobProviderTests extends ESTestCase {
String terms = "the terms and conditions are not valid here";
Map<String, Object> source = new HashMap<>();
String categoryId = String.valueOf(source.hashCode());
long categoryId = source.hashCode();
source.put("job_id", "foo");
source.put("category_id", categoryId);
source.put("terms", terms);

2
plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_categories.json
View File

@ -14,7 +14,7 @@
"description": "The name of the job"
},
"category_id": {
"type" : "string",
"type" : "long",
"description" : "The identifier of the category definition of interest"
}
},

3
plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
View File

@ -73,7 +73,8 @@ setup:
"snapshot_id": "active-snapshot",
"description": "second",
"latest_record_time_stamp": "2016-06-01T00:00:00Z",
"latest_result_time_stamp": "2016-06-01T00:00:00Z"
"latest_result_time_stamp": "2016-06-01T00:00:00Z",
"snapshot_doc_count": 3
}
- do:

16
plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
View File

@ -18,14 +18,26 @@ setup:
index: .ml-anomalies-get-model-snapshots
type: model_snapshot
id: "get-model-snapshots-1"
body: { "job_id": "get-model-snapshots", "snapshot_id": "1", "timestamp": "2016-06-02T00:00:00Z" }
body: >
{
"job_id": "get-model-snapshots",
"snapshot_id": "1",
"timestamp": "2016-06-02T00:00:00Z",
"snapshot_doc_count": 1
}
- do:
index:
index: .ml-anomalies-get-model-snapshots
type: model_snapshot
id: "get-model-snapshots-2"
body: { "job_id": "get-model-snapshots", "snapshot_id": "2", "timestamp": "2016-06-01T00:00:00Z" }
body: >
{
"job_id": "get-model-snapshots",
"snapshot_id": "2",
"timestamp": "2016-06-01T00:00:00Z",
"snapshot_doc_count": 2
}
- do:
indices.refresh:

12
plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
View File

@ -85,21 +85,21 @@ setup:
catch: request
xpack.ml.get_categories:
job_id: "jobs-get-result-categories"
category_id: "1"
category_id: 1
from: 0
- do:
catch: request
xpack.ml.get_categories:
job_id: "jobs-get-result-categories"
category_id: "1"
category_id: 1
size: 1
- do:
catch: request
xpack.ml.get_categories:
job_id: "jobs-get-result-categories"
category_id: "1"
category_id: 1
from: 0
size: 1
@ -109,7 +109,7 @@ setup:
catch: request
xpack.ml.get_categories:
job_id: "jobs-get-result-categories"
category_id: "1"
category_id: 1
body:
from: 0
@ -117,7 +117,7 @@ setup:
catch: request
xpack.ml.get_categories:
job_id: "jobs-get-result-categories"
category_id: "1"
category_id: 1
body:
size: 1
@ -125,7 +125,7 @@ setup:
catch: request
xpack.ml.get_categories:
job_id: "jobs-get-result-categories"
category_id: "1"
category_id: 1
body:
from: 0
size: 1

2
plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
View File

@ -23,6 +23,7 @@ setup:
"job_id" : "update-model-snapshot",
"timestamp": "2016-06-02T00:00:00Z",
"snapshot_id": "snapshot-1",
"snapshot_doc_count": 3,
"retain": false
}
@ -37,6 +38,7 @@ setup:
"timestamp": "2016-06-01T00:00:00Z",
"snapshot_id": "snapshot-2",
"description": "snapshot 2 description",
"snapshot_doc_count": 2,
"retain": true
}