From 30ea37943cc0c0e3cf1a0cb6f7d10c95a802413d Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 17 Mar 2017 17:08:29 +0000 Subject: [PATCH] [ML] Change delimiter in the names of state docs (elastic/x-pack-elasticsearch#768) Original commit: elastic/x-pack-elasticsearch@34acda3ffeb2dba25049b1521cfdb5b90585bb0d --- .../ml/job/persistence/JobDataDeleter.java | 2 +- .../xpack/ml/job/persistence/JobProvider.java | 2 +- .../persistence/JobStorageDeletionTask.java | 10 +++++----- .../autodetect/state/CategorizerState.java | 2 +- .../ml/integration/DeleteExpiredDataIT.java | 1 - .../ml/job/persistence/JobProviderTests.java | 10 ++++------ .../test/ml/delete_model_snapshot.yaml | 4 ++-- .../rest-api-spec/test/ml/index_layout.yaml | 19 ++++++++++++++++++- 8 files changed, 32 insertions(+), 18 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java index f1cae309f6b..18cae149f65 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -114,7 +114,7 @@ public class JobDataDeleter { // too big and has no mappings. // Note: state docs are 1-based for (int i = 1; i <= docCount; ++i) { - String stateId = snapshotDocId + '_' + i; + String stateId = snapshotDocId + '#' + i; bulkRequestBuilder.add(client.prepareDelete(stateIndexName, ModelState.TYPE.getPreferredName(), stateId)); ++deletedModelStateCount; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index 0ffacce7486..8fe9b607e81 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -927,7 +927,7 @@ public class JobProvider { // the order the C++ process expects. int numDocs = modelSnapshot.getSnapshotDocCount(); for (docNum = 1; docNum <= numDocs; ++docNum) { - String docId = String.format(Locale.ROOT, "%s_%d", ModelSnapshot.documentId(modelSnapshot), docNum); + String docId = String.format(Locale.ROOT, "%s#%d", ModelSnapshot.documentId(modelSnapshot), docNum); LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, ModelState.TYPE, indexName); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java index 15c9777228d..0e7135d0bac 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java @@ -24,8 +24,8 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; -import org.elasticsearch.index.query.PrefixQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -156,13 +156,13 @@ public class JobStorageDeletionTask extends Task { } private void deleteCategorizerState(String jobId, Client client, ActionListener finishedHandler) { - SearchRequest searchRequest = new SearchRequest(); + SearchRequest searchRequest = new SearchRequest(AnomalyDetectorsIndex.jobStateIndexName()); DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest); - PrefixQueryBuilder query = new PrefixQueryBuilder(UidFieldMapper.NAME, Uid.createUid(CategorizerState.TYPE, jobId)); - searchRequest.source(new SearchSourceBuilder().query(query)); - searchRequest.indicesOptions(JobProvider.addIgnoreUnavailable(IndicesOptions.lenientExpandOpen())); request.setSlices(5); + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + WildcardQueryBuilder query = new WildcardQueryBuilder(UidFieldMapper.NAME, Uid.createUid(CategorizerState.TYPE, jobId + "#*")); + searchRequest.source(new SearchSourceBuilder().query(query)); client.execute(MlDeleteByQueryAction.INSTANCE, request, new ActionListener() { @Override public void onResponse(BulkByScrollResponse bulkByScrollResponse) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java index 7b6430ab880..bab37a2a646 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java @@ -19,7 +19,7 @@ public class CategorizerState { public static final String TYPE = "categorizer_state"; public static final String categorizerStateDocId(String jobId, int docNum) { - return jobId + "_" + docNum; + return jobId + "#" + docNum; } private CategorizerState() { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index e699ae7886f..26c8c2e6b79 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -169,7 +169,6 @@ public class DeleteExpiredDataIT extends SecurityIntegTestCase { assertThat(getRecords(job.getId()).size(), equalTo(1)); List modelSnapshots = getModelSnapshots(job.getId()); assertThat(modelSnapshots.size(), equalTo(2)); - } long totalModelSizeStatsBeforeDelete = client().prepareSearch("*").setTypes("result") diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java index 742cf9d4fe7..5887cced56a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ml.MlMetadata; -import org.elasticsearch.xpack.ml.action.DeleteJobAction; import org.elasticsearch.xpack.ml.action.util.QueryPage; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; @@ -70,7 +69,6 @@ import java.util.function.Consumer; import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; @@ -1039,11 +1037,11 @@ public class JobProviderTests extends ESTestCase { GetResponse modelStateGetResponse2 = createGetResponse(true, modelState); MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() - .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), CategorizerState.TYPE, JOB_ID + "_1", categorizerStateGetResponse1) - .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), CategorizerState.TYPE, JOB_ID + "_2", categorizerStateGetResponse2) - .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ModelState.TYPE.getPreferredName(), JOB_ID + "-123_1", + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), CategorizerState.TYPE, JOB_ID + "#1", categorizerStateGetResponse1) + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), CategorizerState.TYPE, JOB_ID + "#2", categorizerStateGetResponse2) + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ModelState.TYPE.getPreferredName(), JOB_ID + "-123#1", modelStateGetResponse1) - .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ModelState.TYPE.getPreferredName(), JOB_ID + "-123_2", + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ModelState.TYPE.getPreferredName(), JOB_ID + "-123#2", modelStateGetResponse2); JobProvider provider = createProvider(clientBuilder.build()); diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yaml b/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yaml index 99023325ff0..fe076ad9576 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yaml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yaml @@ -51,7 +51,7 @@ setup: index: index: .ml-state type: model_state - id: "foo-inactive-snapshot_1" + id: "foo-inactive-snapshot#1" body: > { "state": "a" @@ -61,7 +61,7 @@ setup: index: index: .ml-state type: model_state - id: "foo-inactive-snapshot_2" + id: "foo-inactive-snapshot#2" body: > { "state": "b" diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yaml b/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yaml index 33a8118e880..a484fbc3768 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yaml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yaml @@ -163,6 +163,23 @@ setup: job_id: farequote2 - match: {count: 4} +# Put some categorizer state + - do: + index: + index: .ml-state + type: categorizer_state + id: farequote2#1 + body: + key: value + + - do: + index: + index: .ml-state + type: categorizer_state + id: farequote2#2 + body: + key: value + - do: xpack.ml.delete_job: job_id: "farequote" @@ -179,7 +196,7 @@ setup: - do: count: index: .ml-state - - match: {count: 2} + - match: {count: 4} - do: count: