From 78fdca65d5d5058467e5ec9342b72ac8616b5f21 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 20 Dec 2016 10:47:37 +0000 Subject: [PATCH] Add job Id to state document IDs (elastic/elasticsearch#579) * Persist quantile documents with the jobId in the document Id * Add job Id to snapshot Id * Add job Id to categoriser state document Id * Rename quantiles doc to start with job id as the other state docs do * Fix restoring categoriser state Original commit: elastic/x-pack-elasticsearch@3e5d3368b51c9646ca2494343746c14640fe1789 --- .../xpack/prelert/job/CategorizerState.java | 4 ++++ .../xpack/prelert/job/persistence/JobProvider.java | 9 +++++---- .../prelert/job/persistence/JobResultsPersister.java | 5 ++--- .../xpack/prelert/job/quantiles/Quantiles.java | 5 ++++- .../prelert/job/persistence/JobProviderTests.java | 12 ++++++------ 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java index 61cbeb7df91..356dc21c946 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java @@ -18,6 +18,10 @@ public class CategorizerState { */ public static final String TYPE = "categorizer_state"; + public static final String categorizerStateDocId(String jobId, int docNum) { + return jobId + "_" + docNum; + } + private CategorizerState() { } } diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java index c8677dcfc3f..0760ebb81a9 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java @@ -857,10 +857,11 @@ public class JobProvider { public Optional getQuantiles(String jobId) { String indexName = AnomalyDetectorsIndex.getJobIndexName(jobId); try { - LOGGER.trace("ES API CALL: get ID " + Quantiles.QUANTILES_ID + - " type " + Quantiles.TYPE + " from index " + indexName); + String quantilesId = Quantiles.quantilesId(jobId); + + LOGGER.trace("ES API CALL: get ID {} type {} from index {}", quantilesId, Quantiles.TYPE.getPreferredName(), indexName); GetResponse response = client.prepareGet( - indexName, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID).get(); + indexName, Quantiles.TYPE.getPreferredName(), quantilesId).get(); if (!response.isExists()) { LOGGER.info("There are currently no quantiles for job " + jobId); return Optional.empty(); @@ -982,7 +983,7 @@ public class JobProvider { // count up until a document is not found. It's NOT an error to have no categorizer state. int docNum = 0; while (true) { - String docId = Integer.toString(++docNum); + String docId = CategorizerState.categorizerStateDocId(jobId, ++docNum); LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, CategorizerState.TYPE, indexName); diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java index b66ebf532b5..d0a1e942965 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java @@ -209,7 +209,7 @@ public class JobResultsPersister extends AbstractComponent { */ public void persistQuantiles(Quantiles quantiles) { Persistable persistable = new Persistable(quantiles.getJobId(), quantiles, Quantiles.TYPE.getPreferredName(), - Quantiles.QUANTILES_ID); + Quantiles.quantilesId(quantiles.getJobId())); if (persistable.persist()) { // Refresh the index when persisting quantiles so that previously // persisted results will be available for searching. Do this using the @@ -282,8 +282,7 @@ public class JobResultsPersister extends AbstractComponent { /** * Once all the job data has been written this function will be - * called to commit the data if the implementing persister requires - * it. + * called to commit the writes to the datastore. * * @return True if successful */ diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java index 1733cccba89..e672dc9047e 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java @@ -24,7 +24,6 @@ import java.util.Objects; * Quantiles Result POJO */ public class Quantiles extends ToXContentToBytes implements Writeable { - public static final String QUANTILES_ID = "hierarchical"; /** * Field Names @@ -46,6 +45,10 @@ public class Quantiles extends ToXContentToBytes implements Writeable { PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); } + public static String quantilesId(String jobId) { + return jobId + "-" + TYPE.getPreferredName(); + } + private final String jobId; private final Date timestamp; private final String quantileState; diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderTests.java index 03d4af750af..faef73f2025 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderTests.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderTests.java @@ -74,7 +74,7 @@ public class JobProviderTests extends ESTestCase { MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) - .throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID); + .throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID)); JobProvider provider = createProvider(clientBuilder.build()); @@ -86,7 +86,7 @@ public class JobProviderTests extends ESTestCase { MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) - .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); + .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID), getResponse); JobProvider provider = createProvider(clientBuilder.build()); @@ -104,7 +104,7 @@ public class JobProviderTests extends ESTestCase { MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) - .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); + .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID), getResponse); JobProvider provider = createProvider(clientBuilder.build()); @@ -123,7 +123,7 @@ public class JobProviderTests extends ESTestCase { MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) - .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); + .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId("foo"), getResponse); JobProvider provider = createProvider(clientBuilder.build()); @@ -1077,8 +1077,8 @@ public class JobProviderTests extends ESTestCase { MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) - .prepareGet(INDEX_NAME, CategorizerState.TYPE, "1", categorizerStateGetResponse1) - .prepareGet(INDEX_NAME, CategorizerState.TYPE, "2", categorizerStateGetResponse2) + .prepareGet(INDEX_NAME, CategorizerState.TYPE, JOB_ID + "_1", categorizerStateGetResponse1) + .prepareGet(INDEX_NAME, CategorizerState.TYPE, JOB_ID + "_2", categorizerStateGetResponse2) .prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_1", modelStateGetResponse1) .prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_2", modelStateGetResponse2);