honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add job Id to state document IDs (elastic/elasticsearch#579) 

* Persist quantile documents with the jobId in the document Id

* Add job Id to snapshot Id

* Add job Id to categoriser state document Id

* Rename quantiles doc to start with job id as the other state docs do

* Fix restoring categoriser state

Original commit: elastic/x-pack-elasticsearch@3e5d3368b5
This commit is contained in:
David Kyle 2016-12-20 10:47:37 +00:00 committed by GitHub
parent 316f6bf6cb
commit 78fdca65d5
5 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java
View File

@ -18,6 +18,10 @@ public class CategorizerState {
*/ */
public static final String TYPE = "categorizer_state"; public static final String TYPE = "categorizer_state";
public static final String categorizerStateDocId(String jobId, int docNum) {
return jobId + "_" + docNum;
}
private CategorizerState() { private CategorizerState() {
} }
} }

9
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java
View File

@ -857,10 +857,11 @@ public class JobProvider {
public Optional<Quantiles> getQuantiles(String jobId) { public Optional<Quantiles> getQuantiles(String jobId) {
String indexName = AnomalyDetectorsIndex.getJobIndexName(jobId); String indexName = AnomalyDetectorsIndex.getJobIndexName(jobId);
try { try {
LOGGER.trace("ES API CALL: get ID " + Quantiles.QUANTILES_ID + String quantilesId = Quantiles.quantilesId(jobId);
" type " + Quantiles.TYPE + " from index " + indexName);
LOGGER.trace("ES API CALL: get ID {} type {} from index {}", quantilesId, Quantiles.TYPE.getPreferredName(), indexName);
GetResponse response = client.prepareGet( GetResponse response = client.prepareGet(
indexName, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID).get(); indexName, Quantiles.TYPE.getPreferredName(), quantilesId).get();
if (!response.isExists()) { if (!response.isExists()) {
LOGGER.info("There are currently no quantiles for job " + jobId); LOGGER.info("There are currently no quantiles for job " + jobId);
return Optional.empty(); return Optional.empty();
@ -982,7 +983,7 @@ public class JobProvider {
// count up until a document is not found. It's NOT an error to have no categorizer state. // count up until a document is not found. It's NOT an error to have no categorizer state.
int docNum = 0; int docNum = 0;
while (true) { while (true) {
String docId = Integer.toString(++docNum); String docId = CategorizerState.categorizerStateDocId(jobId, ++docNum);
LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, CategorizerState.TYPE, indexName); LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, CategorizerState.TYPE, indexName);

5
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java
View File

@ -209,7 +209,7 @@ public class JobResultsPersister extends AbstractComponent {
*/ */
public void persistQuantiles(Quantiles quantiles) { public void persistQuantiles(Quantiles quantiles) {
Persistable persistable = new Persistable(quantiles.getJobId(), quantiles, Quantiles.TYPE.getPreferredName(), Persistable persistable = new Persistable(quantiles.getJobId(), quantiles, Quantiles.TYPE.getPreferredName(),
Quantiles.QUANTILES_ID); Quantiles.quantilesId(quantiles.getJobId()));
if (persistable.persist()) { if (persistable.persist()) {
// Refresh the index when persisting quantiles so that previously // Refresh the index when persisting quantiles so that previously
// persisted results will be available for searching. Do this using the // persisted results will be available for searching. Do this using the
@ -282,8 +282,7 @@ public class JobResultsPersister extends AbstractComponent {
/** /**
* Once all the job data has been written this function will be * Once all the job data has been written this function will be
* called to commit the data if the implementing persister requires * called to commit the writes to the datastore.
* it.
* *
* @return True if successful * @return True if successful
*/ */

5
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java
View File

@ -24,7 +24,6 @@ import java.util.Objects;
* Quantiles Result POJO * Quantiles Result POJO
*/ */
public class Quantiles extends ToXContentToBytes implements Writeable { public class Quantiles extends ToXContentToBytes implements Writeable {
public static final String QUANTILES_ID = "hierarchical";
/** /**
* Field Names * Field Names
@ -46,6 +45,10 @@ public class Quantiles extends ToXContentToBytes implements Writeable {
PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE);
} }
public static String quantilesId(String jobId) {
return jobId + "-" + TYPE.getPreferredName();
}
private final String jobId; private final String jobId;
private final Date timestamp; private final Date timestamp;
private final String quantileState; private final String quantileState;

12
elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderTests.java
View File

@ -74,7 +74,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID); .throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID));
JobProvider provider = createProvider(clientBuilder.build()); JobProvider provider = createProvider(clientBuilder.build());
@ -86,7 +86,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID), getResponse);
JobProvider provider = createProvider(clientBuilder.build()); JobProvider provider = createProvider(clientBuilder.build());
@ -104,7 +104,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID), getResponse);
JobProvider provider = createProvider(clientBuilder.build()); JobProvider provider = createProvider(clientBuilder.build());
@ -123,7 +123,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId("foo"), getResponse);
JobProvider provider = createProvider(clientBuilder.build()); JobProvider provider = createProvider(clientBuilder.build());
@ -1077,8 +1077,8 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true) .addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, CategorizerState.TYPE, "1", categorizerStateGetResponse1) .prepareGet(INDEX_NAME, CategorizerState.TYPE, JOB_ID + "_1", categorizerStateGetResponse1)
.prepareGet(INDEX_NAME, CategorizerState.TYPE, "2", categorizerStateGetResponse2) .prepareGet(INDEX_NAME, CategorizerState.TYPE, JOB_ID + "_2", categorizerStateGetResponse2)
.prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_1", modelStateGetResponse1) .prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_1", modelStateGetResponse1)
.prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_2", modelStateGetResponse2); .prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_2", modelStateGetResponse2);