honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add job Id to state document IDs (elastic/elasticsearch#579) 

* Persist quantile documents with the jobId in the document Id

* Add job Id to snapshot Id

* Add job Id to categoriser state document Id

* Rename quantiles doc to start with job id as the other state docs do

* Fix restoring categoriser state

Original commit: elastic/x-pack-elasticsearch@3e5d3368b5
This commit is contained in:
David Kyle 2016-12-20 10:47:37 +00:00 committed by GitHub
parent 316f6bf6cb
commit 78fdca65d5
5 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java
View File

@ -18,6 +18,10 @@ public class CategorizerState {
*/
public static final String TYPE = "categorizer_state";
public static final String categorizerStateDocId(String jobId, int docNum) {
return jobId + "_" + docNum;
}
private CategorizerState() {
}
}

9
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java
View File

@ -857,10 +857,11 @@ public class JobProvider {
public Optional<Quantiles> getQuantiles(String jobId) {
String indexName = AnomalyDetectorsIndex.getJobIndexName(jobId);
try {
LOGGER.trace("ES API CALL: get ID " + Quantiles.QUANTILES_ID +
" type " + Quantiles.TYPE + " from index " + indexName);
String quantilesId = Quantiles.quantilesId(jobId);
LOGGER.trace("ES API CALL: get ID {} type {} from index {}", quantilesId, Quantiles.TYPE.getPreferredName(), indexName);
GetResponse response = client.prepareGet(
indexName, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID).get();
indexName, Quantiles.TYPE.getPreferredName(), quantilesId).get();
if (!response.isExists()) {
LOGGER.info("There are currently no quantiles for job " + jobId);
return Optional.empty();
@ -982,7 +983,7 @@ public class JobProvider {
// count up until a document is not found. It's NOT an error to have no categorizer state.
int docNum = 0;
while (true) {
String docId = Integer.toString(++docNum);
String docId = CategorizerState.categorizerStateDocId(jobId, ++docNum);
LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, CategorizerState.TYPE, indexName);

5
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java
View File

@ -209,7 +209,7 @@ public class JobResultsPersister extends AbstractComponent {
*/
public void persistQuantiles(Quantiles quantiles) {
Persistable persistable = new Persistable(quantiles.getJobId(), quantiles, Quantiles.TYPE.getPreferredName(),
Quantiles.QUANTILES_ID);
Quantiles.quantilesId(quantiles.getJobId()));
if (persistable.persist()) {
// Refresh the index when persisting quantiles so that previously
// persisted results will be available for searching. Do this using the
@ -282,8 +282,7 @@ public class JobResultsPersister extends AbstractComponent {
/**
* Once all the job data has been written this function will be
* called to commit the data if the implementing persister requires
* it.
* called to commit the writes to the datastore.
*
* @return True if successful
*/

5
elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java
View File

@ -24,7 +24,6 @@ import java.util.Objects;
* Quantiles Result POJO
*/
public class Quantiles extends ToXContentToBytes implements Writeable {
public static final String QUANTILES_ID = "hierarchical";
/**
* Field Names
@ -46,6 +45,10 @@ public class Quantiles extends ToXContentToBytes implements Writeable {
PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE);
}
public static String quantilesId(String jobId) {
return jobId + "-" + TYPE.getPreferredName();
}
private final String jobId;
private final Date timestamp;
private final String quantileState;

12
elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderTests.java
View File

@ -74,7 +74,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID);
.throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID));
JobProvider provider = createProvider(clientBuilder.build());
@ -86,7 +86,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse);
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID), getResponse);
JobProvider provider = createProvider(clientBuilder.build());
@ -104,7 +104,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse);
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId(JOB_ID), getResponse);
JobProvider provider = createProvider(clientBuilder.build());
@ -123,7 +123,7 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse);
.prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.quantilesId("foo"), getResponse);
JobProvider provider = createProvider(clientBuilder.build());
@ -1077,8 +1077,8 @@ public class JobProviderTests extends ESTestCase {
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse()
.addIndicesExistsResponse(JobProvider.PRELERT_USAGE_INDEX, true)
.prepareGet(INDEX_NAME, CategorizerState.TYPE, "1", categorizerStateGetResponse1)
.prepareGet(INDEX_NAME, CategorizerState.TYPE, "2", categorizerStateGetResponse2)
.prepareGet(INDEX_NAME, CategorizerState.TYPE, JOB_ID + "_1", categorizerStateGetResponse1)
.prepareGet(INDEX_NAME, CategorizerState.TYPE, JOB_ID + "_2", categorizerStateGetResponse2)
.prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_1", modelStateGetResponse1)
.prepareGet(INDEX_NAME, ModelState.TYPE.getPreferredName(), "123_2", modelStateGetResponse2);