honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TEST] Work around _cat/indices bug with security enabled (#47160) 

When the ML native multi-node tests use _cat/indices/_all
and the request goes to a non-master node, _all is
translated to a list of concrete indices by the authz layer
on the coordinating node before the request is forwarded
to the master node. Then it is possible for the master
node to return an index_not_found_exception if one of
the concrete indices that was expanded on the
coordinating node has been deleted in the meantime.
(#47159 has been opened to track the underlying problem.)

It has been observed that the index that gets deleted when
the problem affects the ML native multi-node tests is
always the ML notifications index. The tests that fail are
only interested in the presence or absense of ML results
indices. Therefore the workaround is to only _cat indices
that match the ML results index pattern.

Fixes #45652
This commit is contained in:
David Roberts 2019-09-26 13:19:10 +01:00
parent 0765bd4bf7
commit 77cc6d5bad
1 changed files with 27 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
View File

@ -208,7 +208,9 @@ public class MlJobIT extends ESRestTestCase {
} }
}); });
String responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(responseAsString, assertThat(responseAsString,
containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
@ -272,7 +274,8 @@ public class MlJobIT extends ESRestTestCase {
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
client().performRequest(new Request("POST", "/_refresh")); client().performRequest(new Request("POST", "/_refresh"));
@ -287,7 +290,8 @@ public class MlJobIT extends ESRestTestCase {
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
client().performRequest(new Request("POST", "/_refresh")); client().performRequest(new Request("POST", "/_refresh"));
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)));
} }
@ -394,19 +398,21 @@ public class MlJobIT extends ESRestTestCase {
"avoid the clash by assigning a dedicated results index")); "avoid the clash by assigning a dedicated results index"));
} }
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652")
public void testDeleteJob() throws Exception { public void testDeleteJob() throws Exception {
String jobId = "delete-job-job"; String jobId = "delete-job-job";
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName)); assertThat(indicesBeforeDelete, containsString(indexName));
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
// check that the index still exists (it's shared by default) // check that the index still exists (it's shared by default)
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, containsString(indexName)); assertThat(indicesAfterDelete, containsString(indexName));
waitUntilIndexIsEmpty(indexName); waitUntilIndexIsEmpty(indexName);
@ -465,13 +471,14 @@ public class MlJobIT extends ESRestTestCase {
assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404)); assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404));
} }
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652")
public void testDeleteJobAsync() throws Exception { public void testDeleteJobAsync() throws Exception {
String jobId = "delete-job-async-job"; String jobId = "delete-job-async-job";
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName)); assertThat(indicesBeforeDelete, containsString(indexName));
Response response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId Response response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId
@ -483,7 +490,8 @@ public class MlJobIT extends ESRestTestCase {
assertThat(EntityUtils.toString(taskResponse.getEntity()), containsString("\"acknowledged\":true")); assertThat(EntityUtils.toString(taskResponse.getEntity()), containsString("\"acknowledged\":true"));
// check that the index still exists (it's shared by default) // check that the index still exists (it's shared by default)
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, containsString(indexName)); assertThat(indicesAfterDelete, containsString(indexName));
waitUntilIndexIsEmpty(indexName); waitUntilIndexIsEmpty(indexName);
@ -518,7 +526,9 @@ public class MlJobIT extends ESRestTestCase {
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId); createFarequoteJob(jobId);
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName)); assertThat(indicesBeforeDelete, containsString(indexName));
// Manually delete the index so that we can test that deletion proceeds // Manually delete the index so that we can test that deletion proceeds
@ -528,7 +538,8 @@ public class MlJobIT extends ESRestTestCase {
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
// check index was deleted // check index was deleted
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, not(containsString(aliasName))); assertThat(indicesAfterDelete, not(containsString(aliasName)));
assertThat(indicesAfterDelete, not(containsString(indexName))); assertThat(indicesAfterDelete, not(containsString(indexName)));
@ -598,7 +609,9 @@ public class MlJobIT extends ESRestTestCase {
"}"); "}");
client().performRequest(extraIndex2); client().performRequest(extraIndex2);
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName)); assertThat(indicesBeforeDelete, containsString(indexName));
assertThat(indicesBeforeDelete, containsString(indexName + "-001")); assertThat(indicesBeforeDelete, containsString(indexName + "-001"));
assertThat(indicesBeforeDelete, containsString(indexName + "-002")); assertThat(indicesBeforeDelete, containsString(indexName + "-002"));
@ -637,7 +650,8 @@ public class MlJobIT extends ESRestTestCase {
client().performRequest(new Request("POST", "/_refresh")); client().performRequest(new Request("POST", "/_refresh"));
// check that the indices still exist but are empty // check that the indices still exist but are empty
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, containsString(indexName)); assertThat(indicesAfterDelete, containsString(indexName));
assertThat(indicesAfterDelete, containsString(indexName + "-001")); assertThat(indicesAfterDelete, containsString(indexName + "-001"));
assertThat(indicesAfterDelete, containsString(indexName + "-002")); assertThat(indicesAfterDelete, containsString(indexName + "-002"));