fixes to allow bad apple tests to pass

This commit fixes the bad apple tests that failed when running them. The
IndexAuditTrailEnabledTest was removed and the test was folded into the
IndexAuditIT. Some watcher tests that relied on mustache were moved
into the QA tests with the mustache plugin.

Additionally, fixing these tests uncovered a issue with the privileges needed
for writing data into an index. If the mappings need to be updated because
of a write, then the update mapping action gets executed. In 2.x this was
handled by the system user, but now is executed under the user's context,
which is the correct thing to do. The update mapping action is now added to
the read, index, crud, and write privileges for an index.

Original commit: elastic/x-pack-elasticsearch@30711f9625
This commit is contained in:
jaymode 2016-01-29 10:11:23 -05:00
parent 7f5349db57
commit aa2eb15f31
11 changed files with 118 additions and 154 deletions

View File

@ -3,7 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.watcher.test.integration;
package org.elasticsearch.messy.tests;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.elasticsearch.ExceptionsHelper;
@ -18,6 +18,9 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.discovery.zen.ping.ZenPing;
import org.elasticsearch.discovery.zen.ping.ZenPingService;
import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.MockMustacheScriptEngine;
import org.elasticsearch.script.mustache.MustachePlugin;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.SuppressLocalMode;
import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
@ -33,6 +36,10 @@ import org.elasticsearch.watcher.test.WatcherTestUtils;
import org.elasticsearch.watcher.transport.actions.delete.DeleteWatchResponse;
import org.elasticsearch.watcher.transport.actions.stats.WatcherStatsResponse;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@ -81,6 +88,23 @@ public class NoMasterNodeTests extends AbstractWatcherIntegrationTestCase {
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Collection<Class<? extends Plugin>> types = new ArrayList<>();
types.addAll(super.nodePlugins());
// TODO remove dependency on mustache
types.add(MustachePlugin.class);
return types;
}
@Override
protected Collection<Class<? extends Plugin>> getMockPlugins() {
Set<Class<? extends Plugin>> plugins = new HashSet<>(super.getMockPlugins());
// remove the mock because we use mustache here...
plugins.remove(MockMustacheScriptEngine.TestPlugin.class);
return plugins;
}
public void testSimpleFailure() throws Exception {
// we need 3 hosts here because we stop the master and start another - it doesn't restart the pre-existing node...
config = new ClusterDiscoveryConfiguration.UnicastZen(3, Settings.EMPTY);

View File

@ -3,7 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.watcher.transport.action.ack;
package org.elasticsearch.messy.tests;
import org.apache.lucene.util.LuceneTestCase.BadApple;
@ -13,6 +13,9 @@ import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.MockMustacheScriptEngine;
import org.elasticsearch.script.mustache.MustachePlugin;
import org.elasticsearch.watcher.actions.ActionStatus;
import org.elasticsearch.watcher.client.WatcherClient;
import org.elasticsearch.watcher.condition.compare.CompareCondition;
@ -29,6 +32,10 @@ import org.elasticsearch.watcher.watch.Watch;
import org.elasticsearch.watcher.watch.WatchStore;
import org.hamcrest.Matchers;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@ -52,6 +59,24 @@ import static org.hamcrest.core.IsEqual.equalTo;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
public class WatchAckTests extends AbstractWatcherIntegrationTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Collection<Class<? extends Plugin>> types = new ArrayList<>();
types.addAll(super.nodePlugins());
// TODO remove dependency on mustache
types.add(MustachePlugin.class);
return types;
}
@Override
protected Collection<Class<? extends Plugin>> getMockPlugins() {
Set<Class<? extends Plugin>> plugins = new HashSet<>(super.getMockPlugins());
// remove the mock because we use mustache here...
plugins.remove(MockMustacheScriptEngine.TestPlugin.class);
return plugins;
}
private IndexResponse indexTestDoc() {
createIndex("actions", "events");
ensureGreen("actions", "events");

View File

@ -5,10 +5,14 @@
*/
package org.elasticsearch.shield.audit;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.shield.audit.index.IndexAuditTrail;
import org.elasticsearch.shield.authc.support.SecuredString;
import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
import org.elasticsearch.test.ESIntegTestCase;
@ -49,6 +53,35 @@ public class IndexAuditIT extends ESIntegTestCase {
assertThat((String) searchResponse.getHits().getAt(0).sourceAsMap().get("principal"), is(USER));
}
public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
// this is already "tested" by the test framework since we wipe the templates before and after,
// but lets be explicit about the behavior
awaitIndexTemplateCreation();
// delete the template
DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
.prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
assertThat(deleteResponse.isAcknowledged(), is(true));
awaitIndexTemplateCreation();
}
private void awaitIndexTemplateCreation() throws InterruptedException {
boolean found = awaitBusy(() -> {
GetIndexTemplatesResponse response = client().admin().indices()
.prepareGetTemplates(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
if (response.getIndexTemplates().size() > 0) {
for (IndexTemplateMetaData indexTemplateMetaData : response.getIndexTemplates()) {
if (IndexAuditTrail.INDEX_TEMPLATE_NAME.equals(indexTemplateMetaData.name())) {
return true;
}
}
}
return false;
});
assertThat("index template [" + IndexAuditTrail.INDEX_TEMPLATE_NAME + "] was not created", found, is(true));
}
@Override
protected Settings externalClusterClientSettings() {
return Settings.builder()

View File

@ -32,16 +32,18 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege<IndexPrivilege> {
public static final IndexPrivilege CREATE_INDEX = new IndexPrivilege("create_index", CreateIndexAction.NAME);
public static final IndexPrivilege MANAGE_ALIASES = new IndexPrivilege("manage_aliases", "indices:admin/aliases*");
public static final IndexPrivilege MONITOR = new IndexPrivilege("monitor", "indices:monitor/*");
public static final IndexPrivilege DATA_ACCESS = new IndexPrivilege("data_access", "indices:data/*");
public static final IndexPrivilege CRUD = new IndexPrivilege("crud", "indices:data/write/*", "indices:data/read/*");
public static final IndexPrivilege DATA_ACCESS = new IndexPrivilege("data_access", "indices:data/*", "indices:admin/mapping/put");
public static final IndexPrivilege CRUD =
new IndexPrivilege("crud", "indices:data/write/*", "indices:data/read/*", "indices:admin/mapping/put");
public static final IndexPrivilege READ = new IndexPrivilege("read", "indices:data/read/*");
public static final IndexPrivilege SEARCH = new IndexPrivilege("search", SearchAction.NAME + "*", MultiSearchAction.NAME + "*",
SuggestAction.NAME + "*");
public static final IndexPrivilege SEARCH =
new IndexPrivilege("search", SearchAction.NAME + "*", MultiSearchAction.NAME + "*", SuggestAction.NAME + "*");
public static final IndexPrivilege GET = new IndexPrivilege("get", GetAction.NAME + "*", MultiGetAction.NAME + "*");
public static final IndexPrivilege SUGGEST = new IndexPrivilege("suggest", SuggestAction.NAME + "*");
public static final IndexPrivilege INDEX = new IndexPrivilege("index", "indices:data/write/index*", "indices:data/write/update*");
public static final IndexPrivilege INDEX =
new IndexPrivilege("index", "indices:data/write/index*", "indices:data/write/update*", "indices:admin/mapping/put");
public static final IndexPrivilege DELETE = new IndexPrivilege("delete", "indices:data/write/delete*");
public static final IndexPrivilege WRITE = new IndexPrivilege("write", "indices:data/write/*");
public static final IndexPrivilege WRITE = new IndexPrivilege("write", "indices:data/write/*", "indices:admin/mapping/put");
private static final Set<IndexPrivilege> values = new CopyOnWriteArraySet<>();

View File

@ -22,8 +22,7 @@ public class SystemPrivilege extends Privilege<SystemPrivilege> {
"internal:*",
"indices:monitor/*", // added for marvel
"cluster:monitor/*", // added for marvel
"cluster:admin/reroute", // added for DiskThresholdDecider.DiskListener
"indices:admin/mapping/put" // ES 2.0 MappingUpdatedAction - updateMappingOnMasterSynchronously
"cluster:admin/reroute" // added for DiskThresholdDecider.DiskListener
));
SystemPrivilege() {

View File

@ -8,10 +8,11 @@ package org.elasticsearch.integration;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.rest.client.http.HttpResponse;
import org.junit.Before;
import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@ -21,6 +22,7 @@ import static org.hamcrest.Matchers.is;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false)
public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
private String jsonDoc = "{ \"name\" : \"elasticsearch\"}";
@ -94,8 +96,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
"u11:" + USERS_PASSWD_HASHED + "\n" +
"u12:" + USERS_PASSWD_HASHED + "\n" +
"u13:" + USERS_PASSWD_HASHED + "\n" +
"u14:" + USERS_PASSWD_HASHED + "\n" +
"u15:" + USERS_PASSWD_HASHED + "\n";
"u14:" + USERS_PASSWD_HASHED + "\n";
public static final String USERS_ROLES =
"all_indices_role:admin,u8\n" +
@ -120,7 +121,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder().put(super.nodeSettings(nodeOrdinal))
.put(NetworkModule.HTTP_ENABLED.getKey(), true)
.put("action.disable_shutdown", true)
.build();
}
@ -139,6 +139,17 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
return super.configUsersRoles() + USERS_ROLES;
}
// we reduce the number of shards and replicas to help speed up this test since that is not the focus of this test
@Override
public int maximumNumberOfReplicas() {
return 1;
}
@Override
public int maximumNumberOfShards() {
return 2;
}
@Before
public void insertBaseDocumentsAsAdmin() throws Exception {
// indices: a,b,c,abc
@ -296,24 +307,13 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertUserIsDenied("u14", "all", "c");
}
public void testUserU15() throws Exception {
//u15 has access to manage and search a, so that adding warmer templates work
assertUserIsAllowed("u15", "manage", "a");
assertUserIsAllowed("u15", "search", "a");
assertAccessIsAllowed("u15", "PUT", "/a/_warmer/w1", "{ \"query\" : { \"match_all\" : {} } }");
assertAccessIsAllowed("u15", "DELETE", "/a/_warmer/w1");
assertUserIsDenied("u15", "all", "b");
assertUserIsDenied("u15", "all", "c");
}
public void testThatUnknownUserIsRejectedProperly() throws Exception {
HttpResponse response = executeRequest("idonotexist", "GET", "/", null, new HashMap<>());
assertThat(response.getStatusCode(), is(401));
}
private void assertUserExecutes(String user, String action, String index, boolean userIsAllowed) throws Exception {
Map<String, String> refreshParams = singletonMap("refresh", "true");
Map<String, String> refreshParams = Collections.emptyMap();//singletonMap("refresh", "true");
switch (action) {
case "all" :
@ -353,11 +353,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertAccessIsAllowed("admin", "PUT", "/" + index + "/foo/1", jsonDoc, refreshParams);
client().admin().cluster().prepareHealth(index).setWaitForGreenStatus().get();
assertAccessIsAllowed(user, "GET", "/" + index + "/_mapping/foo/field/name");
// putting warmers only works if the user is allowed to search as well, as the query gets validated, added an own
// test for this
assertAccessIsAllowed("admin", "PUT", "/" + index + "/_warmer/w1", "{ \"query\" : { \"match_all\" : {} } }");
assertAccessIsAllowed(user, "GET", "/" + index + "/_warmer/w1");
assertAccessIsAllowed(user, "DELETE", "/" + index + "/_warmer/w1");
assertAccessIsAllowed(user, "GET", "/" + index + "/_settings");
} else {
assertAccessIsDenied(user, "DELETE", "/" + index);
@ -372,9 +367,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertAccessIsDenied(user, "POST", "/" + index + "/_open");
assertAccessIsDenied(user, "POST", "/" + index + "/_cache/clear");
assertAccessIsDenied(user, "GET", "/" + index + "/_mapping/foo/field/name");
assertAccessIsDenied(user, "PUT", "/" + index + "/_warmer/w1", "{ \"query\" : { \"match_all\" : {} } }");
assertAccessIsDenied(user, "GET", "/" + index + "/_warmer/w1");
assertAccessIsDenied(user, "DELETE", "/" + index + "/_warmer/w1");
assertAccessIsDenied(user, "GET", "/" + index + "/_settings");
}
break;

View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.integration;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@ -33,8 +32,6 @@ import static org.hamcrest.Matchers.hasSize;
* actions that are normally categorized as index actions as cluster actions - for example,
* index template actions.
*/
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
public class PermissionPrecedenceTests extends ShieldIntegTestCase {
protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("test123".toCharArray())));

View File

@ -1,108 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.shield.audit.index;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.ShieldIntegTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.util.Collections;
import java.util.Set;
import static org.hamcrest.Matchers.is;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ClusterScope(scope = Scope.TEST, randomDynamicTemplates = false)
public class IndexAuditTrailEnabledTests extends ShieldIntegTestCase {
IndexNameResolver.Rollover rollover = randomFrom(IndexNameResolver.Rollover.values());
@Override
protected Set<String> excludeTemplates() {
return Collections.singleton(IndexAuditTrail.INDEX_TEMPLATE_NAME);
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()
.put(super.nodeSettings(nodeOrdinal));
builder.put("shield.audit.enabled", true);
if (randomBoolean()) {
builder.putArray("shield.audit.outputs", LoggingAuditTrail.NAME, IndexAuditTrail.NAME);
} else {
builder.putArray("shield.audit.outputs", IndexAuditTrail.NAME);
}
builder.put(IndexAuditTrail.ROLLOVER_SETTING, rollover);
return builder.build();
}
@Override
public void beforeIndexDeletion() {
// For this test, this is a NO-OP because the index audit trail will continue to capture events and index after
// the tests have completed. The default implementation of this method expects that nothing is performing operations
// after the test has completed
}
public void testAuditTrailIndexAndTemplateExists() throws Exception {
awaitIndexTemplateCreation();
// Wait for the index to be created since we have our own startup
awaitAuditDocumentCreation();
}
public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
// this is already "tested" by the test framework since we wipe the templates before and after, but lets be explicit about the
// behavior
awaitIndexTemplateCreation();
// delete the template
DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
.prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
assertThat(deleteResponse.isAcknowledged(), is(true));
awaitIndexTemplateCreation();
}
void awaitAuditDocumentCreation() throws Exception {
final String indexName = IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, DateTime.now(DateTimeZone.UTC), rollover);
boolean success = awaitBusy(() -> {
try {
SearchResponse searchResponse = client().prepareSearch(indexName).setSize(0).setTerminateAfter(1).execute().actionGet();
return searchResponse.getHits().totalHits() > 0;
} catch (Exception e) {
return false;
}
});
assertThat("no audit document exists!", success, is(true));
}
void awaitIndexTemplateCreation() throws InterruptedException {
boolean found = awaitBusy(() -> {
GetIndexTemplatesResponse response = client().admin().indices()
.prepareGetTemplates(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
if (response.getIndexTemplates().size() > 0) {
for (IndexTemplateMetaData indexTemplateMetaData : response.getIndexTemplates()) {
if (IndexAuditTrail.INDEX_TEMPLATE_NAME.equals(indexTemplateMetaData.name())) {
return true;
}
}
}
return false;
});
assertThat("index template [" + IndexAuditTrail.INDEX_TEMPLATE_NAME + "] was not created", found, is(true));
}
}

View File

@ -177,7 +177,8 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
}
};
cluster2 = new InternalTestCluster("network", randomLong(), createTempDir(), numNodes, numNodes, cluster2Name,
cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(), Function.identity());
cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(),
useShield ? getClientWrapper() : Function.identity());
cluster2.beforeTest(getRandom(), 0.5);
remoteClient = cluster2.client();

View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.shield.authc.support;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.shield.User;
@ -25,8 +24,6 @@ import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.sameInstance;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
public class CachingUsernamePasswordRealmTests extends ESTestCase {
private Settings globalSettings;
@ -206,7 +203,9 @@ public class CachingUsernamePasswordRealmTests extends ESTestCase {
};
final CountDownLatch latch = new CountDownLatch(1);
final int numberOfThreads = randomIntBetween(8, 24);
final int numberOfProcessors = Runtime.getRuntime().availableProcessors();
final int numberOfThreads = scaledRandomIntBetween((numberOfProcessors + 1) / 2, numberOfProcessors * 3);
final int numberOfIterations = scaledRandomIntBetween(20, 100);
List<Thread> threads = new ArrayList<>();
for (int i = 0; i < numberOfThreads; i++) {
final boolean invalidPassword = randomBoolean();
@ -215,9 +214,9 @@ public class CachingUsernamePasswordRealmTests extends ESTestCase {
public void run() {
try {
latch.await();
for (int i = 0; i < 100; i++) {
User user = realm.authenticate(
new UsernamePasswordToken(username, invalidPassword ? randomPassword : password));
for (int i = 0; i < numberOfIterations; i++) {
UsernamePasswordToken token = new UsernamePasswordToken(username, invalidPassword ? randomPassword : password);
User user = realm.authenticate(token);
if (invalidPassword && user != null) {
throw new RuntimeException("invalid password led to an authenticated user: " + user.toString());
} else if (invalidPassword == false && user == null) {

View File

@ -245,7 +245,7 @@ public class PrivilegeTests extends ESTestCase {
assertThat(predicate.test("whatever"), is(false));
assertThat(predicate.test("cluster:admin/reroute"), is(true));
assertThat(predicate.test("cluster:admin/whatever"), is(false));
assertThat(predicate.test("indices:admin/mapping/put"), is(true));
assertThat(predicate.test("indices:admin/mapping/put"), is(false));
assertThat(predicate.test("indices:admin/mapping/whatever"), is(false));
}