diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolver.java index 685b7bee03a..ca88b019f74 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolver.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolver.java @@ -56,9 +56,27 @@ public class NodeStatsResolver extends MonitoringIndexNameResolver.Timestamped waiting for documents to be collected"); awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), ClusterStateNodeResolver.TYPE); logger.debug("--> searching for monitoring documents of type [{}]", ClusterStateNodeResolver.TYPE); - SearchResponse response = client().prepareSearch().setTypes(ClusterStateNodeResolver.TYPE).get(); + SearchResponse response = client().prepareSearch(timestampedIndex).setTypes(ClusterStateNodeResolver.TYPE).get(); assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes)); logger.debug("--> checking that every document contains the expected fields"); @@ -133,10 +140,10 @@ public class ClusterStateTests extends MarvelIntegTestCase { logger.debug("--> check that node attributes are indexed"); assertThat(client().prepareSearch().setSize(0) - .setIndices(MONITORING_INDICES_PREFIX + TEMPLATE_VERSION + "-*") + .setIndices(timestampedIndex) .setTypes(ClusterStateNodeResolver.TYPE) - .setQuery(QueryBuilders.matchQuery(SOURCE_NODE.underscore().toString() + ".attributes.custom", randomInt) - ).get().getHits().getTotalHits(), greaterThan(0L)); + .setQuery(QueryBuilders.matchQuery(SOURCE_NODE.underscore().toString() + ".attributes.custom", randomInt)) + .get().getHits().getTotalHits(), greaterThan(0L)); logger.debug("--> cluster state nodes successfully collected"); } @@ -144,7 +151,7 @@ public class ClusterStateTests extends MarvelIntegTestCase { public void testDiscoveryNodes() throws Exception { final long nbNodes = internalCluster().size(); - MonitoringIndexNameResolver.Data dataResolver = internalCluster().getInstance(MonitoringIndexNameResolver.Data.class); + MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(TEMPLATE_VERSION); assertNotNull(dataResolver); String dataIndex = dataResolver.indexPattern(); @@ -154,7 +161,7 @@ public class ClusterStateTests extends MarvelIntegTestCase { awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), DiscoveryNodeResolver.TYPE); logger.debug("--> searching for monitoring documents of type [{}]", DiscoveryNodeResolver.TYPE); - SearchResponse response = client().prepareSearch().setTypes(DiscoveryNodeResolver.TYPE).get(); + SearchResponse response = client().prepareSearch(dataIndex).setTypes(DiscoveryNodeResolver.TYPE).get(); assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes)); logger.debug("--> checking that every document contains the expected fields"); @@ -188,7 +195,7 @@ public class ClusterStateTests extends MarvelIntegTestCase { assertThat(client().prepareGet(dataIndex, DiscoveryNodeResolver.TYPE, nodeId).get().isExists(), is(true)); // checks that document is not indexed - assertHitCount(client().prepareSearch().setSize(0) + assertHitCount(client().prepareSearch(dataIndex).setSize(0) .setTypes(DiscoveryNodeResolver.TYPE) .setQuery(QueryBuilders.boolQuery() .should(matchQuery("node.id", nodeId)) diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java index 1eb80d4a59d..f4257969523 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.monitor.os.OsProbe; import org.elasticsearch.monitor.process.ProcessProbe; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolStats; +import org.elasticsearch.watcher.execution.InternalWatchExecutor; import java.io.IOException; import java.nio.file.Path; @@ -124,9 +125,14 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase< }; Map> statsByShard = new HashMap<>(); statsByShard.put(index, Collections.singletonList(new IndexShardStats(shardId, new ShardStats[]{shardStats}))); - List threadPoolStats = Arrays.asList(new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0), + List threadPoolStats = Arrays.asList( new ThreadPoolStats.Stats(ThreadPool.Names.BULK, 0, 0, 0, 0, 0, 0), - new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0) + new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, 0, 0, 0, 0, 0, 0), + new ThreadPoolStats.Stats(ThreadPool.Names.GET, 0, 0, 0, 0, 0, 0), + new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0), + new ThreadPoolStats.Stats(ThreadPool.Names.MANAGEMENT, 0, 0, 0, 0, 0, 0), + new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0), + new ThreadPoolStats.Stats(InternalWatchExecutor.THREAD_POOL_NAME, 0, 0, 0, 0, 0, 0) ); return new NodeStats(new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, new NodeIndicesStats(new CommonStats(), statsByShard), OsProbe.getInstance().osStats(), diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java index 08e09dd3232..3ec5fb69026 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java @@ -11,13 +11,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; import org.elasticsearch.marvel.agent.exporter.local.LocalExporter; -import org.elasticsearch.marvel.agent.resolver.node.NodeStatsResolver; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.junit.After; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -63,21 +63,37 @@ public class NodeStatsTests extends MarvelIntegTestCase { assertThat(response.getHits().getTotalHits(), greaterThan(0L)); logger.debug("--> checking that every document contains the expected fields"); - String[] filters = NodeStatsResolver.FILTERS; + for (SearchHit searchHit : response.getHits().getHits()) { Map fields = searchHit.sourceAsMap(); - for (String filter : filters) { + for (String filter : nodeStatsFilters(watcherEnabled)) { if (Constants.WINDOWS) { // load average is unavailable on Windows if ("node_stats.os.cpu.load_average.1m".equals(filter)) { continue; } } + assertContains(filter, fields); } } logger.debug("--> node stats successfully collected"); } + + /** + * Optionally exclude {@link NodeStatsResolver#FILTERS} that require Watcher to be enabled. + * + * @param includeWatcher {@code true} to keep watcher filters. + * @return Never {@code null} or empty. + * @see #watcherEnabled + */ + private static String[] nodeStatsFilters(boolean includeWatcher) { + if (includeWatcher) { + return NodeStatsResolver.FILTERS; + } + + return Arrays.stream(NodeStatsResolver.FILTERS).filter(s -> s.contains("watcher") == false).toArray(String[]::new); + } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index b7af75d0320..f4cc408a1b1 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -68,14 +68,29 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { public static final String MONITORING_INDICES_PREFIX = MonitoringIndexNameResolver.PREFIX + MonitoringIndexNameResolver.DELIMITER; - protected static Boolean shieldEnabled; + /** + * Enables individual tests to control the behavior. + *

+ * Control this by overriding {@link #enableShield()}, which defaults to enabling it randomly. + */ + protected Boolean shieldEnabled; + /** + * Enables individual tests to control the behavior. + *

+ * Control this by overriding {@link #enableWatcher()}, which defaults to enabling it randomly. + */ + protected Boolean watcherEnabled; @Override protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { if (shieldEnabled == null) { shieldEnabled = enableShield(); } + if (watcherEnabled == null) { + watcherEnabled = enableWatcher(); + } logger.info("--> shield {}", shieldEnabled ? "enabled" : "disabled"); + logger.debug("--> watcher {}", watcherEnabled ? "enabled" : "disabled"); return super.buildTestCluster(scope, seed); } @@ -83,9 +98,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - - //TODO: for now lets isolate monitoring tests from watcher (randomize this later) - .put(XPackPlugin.featureEnabledSetting(Watcher.NAME), false) + .put(XPackPlugin.featureEnabledSetting(Watcher.NAME), watcherEnabled) // we do this by default in core, but for monitoring this isn't needed and only adds noise. .put("index.store.mock.check_index_on_close", false); @@ -151,12 +164,19 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { } /** - * Override and returns {@code false} to force running without shield + * Override and return {@code false} to force running without Security. */ protected boolean enableShield() { return randomBoolean(); } + /** + * Override and return {@code false} to force running without Watcher. + */ + protected boolean enableWatcher() { + return randomBoolean(); + } + protected void stopCollection() { for (AgentService agent : internalCluster().getInstances(AgentService.class)) { agent.stopCollection(); @@ -340,6 +360,15 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { * it recurses to check if 'bar' exists in the sub-map. */ protected void assertContains(String field, Map values) { + assertContains(field, values, null); + } + + /** + * Checks if a field exist in a map of values. If the field contains a dot like 'foo.bar' + * it checks that 'foo' exists in the map of values and that it points to a sub-map. Then + * it recurses to check if 'bar' exists in the sub-map. + */ + protected void assertContains(String field, Map values, String parent) { assertNotNull("field name should not be null", field); assertNotNull("values map should not be null", values); @@ -351,21 +380,30 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { assertTrue(Strings.hasText(segment)); boolean fieldExists = values.containsKey(segment); - assertTrue("expecting field [" + segment + "] to be present in monitoring document", fieldExists); + assertTrue("expecting field [" + rebuildName(parent, segment) + "] to be present in monitoring document", fieldExists); Object value = values.get(segment); String next = field.substring(point + 1); if (next.length() > 0) { assertTrue(value instanceof Map); - assertContains(next, (Map) value); + assertContains(next, (Map) value, rebuildName(parent, segment)); } else { assertFalse(value instanceof Map); } } else { - assertTrue("expecting field [" + field + "] to be present in monitoring document", values.containsKey(field)); + assertTrue("expecting field [" + rebuildName(parent, field) + "] to be present in monitoring document", + values.containsKey(field)); } } + private String rebuildName(String parent, String field) { + if (Strings.isEmpty(parent)) { + return field; + } + + return parent + "." + field; + } + protected void updateMarvelInterval(long value, TimeUnit timeUnit) { assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings( Settings.builder().put(MarvelSettings.INTERVAL.getKey(), value, timeUnit))); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java index dde8de82277..8dd3ccbf4e5 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java @@ -22,10 +22,10 @@ import static org.hamcrest.Matchers.is; //test is just too slow, please fix it to not be sleep-based @BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") -@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false) +@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false, maxNumDataNodes = 2) public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { - private String jsonDoc = "{ \"name\" : \"elasticsearch\"}"; + private String jsonDoc = "{ \"name\" : \"elasticsearch\", \"body\": \"foo bar\" }"; public static final String ROLES = "all_cluster_role:\n" + @@ -42,6 +42,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { " indices:\n" + " - names: 'a'\n" + " privileges: [ read ]\n" + + "read_b_role:\n" + + " indices:\n" + + " - names: 'b'\n" + + " privileges: [ read ]\n" + "write_a_role:\n" + " indices:\n" + " - names: 'a'\n" + @@ -50,14 +54,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { " indices:\n" + " - names: [ 'a', 'b' ]\n" + " privileges: [ read ]\n" + - "get_b_role:\n" + - " indices:\n" + - " - names: 'b'\n" + - " privileges: [ get ]\n" + - "search_b_role:\n" + - " indices:\n" + - " - names: 'b'\n" + - " privileges: [ search ]\n" + "all_regex_ab_role:\n" + " indices:\n" + " - names: '/a|b/'\n" + @@ -66,10 +62,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { " indices:\n" + " - names: 'a*'\n" + " privileges: [ manage ]\n" + - "data_access_all_role:\n" + + "read_write_all_role:\n" + " indices:\n" + " - names: '*'\n" + - " privileges: [ data_access ]\n" + + " privileges: [ read, write ]\n" + "create_c_role:\n" + " indices:\n" + " - names: 'c'\n" + @@ -78,10 +74,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { " indices:\n" + " - names: 'b'\n" + " privileges: [ monitor ]\n" + - "crud_a_role:\n" + + "read_write_a_role:\n" + " indices:\n" + " - names: 'a'\n" + - " privileges: [ crud ]\n" + + " privileges: [ read, write ]\n" + "delete_b_role:\n" + " indices:\n" + " - names: 'b'\n" + @@ -90,10 +86,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { " indices:\n" + " - names: 'a'\n" + " privileges: [ index ]\n" + - "search_a_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ search ]\n" + "\n"; public static final String USERS = @@ -107,7 +99,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { "u7:" + USERS_PASSWD_HASHED + "\n"+ "u8:" + USERS_PASSWD_HASHED + "\n"+ "u9:" + USERS_PASSWD_HASHED + "\n" + - "u10:" + USERS_PASSWD_HASHED + "\n" + "u11:" + USERS_PASSWD_HASHED + "\n" + "u12:" + USERS_PASSWD_HASHED + "\n" + "u13:" + USERS_PASSWD_HASHED + "\n" + @@ -118,19 +109,17 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { "all_cluster_role:admin\n" + "all_a_role:u1,u2,u6\n" + "read_a_role:u1,u5,u14\n" + + "read_b_role:u3,u5,u6,u8,u13\n" + "write_a_role:u9\n" + "read_ab_role:u2,u4,u9\n" + - "get_b_role:u3,u5,u8,u10\n" + - "search_b_role:u6,u10,u13\n" + "all_regex_ab_role:u3\n" + - "manage_starts_with_a_role:u4,u15\n" + - "data_access_all_role:u12\n" + + "manage_starts_with_a_role:u4\n" + + "read_write_all_role:u12\n" + "create_c_role:u11\n" + "monitor_b_role:u14\n" + - "crud_a_role:u12\n" + + "read_write_a_role:u12\n" + "delete_b_role:u11\n" + - "index_a_role:u13\n" + - "search_a_role:u15\n" ; + "index_a_role:u13\n"; @Override protected Settings nodeSettings(int nodeOrdinal) { @@ -193,7 +182,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { } public void testUserU3() throws Exception { - // u3 has get b role, but all access to a* and b* via regex + // u3 has read b role, but all access to a* and b* via regex assertUserIsAllowed("u3", "all", "a"); assertUserIsAllowed("u3", "all", "b"); assertUserIsDenied("u3", "all", "c"); @@ -216,21 +205,20 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { } public void testUserU5() throws Exception { - // u5 may read a and get b + // u5 may read a and read b assertUserIsAllowed("u5", "read", "a"); assertUserIsDenied("u5", "manage", "a"); assertUserIsDenied("u5", "write", "a"); - assertUserIsAllowed("u5", "get", "b"); + assertUserIsAllowed("u5", "read", "b"); assertUserIsDenied("u5", "manage", "b"); assertUserIsDenied("u5", "write", "b"); - assertAccessIsDenied("u5", "GET", "/b/_search"); } public void testUserU6() throws Exception { - // u6 has all access on a and search access on b + // u6 has all access on a and read access on b assertUserIsAllowed("u6", "all", "a"); - assertUserIsAllowed("u6", "search", "b"); + assertUserIsAllowed("u6", "read", "b"); assertUserIsDenied("u6", "manage", "b"); assertUserIsDenied("u6", "write", "b"); assertUserIsDenied("u6", "all", "c"); @@ -244,7 +232,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { } public void testUserU8() throws Exception { - // u8 has admin access and get access on b + // u8 has admin access and read access on b assertUserIsAllowed("u8", "all", "a"); assertUserIsAllowed("u8", "all", "b"); assertUserIsAllowed("u8", "all", "c"); @@ -260,15 +248,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { assertUserIsDenied("u9", "all", "c"); } - public void testUserU10() throws Exception { - // u10 has access on get/search on b - assertUserIsDenied("u10", "all", "a"); - assertUserIsDenied("u10", "manage", "b"); - assertUserIsDenied("u10", "write", "b"); - assertUserIsAllowed("u10", "search", "b"); - assertUserIsDenied("u10", "all", "c"); - } - public void testUserU11() throws Exception { // u11 has access to create c and delete b assertUserIsDenied("u11", "all", "a"); @@ -295,7 +274,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { } public void testUserU13() throws Exception { - // u13 has search access on b and index access on a + // u13 has read access on b and index access on a assertUserIsDenied("u13", "manage", "a"); assertUserIsAllowed("u13", "index", "a"); assertUserIsDenied("u13", "delete", "a"); @@ -303,7 +282,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { assertUserIsDenied("u13", "manage", "b"); assertUserIsDenied("u13", "write", "b"); - assertUserIsAllowed("u13", "search", "b"); + assertUserIsAllowed("u13", "read", "b"); assertUserIsDenied("u13", "all", "c"); }