honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into enhancement/node_client_setting_removal 

Original commit: elastic/x-pack-elasticsearch@d3522628d4
This commit is contained in:
javanna 2016-03-25 22:28:40 +01:00 committed by Luca Cavanna
parent 622193ca40 5d3a608786
commit 257ae2cb44
6 changed files with 132 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolver.java
View File

@ -56,9 +56,27 @@ public class NodeStatsResolver extends MonitoringIndexNameResolver.Timestamped<N
"node_stats.jvm.gc.collectors.old",
"node_stats.jvm.gc.collectors.old.collection_count",
"node_stats.jvm.gc.collectors.old.collection_time_in_millis",
"node_stats.thread_pool.index.rejected",
"node_stats.thread_pool.search.rejected",
"node_stats.thread_pool.bulk.threads",
"node_stats.thread_pool.bulk.queue",
"node_stats.thread_pool.bulk.rejected",
"node_stats.thread_pool.generic.threads",
"node_stats.thread_pool.generic.queue",
"node_stats.thread_pool.generic.rejected",
"node_stats.thread_pool.get.threads",
"node_stats.thread_pool.get.queue",
"node_stats.thread_pool.get.rejected",
"node_stats.thread_pool.index.threads",
"node_stats.thread_pool.index.queue",
"node_stats.thread_pool.index.rejected",
"node_stats.thread_pool.management.threads",
"node_stats.thread_pool.management.queue",
"node_stats.thread_pool.management.rejected",
"node_stats.thread_pool.search.threads",
"node_stats.thread_pool.search.queue",
"node_stats.thread_pool.search.rejected",
"node_stats.thread_pool.watcher.threads",
"node_stats.thread_pool.watcher.queue",
"node_stats.thread_pool.watcher.rejected",
};
public NodeStatsResolver(MonitoredSystem id, int version, Settings settings) {

25
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStateTests.java
View File

@ -5,7 +5,7 @@
*/
package org.elasticsearch.marvel.agent.resolver.cluster;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.settings.Settings;
@ -24,6 +24,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.marvel.MonitoredSystem.ES;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.TEMPLATE_VERSION;
import static org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver.Fields.SOURCE_NODE;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@ -32,7 +33,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.core.Is.is;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@LuceneTestCase.BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ClusterScope(scope = Scope.TEST)
public class ClusterStateTests extends MarvelIntegTestCase {
@ -105,11 +106,17 @@ public class ClusterStateTests extends MarvelIntegTestCase {
public void testClusterStateNodes() throws Exception {
final long nbNodes = internalCluster().size();
MonitoringIndexNameResolver.Timestamped timestampedResolver = new MockTimestampedIndexNameResolver(ES, TEMPLATE_VERSION);
assertNotNull(timestampedResolver);
String timestampedIndex = timestampedResolver.indexPattern();
awaitIndexExists(timestampedIndex);
logger.debug("--> waiting for documents to be collected");
awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), ClusterStateNodeResolver.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", ClusterStateNodeResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(ClusterStateNodeResolver.TYPE).get();
SearchResponse response = client().prepareSearch(timestampedIndex).setTypes(ClusterStateNodeResolver.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes));
logger.debug("--> checking that every document contains the expected fields");
@ -133,10 +140,10 @@ public class ClusterStateTests extends MarvelIntegTestCase {
logger.debug("--> check that node attributes are indexed");
assertThat(client().prepareSearch().setSize(0)
.setIndices(MONITORING_INDICES_PREFIX + TEMPLATE_VERSION + "-*")
.setIndices(timestampedIndex)
.setTypes(ClusterStateNodeResolver.TYPE)
.setQuery(QueryBuilders.matchQuery(SOURCE_NODE.underscore().toString() + ".attributes.custom", randomInt)
).get().getHits().getTotalHits(), greaterThan(0L));
.setQuery(QueryBuilders.matchQuery(SOURCE_NODE.underscore().toString() + ".attributes.custom", randomInt))
.get().getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> cluster state nodes successfully collected");
}
@ -144,7 +151,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
public void testDiscoveryNodes() throws Exception {
final long nbNodes = internalCluster().size();
MonitoringIndexNameResolver.Data dataResolver = internalCluster().getInstance(MonitoringIndexNameResolver.Data.class);
MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(TEMPLATE_VERSION);
assertNotNull(dataResolver);
String dataIndex = dataResolver.indexPattern();
@ -154,7 +161,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), DiscoveryNodeResolver.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", DiscoveryNodeResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(DiscoveryNodeResolver.TYPE).get();
SearchResponse response = client().prepareSearch(dataIndex).setTypes(DiscoveryNodeResolver.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes));
logger.debug("--> checking that every document contains the expected fields");
@ -188,7 +195,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
assertThat(client().prepareGet(dataIndex, DiscoveryNodeResolver.TYPE, nodeId).get().isExists(), is(true));
// checks that document is not indexed
assertHitCount(client().prepareSearch().setSize(0)
assertHitCount(client().prepareSearch(dataIndex).setSize(0)
.setTypes(DiscoveryNodeResolver.TYPE)
.setQuery(QueryBuilders.boolQuery()
.should(matchQuery("node.id", nodeId))

10
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.monitor.os.OsProbe;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.threadpool.ThreadPoolStats;
import org.elasticsearch.watcher.execution.InternalWatchExecutor;
import java.io.IOException;
import java.nio.file.Path;
@ -124,9 +125,14 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase<
};
Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
statsByShard.put(index, Collections.singletonList(new IndexShardStats(shardId, new ShardStats[]{shardStats})));
List<ThreadPoolStats.Stats> threadPoolStats = Arrays.asList(new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0),
List<ThreadPoolStats.Stats> threadPoolStats = Arrays.asList(
new ThreadPoolStats.Stats(ThreadPool.Names.BULK, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0)
new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.GET, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.MANAGEMENT, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(InternalWatchExecutor.THREAD_POOL_NAME, 0, 0, 0, 0, 0, 0)
);
return new NodeStats(new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
new NodeIndicesStats(new CommonStats(), statsByShard), OsProbe.getInstance().osStats(),

22
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java
View File

@ -11,13 +11,13 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.MarvelSettings;
import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
import org.elasticsearch.marvel.agent.exporter.local.LocalExporter;
import org.elasticsearch.marvel.agent.resolver.node.NodeStatsResolver;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.After;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -63,21 +63,37 @@ public class NodeStatsTests extends MarvelIntegTestCase {
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that every document contains the expected fields");
String[] filters = NodeStatsResolver.FILTERS;
for (SearchHit searchHit : response.getHits().getHits()) {
Map<String, Object> fields = searchHit.sourceAsMap();
for (String filter : filters) {
for (String filter : nodeStatsFilters(watcherEnabled)) {
if (Constants.WINDOWS) {
// load average is unavailable on Windows
if ("node_stats.os.cpu.load_average.1m".equals(filter)) {
continue;
}
}
assertContains(filter, fields);
}
}
logger.debug("--> node stats successfully collected");
}
/**
* Optionally exclude {@link NodeStatsResolver#FILTERS} that require Watcher to be enabled.
*
* @param includeWatcher {@code true} to keep watcher filters.
* @return Never {@code null} or empty.
* @see #watcherEnabled
*/
private static String[] nodeStatsFilters(boolean includeWatcher) {
if (includeWatcher) {
return NodeStatsResolver.FILTERS;
}
return Arrays.stream(NodeStatsResolver.FILTERS).filter(s -> s.contains("watcher") == false).toArray(String[]::new);
}
}

54
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java
View File

@ -68,14 +68,29 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
public static final String MONITORING_INDICES_PREFIX = MonitoringIndexNameResolver.PREFIX + MonitoringIndexNameResolver.DELIMITER;
protected static Boolean shieldEnabled;
/**
* Enables individual tests to control the behavior.
* <p>
* Control this by overriding {@link #enableShield()}, which defaults to enabling it randomly.
*/
protected Boolean shieldEnabled;
/**
* Enables individual tests to control the behavior.
* <p>
* Control this by overriding {@link #enableWatcher()}, which defaults to enabling it randomly.
*/
protected Boolean watcherEnabled;
@Override
protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
if (shieldEnabled == null) {
shieldEnabled = enableShield();
}
if (watcherEnabled == null) {
watcherEnabled = enableWatcher();
}
logger.info("--> shield {}", shieldEnabled ? "enabled" : "disabled");
logger.debug("--> watcher {}", watcherEnabled ? "enabled" : "disabled");
return super.buildTestCluster(scope, seed);
}
@ -83,9 +98,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
//TODO: for now lets isolate monitoring tests from watcher (randomize this later)
.put(XPackPlugin.featureEnabledSetting(Watcher.NAME), false)
.put(XPackPlugin.featureEnabledSetting(Watcher.NAME), watcherEnabled)
// we do this by default in core, but for monitoring this isn't needed and only adds noise.
.put("index.store.mock.check_index_on_close", false);
@ -151,12 +164,19 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}
/**
* Override and returns {@code false} to force running without shield
* Override and return {@code false} to force running without Security.
*/
protected boolean enableShield() {
return randomBoolean();
}
/**
* Override and return {@code false} to force running without Watcher.
*/
protected boolean enableWatcher() {
return randomBoolean();
}
protected void stopCollection() {
for (AgentService agent : internalCluster().getInstances(AgentService.class)) {
agent.stopCollection();
@ -340,6 +360,15 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
* it recurses to check if 'bar' exists in the sub-map.
*/
protected void assertContains(String field, Map<String, Object> values) {
assertContains(field, values, null);
}
/**
* Checks if a field exist in a map of values. If the field contains a dot like 'foo.bar'
* it checks that 'foo' exists in the map of values and that it points to a sub-map. Then
* it recurses to check if 'bar' exists in the sub-map.
*/
protected void assertContains(String field, Map<String, Object> values, String parent) {
assertNotNull("field name should not be null", field);
assertNotNull("values map should not be null", values);
@ -351,21 +380,30 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
assertTrue(Strings.hasText(segment));
boolean fieldExists = values.containsKey(segment);
assertTrue("expecting field [" + segment + "] to be present in monitoring document", fieldExists);
assertTrue("expecting field [" + rebuildName(parent, segment) + "] to be present in monitoring document", fieldExists);
Object value = values.get(segment);
String next = field.substring(point + 1);
if (next.length() > 0) {
assertTrue(value instanceof Map);
assertContains(next, (Map<String, Object>) value);
assertContains(next, (Map<String, Object>) value, rebuildName(parent, segment));
} else {
assertFalse(value instanceof Map);
}
} else {
assertTrue("expecting field [" + field + "] to be present in monitoring document", values.containsKey(field));
assertTrue("expecting field [" + rebuildName(parent, field) + "] to be present in monitoring document",
values.containsKey(field));
}
}
private String rebuildName(String parent, String field) {
if (Strings.isEmpty(parent)) {
return field;
}
return parent + "." + field;
}
protected void updateMarvelInterval(long value, TimeUnit timeUnit) {
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
Settings.builder().put(MarvelSettings.INTERVAL.getKey(), value, timeUnit)));

67
elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
View File

@ -22,10 +22,10 @@ import static org.hamcrest.Matchers.is;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false)
@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false, maxNumDataNodes = 2)
public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
private String jsonDoc = "{ \"name\" : \"elasticsearch\"}";
private String jsonDoc = "{ \"name\" : \"elasticsearch\", \"body\": \"foo bar\" }";
public static final String ROLES =
"all_cluster_role:\n" +
@ -42,6 +42,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ read ]\n" +
"read_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ read ]\n" +
"write_a_role:\n" +
" indices:\n" +
" - names: 'a'\n" +
@ -50,14 +54,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: [ 'a', 'b' ]\n" +
" privileges: [ read ]\n" +
"get_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ get ]\n" +
"search_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ search ]\n" +
"all_regex_ab_role:\n" +
" indices:\n" +
" - names: '/a|b/'\n" +
@ -66,10 +62,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'a*'\n" +
" privileges: [ manage ]\n" +
"data_access_all_role:\n" +
"read_write_all_role:\n" +
" indices:\n" +
" - names: '*'\n" +
" privileges: [ data_access ]\n" +
" privileges: [ read, write ]\n" +
"create_c_role:\n" +
" indices:\n" +
" - names: 'c'\n" +
@ -78,10 +74,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ monitor ]\n" +
"crud_a_role:\n" +
"read_write_a_role:\n" +
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ crud ]\n" +
" privileges: [ read, write ]\n" +
"delete_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
@ -90,10 +86,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ index ]\n" +
"search_a_role:\n" +
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ search ]\n" +
"\n";
public static final String USERS =
@ -107,7 +99,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
"u7:" + USERS_PASSWD_HASHED + "\n"+
"u8:" + USERS_PASSWD_HASHED + "\n"+
"u9:" + USERS_PASSWD_HASHED + "\n" +
"u10:" + USERS_PASSWD_HASHED + "\n" +
"u11:" + USERS_PASSWD_HASHED + "\n" +
"u12:" + USERS_PASSWD_HASHED + "\n" +
"u13:" + USERS_PASSWD_HASHED + "\n" +
@ -118,19 +109,17 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
"all_cluster_role:admin\n" +
"all_a_role:u1,u2,u6\n" +
"read_a_role:u1,u5,u14\n" +
"read_b_role:u3,u5,u6,u8,u13\n" +
"write_a_role:u9\n" +
"read_ab_role:u2,u4,u9\n" +
"get_b_role:u3,u5,u8,u10\n" +
"search_b_role:u6,u10,u13\n" +
"all_regex_ab_role:u3\n" +
"manage_starts_with_a_role:u4,u15\n" +
"data_access_all_role:u12\n" +
"manage_starts_with_a_role:u4\n" +
"read_write_all_role:u12\n" +
"create_c_role:u11\n" +
"monitor_b_role:u14\n" +
"crud_a_role:u12\n" +
"read_write_a_role:u12\n" +
"delete_b_role:u11\n" +
"index_a_role:u13\n" +
"search_a_role:u15\n" ;
"index_a_role:u13\n";
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -193,7 +182,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU3() throws Exception {
// u3 has get b role, but all access to a* and b* via regex
// u3 has read b role, but all access to a* and b* via regex
assertUserIsAllowed("u3", "all", "a");
assertUserIsAllowed("u3", "all", "b");
assertUserIsDenied("u3", "all", "c");
@ -216,21 +205,20 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU5() throws Exception {
// u5 may read a and get b
// u5 may read a and read b
assertUserIsAllowed("u5", "read", "a");
assertUserIsDenied("u5", "manage", "a");
assertUserIsDenied("u5", "write", "a");
assertUserIsAllowed("u5", "get", "b");
assertUserIsAllowed("u5", "read", "b");
assertUserIsDenied("u5", "manage", "b");
assertUserIsDenied("u5", "write", "b");
assertAccessIsDenied("u5", "GET", "/b/_search");
}
public void testUserU6() throws Exception {
// u6 has all access on a and search access on b
// u6 has all access on a and read access on b
assertUserIsAllowed("u6", "all", "a");
assertUserIsAllowed("u6", "search", "b");
assertUserIsAllowed("u6", "read", "b");
assertUserIsDenied("u6", "manage", "b");
assertUserIsDenied("u6", "write", "b");
assertUserIsDenied("u6", "all", "c");
@ -244,7 +232,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU8() throws Exception {
// u8 has admin access and get access on b
// u8 has admin access and read access on b
assertUserIsAllowed("u8", "all", "a");
assertUserIsAllowed("u8", "all", "b");
assertUserIsAllowed("u8", "all", "c");
@ -260,15 +248,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertUserIsDenied("u9", "all", "c");
}
public void testUserU10() throws Exception {
// u10 has access on get/search on b
assertUserIsDenied("u10", "all", "a");
assertUserIsDenied("u10", "manage", "b");
assertUserIsDenied("u10", "write", "b");
assertUserIsAllowed("u10", "search", "b");
assertUserIsDenied("u10", "all", "c");
}
public void testUserU11() throws Exception {
// u11 has access to create c and delete b
assertUserIsDenied("u11", "all", "a");
@ -295,7 +274,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU13() throws Exception {
// u13 has search access on b and index access on a
// u13 has read access on b and index access on a
assertUserIsDenied("u13", "manage", "a");
assertUserIsAllowed("u13", "index", "a");
assertUserIsDenied("u13", "delete", "a");
@ -303,7 +282,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertUserIsDenied("u13", "manage", "b");
assertUserIsDenied("u13", "write", "b");
assertUserIsAllowed("u13", "search", "b");
assertUserIsAllowed("u13", "read", "b");
assertUserIsDenied("u13", "all", "c");
}