honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into feature/rename-shield 

Original commit: elastic/x-pack-elasticsearch@cdef13a255
This commit is contained in:
Lukas Olson 2016-03-25 10:26:50 -07:00
parent 7fbd30c801 ea2be5d4d9
commit d52add9f29
5 changed files with 69 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolver.java
View File

@ -56,9 +56,27 @@ public class NodeStatsResolver extends MonitoringIndexNameResolver.Timestamped<N
"node_stats.jvm.gc.collectors.old",
"node_stats.jvm.gc.collectors.old.collection_count",
"node_stats.jvm.gc.collectors.old.collection_time_in_millis",
"node_stats.thread_pool.index.rejected",
"node_stats.thread_pool.search.rejected",
"node_stats.thread_pool.bulk.threads",
"node_stats.thread_pool.bulk.queue",
"node_stats.thread_pool.bulk.rejected",
"node_stats.thread_pool.generic.threads",
"node_stats.thread_pool.generic.queue",
"node_stats.thread_pool.generic.rejected",
"node_stats.thread_pool.get.threads",
"node_stats.thread_pool.get.queue",
"node_stats.thread_pool.get.rejected",
"node_stats.thread_pool.index.threads",
"node_stats.thread_pool.index.queue",
"node_stats.thread_pool.index.rejected",
"node_stats.thread_pool.management.threads",
"node_stats.thread_pool.management.queue",
"node_stats.thread_pool.management.rejected",
"node_stats.thread_pool.search.threads",
"node_stats.thread_pool.search.queue",
"node_stats.thread_pool.search.rejected",
"node_stats.thread_pool.watcher.threads",
"node_stats.thread_pool.watcher.queue",
"node_stats.thread_pool.watcher.rejected",
};
public NodeStatsResolver(MonitoredSystem id, int version, Settings settings) {

25
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStateTests.java
View File

@ -5,7 +5,7 @@
*/
package org.elasticsearch.marvel.agent.resolver.cluster;
import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.settings.Settings;
@ -24,6 +24,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.marvel.MonitoredSystem.ES;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.TEMPLATE_VERSION;
import static org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver.Fields.SOURCE_NODE;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@ -32,7 +33,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.core.Is.is;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@LuceneTestCase.BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ClusterScope(scope = Scope.TEST)
public class ClusterStateTests extends MarvelIntegTestCase {
@ -105,11 +106,17 @@ public class ClusterStateTests extends MarvelIntegTestCase {
public void testClusterStateNodes() throws Exception {
final long nbNodes = internalCluster().size();
MonitoringIndexNameResolver.Timestamped timestampedResolver = new MockTimestampedIndexNameResolver(ES, TEMPLATE_VERSION);
assertNotNull(timestampedResolver);
String timestampedIndex = timestampedResolver.indexPattern();
awaitIndexExists(timestampedIndex);
logger.debug("--> waiting for documents to be collected");
awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), ClusterStateNodeResolver.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", ClusterStateNodeResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(ClusterStateNodeResolver.TYPE).get();
SearchResponse response = client().prepareSearch(timestampedIndex).setTypes(ClusterStateNodeResolver.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes));
logger.debug("--> checking that every document contains the expected fields");
@ -133,10 +140,10 @@ public class ClusterStateTests extends MarvelIntegTestCase {
logger.debug("--> check that node attributes are indexed");
assertThat(client().prepareSearch().setSize(0)
.setIndices(MONITORING_INDICES_PREFIX + TEMPLATE_VERSION + "-*")
.setIndices(timestampedIndex)
.setTypes(ClusterStateNodeResolver.TYPE)
.setQuery(QueryBuilders.matchQuery(SOURCE_NODE.underscore().toString() + ".attributes.custom", randomInt)
).get().getHits().getTotalHits(), greaterThan(0L));
.setQuery(QueryBuilders.matchQuery(SOURCE_NODE.underscore().toString() + ".attributes.custom", randomInt))
.get().getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> cluster state nodes successfully collected");
}
@ -144,7 +151,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
public void testDiscoveryNodes() throws Exception {
final long nbNodes = internalCluster().size();
MonitoringIndexNameResolver.Data dataResolver = internalCluster().getInstance(MonitoringIndexNameResolver.Data.class);
MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(TEMPLATE_VERSION);
assertNotNull(dataResolver);
String dataIndex = dataResolver.indexPattern();
@ -154,7 +161,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
awaitMarvelDocsCount(greaterThanOrEqualTo(nbNodes), DiscoveryNodeResolver.TYPE);
logger.debug("--> searching for monitoring documents of type [{}]", DiscoveryNodeResolver.TYPE);
SearchResponse response = client().prepareSearch().setTypes(DiscoveryNodeResolver.TYPE).get();
SearchResponse response = client().prepareSearch(dataIndex).setTypes(DiscoveryNodeResolver.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes));
logger.debug("--> checking that every document contains the expected fields");
@ -188,7 +195,7 @@ public class ClusterStateTests extends MarvelIntegTestCase {
assertThat(client().prepareGet(dataIndex, DiscoveryNodeResolver.TYPE, nodeId).get().isExists(), is(true));
// checks that document is not indexed
assertHitCount(client().prepareSearch().setSize(0)
assertHitCount(client().prepareSearch(dataIndex).setSize(0)
.setTypes(DiscoveryNodeResolver.TYPE)
.setQuery(QueryBuilders.boolQuery()
.should(matchQuery("node.id", nodeId))

10
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.monitor.os.OsProbe;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.threadpool.ThreadPoolStats;
import org.elasticsearch.watcher.execution.InternalWatchExecutor;
import java.io.IOException;
import java.nio.file.Path;
@ -122,9 +123,14 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase<
};
Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
statsByShard.put(index, Collections.singletonList(new IndexShardStats(shardId, new ShardStats[]{shardStats})));
List<ThreadPoolStats.Stats> threadPoolStats = Arrays.asList(new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0),
List<ThreadPoolStats.Stats> threadPoolStats = Arrays.asList(
new ThreadPoolStats.Stats(ThreadPool.Names.BULK, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0)
new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.GET, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.MANAGEMENT, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0),
new ThreadPoolStats.Stats(InternalWatchExecutor.THREAD_POOL_NAME, 0, 0, 0, 0, 0, 0)
);
return new NodeStats(new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
new NodeIndicesStats(new CommonStats(), statsByShard), OsProbe.getInstance().osStats(),

15
elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java
View File

@ -514,23 +514,12 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore,
for (SearchHit hit : response.getHits().getHits()) {
final String roleName = hit.getId();
final long version = hit.version();
final boolean existed = existingRoles.remove(roleName);
existingRoles.remove(roleName);
// we use the locking mechanisms provided by the map/cache to help protect against concurrent operations
// that will leave the cache in a bad state
roleCache.compute(roleName, new BiFunction<String, RoleAndVersion, RoleAndVersion>() {
roleCache.computeIfPresent(roleName, new BiFunction<String, RoleAndVersion, RoleAndVersion>() {
@Override
public RoleAndVersion apply(String roleName, RoleAndVersion existing) {
if (existing == null) {
if (existed) {
// the cache doesn't have this role anymore, it got cleared by something else, do nothing.
return null;
} else {
// it is new, we can cache it
RoleDescriptor rd = transformRole(hit.getId(), hit.getSourceRef());
return new RoleAndVersion(rd, version);
}
}
if (version > existing.getVersion()) {
RoleDescriptor rd = transformRole(hit.getId(), hit.getSourceRef());
if (rd != null) {

67
elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
View File

@ -22,10 +22,10 @@ import static org.hamcrest.Matchers.is;
//test is just too slow, please fix it to not be sleep-based
@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false)
@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false, maxNumDataNodes = 2)
public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
private String jsonDoc = "{ \"name\" : \"elasticsearch\"}";
private String jsonDoc = "{ \"name\" : \"elasticsearch\", \"body\": \"foo bar\" }";
public static final String ROLES =
"all_cluster_role:\n" +
@ -42,6 +42,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ read ]\n" +
"read_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ read ]\n" +
"write_a_role:\n" +
" indices:\n" +
" - names: 'a'\n" +
@ -50,14 +54,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: [ 'a', 'b' ]\n" +
" privileges: [ read ]\n" +
"get_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ get ]\n" +
"search_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ search ]\n" +
"all_regex_ab_role:\n" +
" indices:\n" +
" - names: '/a|b/'\n" +
@ -66,10 +62,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'a*'\n" +
" privileges: [ manage ]\n" +
"data_access_all_role:\n" +
"read_write_all_role:\n" +
" indices:\n" +
" - names: '*'\n" +
" privileges: [ data_access ]\n" +
" privileges: [ read, write ]\n" +
"create_c_role:\n" +
" indices:\n" +
" - names: 'c'\n" +
@ -78,10 +74,10 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'b'\n" +
" privileges: [ monitor ]\n" +
"crud_a_role:\n" +
"read_write_a_role:\n" +
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ crud ]\n" +
" privileges: [ read, write ]\n" +
"delete_b_role:\n" +
" indices:\n" +
" - names: 'b'\n" +
@ -90,10 +86,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ index ]\n" +
"search_a_role:\n" +
" indices:\n" +
" - names: 'a'\n" +
" privileges: [ search ]\n" +
"\n";
public static final String USERS =
@ -107,7 +99,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
"u7:" + USERS_PASSWD_HASHED + "\n"+
"u8:" + USERS_PASSWD_HASHED + "\n"+
"u9:" + USERS_PASSWD_HASHED + "\n" +
"u10:" + USERS_PASSWD_HASHED + "\n" +
"u11:" + USERS_PASSWD_HASHED + "\n" +
"u12:" + USERS_PASSWD_HASHED + "\n" +
"u13:" + USERS_PASSWD_HASHED + "\n" +
@ -118,19 +109,17 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
"all_cluster_role:admin\n" +
"all_a_role:u1,u2,u6\n" +
"read_a_role:u1,u5,u14\n" +
"read_b_role:u3,u5,u6,u8,u13\n" +
"write_a_role:u9\n" +
"read_ab_role:u2,u4,u9\n" +
"get_b_role:u3,u5,u8,u10\n" +
"search_b_role:u6,u10,u13\n" +
"all_regex_ab_role:u3\n" +
"manage_starts_with_a_role:u4,u15\n" +
"data_access_all_role:u12\n" +
"manage_starts_with_a_role:u4\n" +
"read_write_all_role:u12\n" +
"create_c_role:u11\n" +
"monitor_b_role:u14\n" +
"crud_a_role:u12\n" +
"read_write_a_role:u12\n" +
"delete_b_role:u11\n" +
"index_a_role:u13\n" +
"search_a_role:u15\n" ;
"index_a_role:u13\n";
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -193,7 +182,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU3() throws Exception {
// u3 has get b role, but all access to a* and b* via regex
// u3 has read b role, but all access to a* and b* via regex
assertUserIsAllowed("u3", "all", "a");
assertUserIsAllowed("u3", "all", "b");
assertUserIsDenied("u3", "all", "c");
@ -216,21 +205,20 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU5() throws Exception {
// u5 may read a and get b
// u5 may read a and read b
assertUserIsAllowed("u5", "read", "a");
assertUserIsDenied("u5", "manage", "a");
assertUserIsDenied("u5", "write", "a");
assertUserIsAllowed("u5", "get", "b");
assertUserIsAllowed("u5", "read", "b");
assertUserIsDenied("u5", "manage", "b");
assertUserIsDenied("u5", "write", "b");
assertAccessIsDenied("u5", "GET", "/b/_search");
}
public void testUserU6() throws Exception {
// u6 has all access on a and search access on b
// u6 has all access on a and read access on b
assertUserIsAllowed("u6", "all", "a");
assertUserIsAllowed("u6", "search", "b");
assertUserIsAllowed("u6", "read", "b");
assertUserIsDenied("u6", "manage", "b");
assertUserIsDenied("u6", "write", "b");
assertUserIsDenied("u6", "all", "c");
@ -244,7 +232,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU8() throws Exception {
// u8 has admin access and get access on b
// u8 has admin access and read access on b
assertUserIsAllowed("u8", "all", "a");
assertUserIsAllowed("u8", "all", "b");
assertUserIsAllowed("u8", "all", "c");
@ -260,15 +248,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertUserIsDenied("u9", "all", "c");
}
public void testUserU10() throws Exception {
// u10 has access on get/search on b
assertUserIsDenied("u10", "all", "a");
assertUserIsDenied("u10", "manage", "b");
assertUserIsDenied("u10", "write", "b");
assertUserIsAllowed("u10", "search", "b");
assertUserIsDenied("u10", "all", "c");
}
public void testUserU11() throws Exception {
// u11 has access to create c and delete b
assertUserIsDenied("u11", "all", "a");
@ -295,7 +274,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
}
public void testUserU13() throws Exception {
// u13 has search access on b and index access on a
// u13 has read access on b and index access on a
assertUserIsDenied("u13", "manage", "a");
assertUserIsAllowed("u13", "index", "a");
assertUserIsDenied("u13", "delete", "a");
@ -303,7 +282,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
assertUserIsDenied("u13", "manage", "b");
assertUserIsDenied("u13", "write", "b");
assertUserIsAllowed("u13", "search", "b");
assertUserIsAllowed("u13", "read", "b");
assertUserIsDenied("u13", "all", "c");
}