From 67efe608d1e691e2ff4ac3aeb8251d0a15129592 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 14 Mar 2016 20:51:29 +0100 Subject: [PATCH 01/27] add assertion DefaultIndicesAndAliasesResolver that PutMapping special case holds Original commit: elastic/x-pack-elasticsearch@417123150ad2c3a5266424638927340d2ee260ac --- .../indicesresolver/DefaultIndicesAndAliasesResolver.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java index 46e1896fe0b..dfdca2557a2 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.authz.indicesresolver; +import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.IndicesRequest; @@ -78,6 +79,8 @@ public class DefaultIndicesAndAliasesResolver implements IndicesAndAliasesResolv * the list of indices in there, if we do so it will result in an invalid request and the update will fail. */ indices = Collections.singleton(((PutMappingRequest) indicesRequest).getConcreteIndex().getName()); + assert indicesRequest.indices() == null || indicesRequest.indices().length == 0 + : "indices are: " + Arrays.toString(indicesRequest.indices()); // Arrays.toString() can handle null values - all good } else { if (indicesRequest.indicesOptions().expandWildcardsOpen() || indicesRequest.indicesOptions().expandWildcardsClosed()) { if (indicesRequest instanceof IndicesRequest.Replaceable) { From 6b7dadce43f145721b42c439718fbfcfb6bc9332 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 14 Mar 2016 20:52:49 +0100 Subject: [PATCH 02/27] remove unused imports Original commit: elastic/x-pack-elasticsearch@4a8f4285f89575f5fa60f2f589b74753c6284718 --- .../authz/indicesresolver/DefaultIndicesAndAliasesResolver.java | 1 - 1 file changed, 1 deletion(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java index dfdca2557a2..84c29bc7e6c 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.shield.authz.indicesresolver; -import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.IndicesRequest; From 1b21f20e5bcb091c63de6128cb3c962cbcdbd0bf Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 14 Mar 2016 17:22:05 -0700 Subject: [PATCH 03/27] Lazily generate san string for certificates The san string used by certificate generation for ssl tests currently runs at gradle configuration time. This takes several seconds, and significantly slows down gradle configuration on every invocation. This change wraps the code into a lazy evaluator that will be invoked at runtime, and cache the string once it is generated. Original commit: elastic/x-pack-elasticsearch@812036f4160951c3d1c5616a6ab5ca1dbc47bac3 --- .../qa/smoke-test-plugins-ssl/build.gradle | 261 +++++++++--------- 1 file changed, 138 insertions(+), 123 deletions(-) diff --git a/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle index 1b87fb0980b..733e6c19856 100644 --- a/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle +++ b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle @@ -8,7 +8,7 @@ dependencies { } // needed to be consistent with ssl host checking -String san = getSubjectAlternativeNameString() +Object san = new SanEvaluator() // location of generated keystores and certificates File keystoreDir = new File(project.buildDir, 'keystore') @@ -200,144 +200,159 @@ processTestResources { } } +/** A lazy evaluator to find the san to use for certificate generation. */ +class SanEvaluator { -// Code stolen from NetworkUtils/InetAddresses/NetworkAddress to support SAN -/** Return all interfaces (and subinterfaces) on the system */ -static List getInterfaces() throws SocketException { - List all = new ArrayList<>(); - addAllInterfaces(all, Collections.list(NetworkInterface.getNetworkInterfaces())); - Collections.sort(all, new Comparator() { - @Override - public int compare(NetworkInterface left, NetworkInterface right) { - return Integer.compare(left.getIndex(), right.getIndex()); + private static String san = null + + String toString() { + synchronized (SanEvaluator.class) { + if (san == null) { + san = getSubjectAlternativeNameString() + } } - }); - return all; -} + return san + } -/** Helper for getInterfaces, recursively adds subinterfaces to {@code target} */ -private static void addAllInterfaces(List target, List level) { - if (!level.isEmpty()) { - target.addAll(level); - for (NetworkInterface intf : level) { - addAllInterfaces(target, Collections.list(intf.getSubInterfaces())); + // Code stolen from NetworkUtils/InetAddresses/NetworkAddress to support SAN + /** Return all interfaces (and subinterfaces) on the system */ + private static List getInterfaces() throws SocketException { + List all = new ArrayList<>(); + addAllInterfaces(all, Collections.list(NetworkInterface.getNetworkInterfaces())); + Collections.sort(all, new Comparator() { + @Override + public int compare(NetworkInterface left, NetworkInterface right) { + return Integer.compare(left.getIndex(), right.getIndex()); + } + }); + return all; + } + + /** Helper for getInterfaces, recursively adds subinterfaces to {@code target} */ + private static void addAllInterfaces(List target, List level) { + if (!level.isEmpty()) { + target.addAll(level); + for (NetworkInterface intf : level) { + addAllInterfaces(target, Collections.list(intf.getSubInterfaces())); + } } } -} -private static String getSubjectAlternativeNameString() { - List list = new ArrayList<>(); - for (NetworkInterface intf : getInterfaces()) { - if (intf.isUp()) { - // NOTE: some operating systems (e.g. BSD stack) assign a link local address to the loopback interface - // while technically not a loopback address, some of these treat them as one (e.g. OS X "localhost") so we must too, - // otherwise things just won't work out of box. So we include all addresses from loopback interfaces. - for (InetAddress address : Collections.list(intf.getInetAddresses())) { - if (intf.isLoopback() || address.isLoopbackAddress()) { - list.add(address); + private static String getSubjectAlternativeNameString() { + List list = new ArrayList<>(); + for (NetworkInterface intf : getInterfaces()) { + if (intf.isUp()) { + // NOTE: some operating systems (e.g. BSD stack) assign a link local address to the loopback interface + // while technically not a loopback address, some of these treat them as one (e.g. OS X "localhost") so we must too, + // otherwise things just won't work out of box. So we include all addresses from loopback interfaces. + for (InetAddress address : Collections.list(intf.getInetAddresses())) { + if (intf.isLoopback() || address.isLoopbackAddress()) { + list.add(address); + } } } } - } - if (list.isEmpty()) { - throw new IllegalArgumentException("no up-and-running loopback addresses found, got " + getInterfaces()); - } - - StringBuilder builder = new StringBuilder("san="); - for (int i = 0; i < list.size(); i++) { - InetAddress address = list.get(i); - String hostAddress; - if (address instanceof Inet6Address) { - hostAddress = compressedIPV6Address((Inet6Address)address); - } else { - hostAddress = address.getHostAddress(); - } - builder.append("ip:").append(hostAddress); - String hostname = address.getHostName(); - if (hostname.equals(address.getHostAddress()) == false) { - builder.append(",dns:").append(hostname); + if (list.isEmpty()) { + throw new IllegalArgumentException("no up-and-running loopback addresses found, got " + getInterfaces()); } - if (i != (list.size() - 1)) { - builder.append(","); - } - } - - return builder.toString(); -} - -private static String compressedIPV6Address(Inet6Address inet6Address) { - byte[] bytes = inet6Address.getAddress(); - int[] hextets = new int[8]; - for (int i = 0; i < hextets.length; i++) { - hextets[i] = (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255; - } - compressLongestRunOfZeroes(hextets); - return hextetsToIPv6String(hextets); -} - -/** - * Identify and mark the longest run of zeroes in an IPv6 address. - * - *

Only runs of two or more hextets are considered. In case of a tie, the - * leftmost run wins. If a qualifying run is found, its hextets are replaced - * by the sentinel value -1. - * - * @param hextets {@code int[]} mutable array of eight 16-bit hextets - */ -private static void compressLongestRunOfZeroes(int[] hextets) { - int bestRunStart = -1; - int bestRunLength = -1; - int runStart = -1; - for (int i = 0; i < hextets.length + 1; i++) { - if (i < hextets.length && hextets[i] == 0) { - if (runStart < 0) { - runStart = i; + StringBuilder builder = new StringBuilder("san="); + for (int i = 0; i < list.size(); i++) { + InetAddress address = list.get(i); + String hostAddress; + if (address instanceof Inet6Address) { + hostAddress = compressedIPV6Address((Inet6Address)address); + } else { + hostAddress = address.getHostAddress(); } - } else if (runStart >= 0) { - int runLength = i - runStart; - if (runLength > bestRunLength) { - bestRunStart = runStart; - bestRunLength = runLength; + builder.append("ip:").append(hostAddress); + String hostname = address.getHostName(); + if (hostname.equals(address.getHostAddress()) == false) { + builder.append(",dns:").append(hostname); } - runStart = -1; - } - } - if (bestRunLength >= 2) { - Arrays.fill(hextets, bestRunStart, bestRunStart + bestRunLength, -1); - } -} -/** - * Convert a list of hextets into a human-readable IPv6 address. - * - *

In order for "::" compression to work, the input should contain negative - * sentinel values in place of the elided zeroes. - * - * @param hextets {@code int[]} array of eight 16-bit hextets, or -1s - */ -private static String hextetsToIPv6String(int[] hextets) { - /* - * While scanning the array, handle these state transitions: - * start->num => "num" start->gap => "::" - * num->num => ":num" num->gap => "::" - * gap->num => "num" gap->gap => "" + if (i != (list.size() - 1)) { + builder.append(","); + } + } + + return builder.toString(); + } + + private static String compressedIPV6Address(Inet6Address inet6Address) { + byte[] bytes = inet6Address.getAddress(); + int[] hextets = new int[8]; + for (int i = 0; i < hextets.length; i++) { + hextets[i] = (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255; + } + compressLongestRunOfZeroes(hextets); + return hextetsToIPv6String(hextets); + } + + /** + * Identify and mark the longest run of zeroes in an IPv6 address. + * + *

Only runs of two or more hextets are considered. In case of a tie, the + * leftmost run wins. If a qualifying run is found, its hextets are replaced + * by the sentinel value -1. + * + * @param hextets {@code int[]} mutable array of eight 16-bit hextets */ - StringBuilder buf = new StringBuilder(39); - boolean lastWasNumber = false; - for (int i = 0; i < hextets.length; i++) { - boolean thisIsNumber = hextets[i] >= 0; - if (thisIsNumber) { - if (lastWasNumber) { - buf.append(':'); - } - buf.append(Integer.toHexString(hextets[i])); - } else { - if (i == 0 || lastWasNumber) { - buf.append("::"); + private static void compressLongestRunOfZeroes(int[] hextets) { + int bestRunStart = -1; + int bestRunLength = -1; + int runStart = -1; + for (int i = 0; i < hextets.length + 1; i++) { + if (i < hextets.length && hextets[i] == 0) { + if (runStart < 0) { + runStart = i; + } + } else if (runStart >= 0) { + int runLength = i - runStart; + if (runLength > bestRunLength) { + bestRunStart = runStart; + bestRunLength = runLength; + } + runStart = -1; } } - lastWasNumber = thisIsNumber; + if (bestRunLength >= 2) { + Arrays.fill(hextets, bestRunStart, bestRunStart + bestRunLength, -1); + } + } + + /** + * Convert a list of hextets into a human-readable IPv6 address. + * + *

In order for "::" compression to work, the input should contain negative + * sentinel values in place of the elided zeroes. + * + * @param hextets {@code int[]} array of eight 16-bit hextets, or -1s + */ + private static String hextetsToIPv6String(int[] hextets) { + /* + * While scanning the array, handle these state transitions: + * start->num => "num" start->gap => "::" + * num->num => ":num" num->gap => "::" + * gap->num => "num" gap->gap => "" + */ + StringBuilder buf = new StringBuilder(39); + boolean lastWasNumber = false; + for (int i = 0; i < hextets.length; i++) { + boolean thisIsNumber = hextets[i] >= 0; + if (thisIsNumber) { + if (lastWasNumber) { + buf.append(':'); + } + buf.append(Integer.toHexString(hextets[i])); + } else { + if (i == 0 || lastWasNumber) { + buf.append("::"); + } + } + lastWasNumber = thisIsNumber; + } + return buf.toString(); } - return buf.toString(); } + From 4c4eac692a902064701d6961c84262a658509d36 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Mon, 14 Mar 2016 23:28:05 -0400 Subject: [PATCH 04/27] fix for elasticsearchelastic/elasticsearch#16442 Use index uuid as index folder name to decouple index name from being used as index folder name Original commit: elastic/x-pack-elasticsearch@23193bcd6e9ddce24f8ead80d9245627c23706a6 --- .../agent/resolver/cluster/ClusterStatsResolverTests.java | 4 ++-- .../agent/resolver/indices/IndexStatsResolverTests.java | 4 ++-- .../agent/resolver/indices/IndicesStatsResolverTests.java | 4 ++-- .../marvel/agent/resolver/node/NodeStatsResolverTests.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java index 44ee1ffa3d8..4948b53ee8d 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java @@ -132,14 +132,14 @@ public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCa */ private ShardStats[] randomShardStats() { Index index = new Index("test", UUID.randomUUID().toString()); - Path shardPath = createTempDir().resolve("indices").resolve("test").resolve("0"); + Path shardPath = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0"); ShardRouting shardRouting = ShardRouting.newUnassigned(index, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); CommonStats shardCommonStats = new CommonStats(); shardCommonStats.fieldData = new FieldDataStats(); shardCommonStats.queryCache = new QueryCacheStats(); return new ShardStats[]{ - new ShardStats(shardRouting, new ShardPath(false, shardPath, shardPath, "", new ShardId(index, 0)), shardCommonStats, null) + new ShardStats(shardRouting, new ShardPath(false, shardPath, shardPath, new ShardId(index, 0)), shardCommonStats, null) }; } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndexStatsResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndexStatsResolverTests.java index 5a6542c6a0d..01acb74f3d2 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndexStatsResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndexStatsResolverTests.java @@ -77,7 +77,7 @@ public class IndexStatsResolverTests extends MonitoringIndexNameResolverTestCase private IndexStats randomIndexStats() { Index index = new Index("test-" + randomIntBetween(0, 5), UUID.randomUUID().toString()); ShardId shardId = new ShardId(index, 0); - Path path = createTempDir().resolve("indices").resolve(index.getName()).resolve("0"); + Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0"); ShardRouting shardRouting = ShardRouting.newUnassigned(index, 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); ShardRoutingTestUtils.initialize(shardRouting, "node-0"); @@ -92,7 +92,7 @@ public class IndexStatsResolverTests extends MonitoringIndexNameResolverTestCase stats.segments = new SegmentsStats(); stats.merge = new MergeStats(); stats.refresh = new RefreshStats(); - ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, null, shardId), stats, null); + ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null); return new IndexStats(index.getName(), new ShardStats[]{shardStats}); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndicesStatsResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndicesStatsResolverTests.java index d94307c59b2..a702eab9410 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndicesStatsResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/indices/IndicesStatsResolverTests.java @@ -85,7 +85,7 @@ public class IndicesStatsResolverTests extends MonitoringIndexNameResolverTestCa List shardStats = new ArrayList<>(); for (int i=0; i < randomIntBetween(2, 5); i++) { ShardId shardId = new ShardId(index, i); - Path path = createTempDir().resolve("indices").resolve(index.getName()).resolve(String.valueOf(i)); + Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i)); ShardRouting shardRouting = ShardRouting.newUnassigned(index, i, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); ShardRoutingTestUtils.initialize(shardRouting, "node-0"); @@ -100,7 +100,7 @@ public class IndicesStatsResolverTests extends MonitoringIndexNameResolverTestCa stats.segments = new SegmentsStats(); stats.merge = new MergeStats(); stats.refresh = new RefreshStats(); - shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, null, shardId), stats, null)); + shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null)); } return IndicesStatsResponseTestUtils.newIndicesStatsResponse(shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList()); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java index bac23b49897..4fe42eec663 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsResolverTests.java @@ -103,7 +103,7 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase< private NodeStats randomNodeStats() { Index index = new Index("test-" + randomIntBetween(0, 5), UUID.randomUUID().toString()); ShardId shardId = new ShardId(index, 0); - Path path = createTempDir().resolve("indices").resolve(index.getName()).resolve("0"); + Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0"); ShardRouting shardRouting = ShardRouting.newUnassigned(index, 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); ShardRoutingTestUtils.initialize(shardRouting, "node-0"); @@ -116,7 +116,7 @@ public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase< stats.indexing = new IndexingStats(); stats.search = new SearchStats(); stats.segments = new SegmentsStats(); - ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, null, shardId), stats, null); + ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null); FsInfo.Path[] pathInfo = new FsInfo.Path[]{ new FsInfo.Path("/test", "/dev/sda", 10, -8, 0), }; From 4ec4b0d7e158047e0fbfccb7395040b3cbbf1829 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 15 Mar 2016 10:07:10 +0100 Subject: [PATCH 05/27] Watcher should try to load trust/keystore from `config` directory Today Watcher tries to load stuff from the bin's parent directory which is not readable since the shared data directory has been moved out of the nodes parent in elasticsearchelastic/elasticsearch#17072 which causes security exception now. The test copies trust stores into the config dir and that's where we should read it from by default or even better explicitly configure the path?! Original commit: elastic/x-pack-elasticsearch@1d32a595cf488fd6db594d27abf95745f8264705 --- .../java/org/elasticsearch/marvel/cleaner/CleanerService.java | 4 +++- .../org/elasticsearch/watcher/support/http/HttpClient.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/cleaner/CleanerService.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/cleaner/CleanerService.java index 8804ecc7b42..b36a91c598e 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/cleaner/CleanerService.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/cleaner/CleanerService.java @@ -67,7 +67,9 @@ public class CleanerService extends AbstractLifecycleComponent { @Override protected void doClose() { logger.debug("closing cleaning service"); - runnable.cancel(); + if (runnable != null) { + runnable.cancel(); + } logger.debug("cleaning service closed"); } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java index 5d9082ea5b0..dc751ee27d2 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/http/HttpClient.java @@ -281,7 +281,7 @@ public class HttpClient extends AbstractLifecycleComponent { if (keyStore == null) { return null; } - Path path = env.binFile().getParent().resolve(keyStore); + Path path = env.configFile().resolve(keyStore); if (Files.notExists(path)) { return null; } @@ -304,7 +304,7 @@ public class HttpClient extends AbstractLifecycleComponent { // Load TrustStore KeyStore ks = null; if (trustStore != null) { - Path trustStorePath = env.binFile().getParent().resolve(trustStore); + Path trustStorePath = env.configFile().resolve(trustStore); if (Files.exists(trustStorePath)) { ks = readKeystore(trustStorePath, trustStorePassword); } From 61123bb10761f4cdabb7a3535fb78fe8ec2f885b Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 15 Mar 2016 16:29:34 +0100 Subject: [PATCH 06/27] Remove System.out.println and Throwable.printStackTrace from tests Relates to elastic/elasticsearchelastic/elasticsearch#17112 Original commit: elastic/x-pack-elasticsearch@404e40a4be882db3e43379d552e67c3447570746 --- .../test/java/org/elasticsearch/bench/HasherBenchmark.java | 3 +++ .../watcher/actions/email/service/support/EmailServer.java | 6 +++--- .../watcher/test/bench/ScheduleEngineTriggerBenchmark.java | 2 ++ .../watcher/test/bench/WatcherScheduleEngineBenchmark.java | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java index 13940993f5f..3400aba7d1f 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/bench/HasherBenchmark.java @@ -8,11 +8,13 @@ package org.elasticsearch.bench; import org.elasticsearch.common.Randomness; import com.carrotsearch.randomizedtesting.generators.RandomStrings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; +@SuppressForbidden(reason = "benchmark") public class HasherBenchmark { private static final int WARMING_ITERS = 1000; @@ -63,6 +65,7 @@ public class HasherBenchmark { return metrics; } + @SuppressForbidden(reason = "benchmark") private static class Metrics { final String name; diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java index 141dfd8f09f..f8379a6ec31 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/service/support/EmailServer.java @@ -42,7 +42,7 @@ public class EmailServer { private final SMTPServer server; - public EmailServer(String host, int port, final String username, final String password) { + public EmailServer(String host, int port, final String username, final String password, final ESLogger logger) { server = new SMTPServer(new SimpleMessageListenerAdapter(new SimpleMessageListener() { @Override public boolean accept(String from, String recipient) { @@ -58,8 +58,8 @@ public class EmailServer { try { listener.on(msg); } catch (Exception e) { + logger.error("Unexpected failure", e); fail(e.getMessage()); - e.printStackTrace(); } } } catch (MessagingException me) { @@ -104,7 +104,7 @@ public class EmailServer { @Override public boolean onPortNumber(int port) { try { - EmailServer server = new EmailServer("localhost", port, username, password); + EmailServer server = new EmailServer("localhost", port, username, password, logger); server.start(); emailServer.set(server); return true; diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java index 669d97d0c7f..45119805a49 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/ScheduleEngineTriggerBenchmark.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.test.bench; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.watcher.support.clock.SystemClock; @@ -33,6 +34,7 @@ import static org.elasticsearch.watcher.trigger.schedule.Schedules.interval; /** * */ +@SuppressForbidden(reason = "benchmark") public class ScheduleEngineTriggerBenchmark { public static void main(String[] args) throws Exception { diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java index a1c5ac8a0c4..373f27d8c34 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java @@ -55,6 +55,7 @@ import static org.elasticsearch.watcher.trigger.schedule.Schedules.interval; /** */ +@SuppressForbidden(reason = "benchmark") public class WatcherScheduleEngineBenchmark { private final static Settings SETTINGS = Settings.builder() @@ -64,7 +65,6 @@ public class WatcherScheduleEngineBenchmark { .put("http.cors.enabled", true) .build(); - @SuppressForbidden(reason = "not really code or a test") public static void main(String[] args) throws Exception { System.setProperty("es.logger.prefix", ""); @@ -252,6 +252,7 @@ public class WatcherScheduleEngineBenchmark { } } + @SuppressForbidden(reason = "benchmark") private static class BenchStats { private final String name; From 0136f16ce7777995c11f2b32911b6fc578b56bca Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 15 Mar 2016 16:48:14 +0100 Subject: [PATCH 07/27] [TEST] Suppress exception in Watcher benchmark Original commit: elastic/x-pack-elasticsearch@099cad0ac163b0daf325700e3461d7eb485f1422 --- .../watcher/test/bench/WatcherScheduleEngineBenchmark.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java index 373f27d8c34..87f324d933e 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/bench/WatcherScheduleEngineBenchmark.java @@ -169,9 +169,7 @@ public class WatcherScheduleEngineBenchmark { jvmUsedHeapSpace.inc(heapUsed.bytes()); Thread.sleep(1000); } - } catch (InterruptedException e) { - e.printStackTrace(); - } + } catch (InterruptedException ignored) {} } }); sampleThread.start(); From 03336912bba74b0c7e8d8d976d226c91c672d46b Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 15 Mar 2016 09:27:53 -0400 Subject: [PATCH 08/27] security: native realm is added when defined realms are unlicensed If a user configures only custom realms and they are not licensed to use the custom realms then we need to return our default realms. The default realms should be the esusers and esnative realms. We were only returning the esusers realm previously. Closes elastic/elasticsearch#1491 Original commit: elastic/x-pack-elasticsearch@3dc2b5d3a898cea99e5f184f0d32e80737d2d9f7 --- .../elasticsearch/shield/authc/Realms.java | 25 +++++------ .../shield/authc/RealmsTests.java | 41 ++++++++++++------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java index 35f565949ba..1508fbdb6e0 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java @@ -56,8 +56,7 @@ public class Realms extends AbstractLifecycleComponent implements Iterab } if (internalRealms.isEmpty()) { - // lets create a default one so they can do something - internalRealms.add(factories.get(ESUsersRealm.TYPE).createDefault("default_" + ESUsersRealm.TYPE)); + addInternalRealms(internalRealms); } this.internalRealmsOnly = Collections.unmodifiableList(internalRealms); @@ -130,16 +129,8 @@ public class Realms extends AbstractLifecycleComponent implements Iterab return realms; } - // there is no "realms" configuration, go over all the factories and try to create defaults - // for all the internal realms - Realm.Factory indexRealmFactory = factories.get(ESNativeRealm.TYPE); - if (indexRealmFactory != null) { - realms.add(indexRealmFactory.createDefault("default_" + ESNativeRealm.TYPE)); - } - Realm.Factory esUsersRealm = factories.get(ESUsersRealm.TYPE); - if (esUsersRealm != null) { - realms.add(esUsersRealm.createDefault("default_" + ESUsersRealm.TYPE)); - } + // there is no "realms" configuration, add the defaults + addInternalRealms(realms); return realms; } @@ -168,4 +159,14 @@ public class Realms extends AbstractLifecycleComponent implements Iterab return result != null ? result : Settings.EMPTY; } + private void addInternalRealms(List realms) { + Realm.Factory indexRealmFactory = factories.get(ESNativeRealm.TYPE); + if (indexRealmFactory != null) { + realms.add(indexRealmFactory.createDefault("default_" + ESNativeRealm.TYPE)); + } + Realm.Factory esUsersRealm = factories.get(ESUsersRealm.TYPE); + if (esUsersRealm != null) { + realms.add(esUsersRealm.createDefault("default_" + ESUsersRealm.TYPE)); + } + } } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java index 4dd47dd0081..1d6cbe3a7db 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RealmsTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; import org.elasticsearch.shield.User; +import org.elasticsearch.shield.authc.esnative.ESNativeRealm; import org.elasticsearch.shield.authc.esusers.ESUsersRealm; import org.elasticsearch.shield.authc.ldap.LdapRealm; import org.elasticsearch.shield.license.ShieldLicenseState; @@ -26,6 +27,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isOneOf; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,7 +42,8 @@ public class RealmsTests extends ESTestCase { @Before public void init() throws Exception { factories = new HashMap<>(); - factories.put("esusers", new DummyRealm.Factory("esusers", true)); + factories.put(ESUsersRealm.TYPE, new DummyRealm.Factory(ESUsersRealm.TYPE, true)); + factories.put(ESNativeRealm.TYPE, new DummyRealm.Factory(ESNativeRealm.TYPE, true)); for (int i = 0; i < randomIntBetween(1, 5); i++) { DummyRealm.Factory factory = new DummyRealm.Factory("type_" + i, rarely()); factories.put("type_" + i, factory); @@ -52,13 +55,13 @@ public class RealmsTests extends ESTestCase { public void testWithSettings() throws Exception { Settings.Builder builder = Settings.builder() .put("path.home", createTempDir()); - List orders = new ArrayList<>(factories.size() - 1); - for (int i = 0; i < factories.size() - 1; i++) { + List orders = new ArrayList<>(factories.size() - 2); + for (int i = 0; i < factories.size() - 2; i++) { orders.add(i); } Collections.shuffle(orders, random()); Map orderToIndex = new HashMap<>(); - for (int i = 0; i < factories.size() - 1; i++) { + for (int i = 0; i < factories.size() - 2; i++) { builder.put("shield.authc.realms.realm_" + i + ".type", "type_" + i); builder.put("shield.authc.realms.realm_" + i + ".order", orders.get(i)); orderToIndex.put(orders.get(i), i); @@ -102,6 +105,10 @@ public class RealmsTests extends ESTestCase { assertThat(iter.hasNext(), is(true)); Realm realm = iter.next(); assertThat(realm, notNullValue()); + assertThat(realm.type(), equalTo(ESNativeRealm.TYPE)); + assertThat(realm.name(), equalTo("default_" + ESNativeRealm.TYPE)); + assertThat(iter.hasNext(), is(true)); + realm = iter.next(); assertThat(realm.type(), equalTo(ESUsersRealm.TYPE)); assertThat(realm.name(), equalTo("default_" + ESUsersRealm.TYPE)); assertThat(iter.hasNext(), is(false)); @@ -110,13 +117,13 @@ public class RealmsTests extends ESTestCase { public void testUnlicensedWithOnlyCustomRealms() throws Exception { Settings.Builder builder = Settings.builder() .put("path.home", createTempDir()); - List orders = new ArrayList<>(factories.size() - 1); - for (int i = 0; i < factories.size() - 1; i++) { + List orders = new ArrayList<>(factories.size() - 2); + for (int i = 0; i < factories.size() - 2; i++) { orders.add(i); } Collections.shuffle(orders, random()); Map orderToIndex = new HashMap<>(); - for (int i = 0; i < factories.size() - 1; i++) { + for (int i = 0; i < factories.size() - 2; i++) { builder.put("shield.authc.realms.realm_" + i + ".type", "type_" + i); builder.put("shield.authc.realms.realm_" + i + ".order", orders.get(i)); orderToIndex.put(orders.get(i), i); @@ -138,10 +145,10 @@ public class RealmsTests extends ESTestCase { i = 0; when(shieldLicenseState.customRealmsEnabled()).thenReturn(false); for (Realm realm : realms) { - assertThat(realm.type, is(ESUsersRealm.TYPE)); + assertThat(realm.type, isOneOf(ESUsersRealm.TYPE, ESNativeRealm.TYPE)); i++; } - assertThat(i, is(1)); + assertThat(i, is(2)); } public void testUnlicensedWithInternalRealms() throws Exception { @@ -178,13 +185,13 @@ public class RealmsTests extends ESTestCase { public void testDisabledRealmsAreNotAdded() throws Exception { Settings.Builder builder = Settings.builder() .put("path.home", createTempDir()); - List orders = new ArrayList<>(factories.size() - 1); - for (int i = 0; i < factories.size() - 1; i++) { + List orders = new ArrayList<>(factories.size() - 2); + for (int i = 0; i < factories.size() - 2; i++) { orders.add(i); } Collections.shuffle(orders, random()); Map orderToIndex = new HashMap<>(); - for (int i = 0; i < factories.size() - 1; i++) { + for (int i = 0; i < factories.size() - 2; i++) { builder.put("shield.authc.realms.realm_" + i + ".type", "type_" + i); builder.put("shield.authc.realms.realm_" + i + ".order", orders.get(i)); boolean enabled = randomBoolean(); @@ -205,7 +212,11 @@ public class RealmsTests extends ESTestCase { Realm realm = iterator.next(); Integer index = orderToIndex.get(realm.order()); if (index == null) { - // Default realm is inserted when factories size is 1 and enabled is false + // Default realms are inserted when factories size is 1 and enabled is false + assertThat(realm.type(), equalTo(ESNativeRealm.TYPE)); + assertThat(realm.name(), equalTo("default_" + ESNativeRealm.TYPE)); + assertThat(iterator.hasNext(), is(true)); + realm = iterator.next(); assertThat(realm.type(), equalTo(ESUsersRealm.TYPE)); assertThat(realm.name(), equalTo("default_" + ESUsersRealm.TYPE)); assertThat(iterator.hasNext(), is(false)); @@ -264,8 +275,8 @@ public class RealmsTests extends ESTestCase { @Override public DummyRealm createDefault(String name) { - if (type().equals("esusers")) { - return new DummyRealm("esusers", new RealmConfig(name, Settings.EMPTY, + if (type().equals(ESNativeRealm.TYPE) || type().equals(ESUsersRealm.TYPE)) { + return new DummyRealm(type(), new RealmConfig(name, Settings.EMPTY, Settings.builder().put("path.home", createTempDir()).build())); } return null; From edc9580f66ff6ccfd86145dab49fe7e12f05c691 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 15 Mar 2016 10:06:54 -0400 Subject: [PATCH 09/27] security: validate that security and audit indices can be auto created Adds a check to the settings at startup to ensure that the security and audit indices are allowed to be auto created if a user has disabled auto create explicitly. Additionally fixes a small issue with the error message for watcher passing the incorrect value. Closes elastic/elasticsearch#1453 Original commit: elastic/x-pack-elasticsearch@2b0698ff19f84633329f4aab05769bbff7007788 --- .../java/org/elasticsearch/shield/Shield.java | 77 +++++++++++++++++++ .../shield/ShieldPluginSettingsTests.java | 57 ++++++++++++++ .../org/elasticsearch/watcher/Watcher.java | 2 +- .../watcher/WatcherPluginTests.java | 6 ++ 4 files changed, 141 insertions(+), 1 deletion(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java index 654b02d4dc4..57cc08d3be4 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java @@ -6,11 +6,15 @@ package org.elasticsearch.shield; import org.elasticsearch.action.ActionModule; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; @@ -35,6 +39,8 @@ import org.elasticsearch.shield.action.user.TransportPutUserAction; import org.elasticsearch.shield.action.user.TransportDeleteUserAction; import org.elasticsearch.shield.action.user.TransportGetUsersAction; import org.elasticsearch.shield.audit.AuditTrailModule; +import org.elasticsearch.shield.audit.index.IndexAuditTrail; +import org.elasticsearch.shield.audit.index.IndexNameResolver; import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail; import org.elasticsearch.shield.authc.AuthenticationModule; import org.elasticsearch.shield.authc.Realms; @@ -71,6 +77,8 @@ import org.elasticsearch.shield.transport.filter.IPFilter; import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport; import org.elasticsearch.shield.transport.netty.ShieldNettyTransport; import org.elasticsearch.xpack.XPackPlugin; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import java.util.ArrayList; import java.util.Arrays; @@ -102,6 +110,7 @@ public class Shield { this.enabled = XPackPlugin.featureEnabled(settings, NAME, true); if (enabled && !transportClientMode) { failIfShieldQueryCacheIsNotActive(settings, true); + validateAutoCreateIndex(settings); } } @@ -410,4 +419,72 @@ public class Shield { .INDEX_QUERY_CACHE_TYPE_SETTING.getKey() + "] with value [" + queryCacheImplementation + "]"); } } + + static void validateAutoCreateIndex(Settings settings) { + String value = settings.get("action.auto_create_index"); + if (value == null) { + return; + } + + final boolean indexAuditingEnabled = AuditTrailModule.indexAuditLoggingEnabled(settings); + final String auditIndex = indexAuditingEnabled ? "," + IndexAuditTrail.INDEX_NAME_PREFIX + "*" : ""; + String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too" + + " restrictive. disable [action.auto_create_index] or set it to " + + "[{}{}]", (Object) value, ShieldTemplateService.SECURITY_INDEX_NAME, auditIndex); + if (Booleans.isExplicitFalse(value)) { + throw new IllegalArgumentException(errorMessage); + } + + if (Booleans.isExplicitTrue(value)) { + return; + } + + String[] matches = Strings.commaDelimitedListToStringArray(value); + List indices = new ArrayList<>(); + indices.add(ShieldTemplateService.SECURITY_INDEX_NAME); + if (indexAuditingEnabled) { + DateTime now = new DateTime(DateTimeZone.UTC); + // just use daily rollover + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now, IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusDays(1), IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusMonths(1), IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusMonths(2), IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusMonths(3), IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusMonths(4), IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusMonths(5), IndexNameResolver.Rollover.DAILY)); + indices.add(IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now.plusMonths(6), IndexNameResolver.Rollover.DAILY)); + } + + for (String index : indices) { + boolean matched = false; + for (String match : matches) { + char c = match.charAt(0); + if (c == '-') { + if (Regex.simpleMatch(match.substring(1), index)) { + throw new IllegalArgumentException(errorMessage); + } + } else if (c == '+') { + if (Regex.simpleMatch(match.substring(1), index)) { + matched = true; + break; + } + } else { + if (Regex.simpleMatch(match, index)) { + matched = true; + break; + } + } + } + if (!matched) { + throw new IllegalArgumentException(errorMessage); + } + } + + if (indexAuditingEnabled) { + logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. " + + " for the next 6 months audit indices are allowed to be created, but please make sure" + + " that any future history indices after 6 months with the pattern " + + "[.shield_audit_log*] are allowed to be created", value); + } + } } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java index f9cb0e28196..7689bf174bf 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.shield; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.shield.audit.index.IndexAuditTrail; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.XPackPlugin; @@ -16,6 +17,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.not; public class ShieldPluginSettingsTests extends ESTestCase { @@ -132,4 +134,59 @@ public class ShieldPluginSettingsTests extends ESTestCase { assertThat(additionalSettings.get("tribe.t2.shield.bar"), is("foo")); assertThat(additionalSettings.getAsArray("tribe.t2.shield.something.else.here"), arrayContaining("foo", "bar")); } + + public void testValidAutoCreateIndex() { + Shield.validateAutoCreateIndex(Settings.EMPTY); + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", true).build()); + + try { + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", false).build()); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(ShieldTemplateService.SECURITY_INDEX_NAME)); + assertThat(e.getMessage(), not(containsString(IndexAuditTrail.INDEX_NAME_PREFIX))); + } + + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".security").build()); + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", "*s*").build()); + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".s*").build()); + + try { + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", "foo").build()); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(ShieldTemplateService.SECURITY_INDEX_NAME)); + assertThat(e.getMessage(), not(containsString(IndexAuditTrail.INDEX_NAME_PREFIX))); + } + + try { + Shield.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".shield_audit_log*").build()); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(ShieldTemplateService.SECURITY_INDEX_NAME)); + } + + Shield.validateAutoCreateIndex(Settings.builder() + .put("action.auto_create_index", ".security") + .put("shield.audit.enabled", true) + .build()); + + try { + Shield.validateAutoCreateIndex(Settings.builder() + .put("action.auto_create_index", ".security") + .put("shield.audit.enabled", true) + .put("shield.audit.outputs", randomFrom("index", "logfile,index")) + .build()); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(ShieldTemplateService.SECURITY_INDEX_NAME)); + assertThat(e.getMessage(), containsString(IndexAuditTrail.INDEX_NAME_PREFIX)); + } + + Shield.validateAutoCreateIndex(Settings.builder() + .put("action.auto_create_index", ".shield_audit_log*,.security") + .put("shield.audit.enabled", true) + .put("shield.audit.outputs", randomFrom("index", "logfile,index")) + .build()); + } } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java index 4109f07ab09..1d9a203f54c 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java @@ -271,7 +271,7 @@ public class Watcher { String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too" + " restrictive. disable [action.auto_create_index] or set it to " + - "[.watches,.triggered_watches,.watcher-history*]", (Object) settings); + "[.watches,.triggered_watches,.watcher-history*]", (Object) value); if (Booleans.isExplicitFalse(value)) { throw new IllegalArgumentException(errorMessage); } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java index cbddc934503..a1c9e59aeea 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginTests.java @@ -8,6 +8,8 @@ package org.elasticsearch.watcher; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import static org.hamcrest.Matchers.containsString; + public class WatcherPluginTests extends ESTestCase { public void testValidAutoCreateIndex() { @@ -17,6 +19,7 @@ public class WatcherPluginTests extends ESTestCase { Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", false).build()); fail("IllegalArgumentException expected"); } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history*]")); } Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watches,.triggered_watches,.watcher-history*").build()); @@ -26,16 +29,19 @@ public class WatcherPluginTests extends ESTestCase { Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watches").build()); fail("IllegalArgumentException expected"); } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history*]")); } try { Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".triggered_watch").build()); fail("IllegalArgumentException expected"); } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history*]")); } try { Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watcher-history*").build()); fail("IllegalArgumentException expected"); } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history*]")); } } From 69b69f7af1c206331ed1fdef3d031d7c3f085ece Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Mar 2016 20:01:01 -0400 Subject: [PATCH 10/27] Use setting in integration test cluster config This commit modifies using system properties to configure an integration test cluster and instead use settings in the generated Elasticsearch config file. Original commit: elastic/x-pack-elasticsearch@65211b93d0efcf0ecbf0fc2b6b93c3b71171131d --- .../qa/shield-audit-tests/build.gradle | 4 ++-- .../qa/shield-core-rest-tests/build.gradle | 4 ++-- .../qa/shield-example-realm/build.gradle | 8 +++---- .../qa/shield-reindex-tests/build.gradle | 2 +- .../qa/smoke-test-plugins-ssl/build.gradle | 22 +++++++++---------- .../build.gradle | 6 ++--- .../build.gradle | 6 ++--- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/elasticsearch/qa/shield-audit-tests/build.gradle b/elasticsearch/qa/shield-audit-tests/build.gradle index e06acc7f91c..749613c9cc4 100644 --- a/elasticsearch/qa/shield-audit-tests/build.gradle +++ b/elasticsearch/qa/shield-audit-tests/build.gradle @@ -7,8 +7,8 @@ dependencies { integTest { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') - systemProperty 'es.shield.audit.enabled', 'true' - systemProperty 'es.shield.audit.outputs', 'index' + setting 'shield.audit.enabled', 'true' + setting 'shield.audit.outputs', 'index' setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' waitCondition = { node, ant -> diff --git a/elasticsearch/qa/shield-core-rest-tests/build.gradle b/elasticsearch/qa/shield-core-rest-tests/build.gradle index 97d1915b5a8..d8580fcac10 100644 --- a/elasticsearch/qa/shield-core-rest-tests/build.gradle +++ b/elasticsearch/qa/shield-core-rest-tests/build.gradle @@ -34,8 +34,8 @@ integTest { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') - systemProperty 'es.xpack.watcher.enabled', 'false' - systemProperty 'es.xpack.monitoring.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' waitCondition = { node, ant -> diff --git a/elasticsearch/qa/shield-example-realm/build.gradle b/elasticsearch/qa/shield-example-realm/build.gradle index c7162d4cb45..b6e65102ac4 100644 --- a/elasticsearch/qa/shield-example-realm/build.gradle +++ b/elasticsearch/qa/shield-example-realm/build.gradle @@ -17,10 +17,10 @@ integTest { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') // TODO: these should be settings? - systemProperty 'es.shield.authc.realms.custom.order', '0' - systemProperty 'es.shield.authc.realms.custom.type', 'custom' - systemProperty 'es.shield.authc.realms.esusers.order', '1' - systemProperty 'es.shield.authc.realms.esusers.type', 'esusers' + setting 'shield.authc.realms.custom.order', '0' + setting 'shield.authc.realms.custom.type', 'custom' + setting 'shield.authc.realms.esusers.order', '1' + setting 'shield.authc.realms.esusers.type', 'esusers' setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' diff --git a/elasticsearch/qa/shield-reindex-tests/build.gradle b/elasticsearch/qa/shield-reindex-tests/build.gradle index 4eec32ef935..6bfd5017390 100644 --- a/elasticsearch/qa/shield-reindex-tests/build.gradle +++ b/elasticsearch/qa/shield-reindex-tests/build.gradle @@ -6,7 +6,7 @@ dependencies { integTest { cluster { - systemProperty 'es.script.inline', 'true' + setting 'script.inline', 'true' plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') extraConfigFile 'xpack/roles.yml', 'roles.yml' [ diff --git a/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle index 733e6c19856..8de1ff62414 100644 --- a/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle +++ b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle @@ -145,18 +145,18 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each integTest { cluster { - systemProperty 'es.xpack.monitoring.agent.interval', '3s' - systemProperty 'es.xpack.monitoring.agent.exporters._http.type', 'http' - systemProperty 'es.xpack.monitoring.agent.exporters._http.enabled', 'false' - systemProperty 'es.xpack.monitoring.agent.exporters._http.ssl.truststore.path', clientKeyStore.name - systemProperty 'es.xpack.monitoring.agent.exporters._http.ssl.truststore.password', 'keypass' - systemProperty 'es.xpack.monitoring.agent.exporters._http.auth.username', 'monitoring_agent' - systemProperty 'es.xpack.monitoring.agent.exporters._http.auth.password', 'changeme' + setting 'xpack.monitoring.agent.interval', '3s' + setting 'xpack.monitoring.agent.exporters._http.type', 'http' + setting 'xpack.monitoring.agent.exporters._http.enabled', 'false' + setting 'xpack.monitoring.agent.exporters._http.ssl.truststore.path', clientKeyStore.name + setting 'xpack.monitoring.agent.exporters._http.ssl.truststore.password', 'keypass' + setting 'xpack.monitoring.agent.exporters._http.auth.username', 'monitoring_agent' + setting 'xpack.monitoring.agent.exporters._http.auth.password', 'changeme' - systemProperty 'es.shield.transport.ssl', 'true' - systemProperty 'es.shield.http.ssl', 'true' - systemProperty 'es.shield.ssl.keystore.path', nodeKeystore.name - systemProperty 'es.shield.ssl.keystore.password', 'keypass' + setting 'shield.transport.ssl', 'true' + setting 'shield.http.ssl', 'true' + setting 'shield.ssl.keystore.path', nodeKeystore.name + setting 'shield.ssl.keystore.password', 'keypass' plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') diff --git a/elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle b/elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle index eb357005ccc..9c2fb304d8d 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle +++ b/elasticsearch/qa/smoke-test-watcher-with-groovy/build.gradle @@ -8,8 +8,8 @@ dependencies { integTest { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') - systemProperty 'es.script.inline', 'true' - systemProperty 'es.xpack.shield.enabled', 'false' - systemProperty 'es.xpack.monitoring.enabled', 'false' + setting 'script.inline', 'true' + setting 'xpack.shield.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' } } diff --git a/elasticsearch/qa/smoke-test-watcher-with-mustache/build.gradle b/elasticsearch/qa/smoke-test-watcher-with-mustache/build.gradle index efef108a36d..37575d2605b 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-mustache/build.gradle +++ b/elasticsearch/qa/smoke-test-watcher-with-mustache/build.gradle @@ -8,8 +8,8 @@ dependencies { integTest { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') - systemProperty 'es.xpack.shield.enabled', 'false' - systemProperty 'es.xpack.monitoring.enabled', 'false' - systemProperty 'es.http.port', '9400' + setting 'xpack.shield.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'http.port', '9400' } } From 787ebd58509f8f7010e903a02ac324f1fec9046f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 16 Mar 2016 07:52:15 -0400 Subject: [PATCH 11/27] Handle task status registration cleanup Core reworked how it registered tasks status's with NamedWriteableRegistry so it was more pluggable. It changed a few signatures and x-plugins needs these small changes to keep compiling. Original commit: elastic/x-pack-elasticsearch@3dcf1df152aba3019ee0a3c70b9fa695fab79820 --- .../shield/transport/ShieldClientTransportService.java | 7 ++----- .../shield/transport/ShieldServerTransportService.java | 5 ++--- .../elasticsearch/shield/audit/AuditTrailModuleTests.java | 3 ++- .../shield/transport/TransportFilterTests.java | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java index c181c5e8152..efe014c74c8 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldClientTransportService.java @@ -7,7 +7,6 @@ package org.elasticsearch.shield.transport; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; @@ -23,13 +22,11 @@ public class ShieldClientTransportService extends TransportService { private final ClientTransportFilter clientFilter; @Inject - public ShieldClientTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClientTransportFilter clientFilter, - NamedWriteableRegistry namedWriteableRegistry) { - super(settings, transport, threadPool, namedWriteableRegistry); + public ShieldClientTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClientTransportFilter clientFilter) { + super(settings, transport, threadPool); this.clientFilter = clientFilter; } - @Override public void sendRequest(DiscoveryNode node, String action, TransportRequest request, TransportRequestOptions options, TransportResponseHandler handler) { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java index 00b9d36a26a..0e4dad913f7 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/transport/ShieldServerTransportService.java @@ -63,9 +63,8 @@ public class ShieldServerTransportService extends TransportService { AuthorizationService authzService, ShieldActionMapper actionMapper, ClientTransportFilter clientTransportFilter, - ShieldLicenseState licenseState, - NamedWriteableRegistry namedWriteableRegistry) { - super(settings, transport, threadPool, namedWriteableRegistry); + ShieldLicenseState licenseState) { + super(settings, transport, threadPool); this.authcService = authcService; this.authzService = authzService; this.actionMapper = actionMapper; diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java index d43cc4ed032..3cea12bc6f7 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailModuleTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.shield.audit; import org.elasticsearch.Version; import org.elasticsearch.common.inject.Guice; import org.elasticsearch.common.inject.Injector; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Setting; @@ -60,7 +61,7 @@ public class AuditTrailModuleTests extends ESTestCase { settingsModule.registerSetting(Setting.boolSetting("shield.audit.enabled", true, Setting.Property.NodeScope)); Injector injector = Guice.createInjector( settingsModule, - new NetworkModule(new NetworkService(settings), settings, false, null) { + new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry()) { @Override protected void configure() { bind(Transport.class).to(LocalTransport.class).asEagerSingleton(); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java index 45ebaaaa043..7c025d70698 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/transport/TransportFilterTests.java @@ -309,9 +309,9 @@ public class TransportFilterTests extends ESIntegTestCase { @Inject public InternalPluginServerTransportService(Settings settings, Transport transport, ThreadPool threadPool, AuthenticationService authcService, AuthorizationService authzService, ShieldActionMapper actionMapper, - ClientTransportFilter clientTransportFilter, NamedWriteableRegistry namedWriteableRegistry) { + ClientTransportFilter clientTransportFilter) { super(settings, transport, threadPool, authcService, authzService, actionMapper, clientTransportFilter, - mock(ShieldLicenseState.class), namedWriteableRegistry); + mock(ShieldLicenseState.class)); when(licenseState.securityEnabled()).thenReturn(true); } From e3551a757043a350c74ba209b4aedb09a71a7ff3 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 16 Mar 2016 13:44:33 +0000 Subject: [PATCH 12/27] [TEST] fix timeout test so it checks message but not the reported time waiting The reported time waiting for watches can be slightly different from the actual timeout (e.g. 2.1 seconds instead of 2 seconds) so checking the time waited in the message makes the test sometimes fail Original commit: elastic/x-pack-elasticsearch@c2cd9da4861efc5c7ff35a593a119b879542bc4a --- .../org/elasticsearch/watcher/watch/WatchLockServiceTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java index dbd7108dadd..7529565cac1 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/watch/WatchLockServiceTests.java @@ -18,6 +18,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; /** */ @@ -64,7 +65,7 @@ public class WatchLockServiceTests extends ESTestCase { lockService.stop(); fail("Expected ElasticsearchTimeoutException"); } catch (ElasticsearchTimeoutException e) { - assertThat(e.getMessage(), is("timed out waiting for watches to complete, after waiting for [2s]")); + assertThat(e.getMessage(), startsWith("timed out waiting for watches to complete, after waiting for")); } } From 7d8149cb8633b56b320094ad2f4d9a71a2e8a06c Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Mon, 29 Feb 2016 12:37:11 -0500 Subject: [PATCH 13/27] Suggester refactoring requires a Suggester to parse X-Content to Builder As part of the search refactoring effort, we need to pass a Suggester to the methods that parse X-Content to a SuggestBuilder in every instance where we are parsing search/suggest requests. Original commit: elastic/x-pack-elasticsearch@7f815c617a736feaa6d7f53333ba3fd2c4dd741f --- .../org/elasticsearch/messy/tests/SearchInputIT.java | 3 ++- .../elasticsearch/messy/tests/SearchTransformIT.java | 3 +-- .../SearchGetAndSuggestPermissionsTests.java | 4 ++-- .../watcher/input/search/SearchInput.java | 7 +++++-- .../watcher/input/search/SearchInputFactory.java | 11 +++++++---- .../elasticsearch/watcher/support/WatcherUtils.java | 5 +++-- .../watcher/transform/search/SearchTransform.java | 7 +++++-- .../transform/search/SearchTransformFactory.java | 11 +++++++---- .../watcher/support/WatcherUtilsTests.java | 4 ++-- .../org/elasticsearch/watcher/watch/WatchTests.java | 4 ++-- 10 files changed, 36 insertions(+), 23 deletions(-) diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java index 4129f3a1c31..00eaab9546a 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java @@ -281,7 +281,8 @@ public class SearchInputIT extends ESIntegTestCase { parser.nextToken(); IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class); - SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, WatcherClientProxy.of(client()), indicesQueryRegistry, null); + SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, WatcherClientProxy.of(client()), indicesQueryRegistry, + null, null); SearchInput searchInput = factory.parseInput("_id", parser); assertEquals(SearchInput.TYPE, searchInput.type()); diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java index df2bfd00382..87bc114cc88 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java @@ -319,8 +319,7 @@ public class SearchTransformIT extends ESIntegTestCase { IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class); SearchTransformFactory transformFactory = new SearchTransformFactory(Settings.EMPTY, WatcherClientProxy.of(client()), - indicesQueryRegistry, - null); + indicesQueryRegistry, null, null); ExecutableSearchTransform executable = transformFactory.parseExecutable("_id", parser); assertThat(executable, notNullValue()); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java index 9e3c4604a93..796aab3c3cb 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java @@ -83,14 +83,14 @@ public class SearchGetAndSuggestPermissionsTests extends ShieldIntegTestCase { Map headers = singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("suggest_user", "passwd")); SuggestResponse suggestResponse = client.filterWithHeader(headers) .prepareSuggest("a") - .addSuggestion(SuggestBuilders.termSuggestion("name").field("name").text("val")).get(); + .addSuggestion(randomAsciiOfLengthBetween(3,7), SuggestBuilders.termSuggestion("name").text("val")).get(); assertNoFailures(suggestResponse); assertThat(suggestResponse.getSuggest().size(), is(1)); suggestResponse = client .filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("search_user", "passwd"))) .prepareSuggest("a") - .addSuggestion(SuggestBuilders.termSuggestion("name").field("name").text("val")).get(); + .addSuggestion(randomAsciiOfLengthBetween(3, 7), SuggestBuilders.termSuggestion("name").text("val")).get(); assertNoFailures(suggestResponse); assertThat(suggestResponse.getSuggest().size(), is(1)); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java index 4b387f92c20..fbc247eaba1 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInput.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.watcher.input.Input; import org.elasticsearch.watcher.support.SearchRequestEquivalence; import org.elasticsearch.watcher.support.WatcherDateTimeUtils; @@ -111,7 +112,8 @@ public class SearchInput implements Input { return builder; } - public static SearchInput parse(String watchId, XContentParser parser, QueryParseContext context, AggregatorParsers aggParsers) + public static SearchInput parse(String watchId, XContentParser parser, QueryParseContext context, + AggregatorParsers aggParsers, Suggesters suggesters) throws IOException { SearchRequest request = null; Set extract = null; @@ -125,7 +127,8 @@ public class SearchInput implements Input { currentFieldName = parser.currentName(); } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Field.REQUEST)) { try { - request = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, aggParsers); + request = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, + aggParsers, suggesters); } catch (ElasticsearchParseException srpe) { throw new ElasticsearchParseException("could not parse [{}] input for watch [{}]. failed to parse [{}]", srpe, TYPE, watchId, currentFieldName); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java index 531103ebe11..ca8f6003ffd 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/input/search/SearchInputFactory.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.watcher.input.InputFactory; import org.elasticsearch.watcher.input.simple.ExecutableSimpleInput; import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy; @@ -26,16 +27,18 @@ public class SearchInputFactory extends InputFactory Date: Tue, 15 Mar 2016 10:45:23 -0400 Subject: [PATCH 14/27] security: cleanup the logging in the native stores A lot of messages were being logged at the info level in the native user and roles stores. This changes the logging to be more selective in the cases where the index does not exist or the error is really an error and the user should be notified. Closes elastic/elasticsearch#1339 Original commit: elastic/x-pack-elasticsearch@0bc0d9bf7ae4f939abd366a906002932ffd97b51 --- .../authc/esnative/ESNativeUsersStore.java | 76 ++++++++++++------- .../authz/esnative/ESNativeRolesStore.java | 73 +++++++++++------- 2 files changed, 93 insertions(+), 56 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java index 1e5f15e5f5e..babbb5fd773 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java @@ -60,6 +60,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteTransportException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -148,7 +149,7 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat if (t instanceof IndexNotFoundException) { logger.trace("failed to retrieve user [{}] since security index does not exist", username); } else { - logger.info("failed to retrieve user [{}]", t, username); + logger.debug("failed to retrieve user [{}]", t, username); } // We don't invoke the onFailure listener here, instead @@ -186,8 +187,12 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat // This function is MADNESS! But it works, don't think about it too hard... client.search(request, new ActionListener() { + + private SearchResponse lastResponse = null; + @Override public void onResponse(final SearchResponse resp) { + lastResponse = resp; boolean hasHits = resp.getHits().getHits().length > 0; if (hasHits) { for (SearchHit hit : resp.getHits().getHits()) { @@ -200,19 +205,9 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat .setScroll(scrollKeepAlive).request(); client.searchScroll(scrollRequest, this); } else { - ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(resp.getScrollId()).request(); - client.clearScroll(clearScrollRequest, new ActionListener() { - @Override - public void onResponse(ClearScrollResponse response) { - // cool, it cleared, we don't really care though... - } - - @Override - public void onFailure(Throwable t) { - // Not really much to do here except for warn about it... - logger.warn("failed to clear scroll [{}] after retrieving all users", t, resp.getScrollId()); - } - }); + if (resp.getScrollId() != null) { + clearScrollResponse(resp.getScrollId()); + } // Finally, return the list of users listener.onResponse(Collections.unmodifiableList(users)); } @@ -220,23 +215,28 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat @Override public void onFailure(Throwable t) { + // attempt to clear scroll response + if (lastResponse != null && lastResponse.getScrollId() != null) { + clearScrollResponse(lastResponse.getScrollId()); + } + if (t instanceof IndexNotFoundException) { logger.trace("could not retrieve users because security index does not exist"); + // We don't invoke the onFailure listener here, instead just pass an empty list + listener.onResponse(Collections.emptyList()); } else { - logger.info("failed to retrieve users", t); + listener.onFailure(t); } - // We don't invoke the onFailure listener here, instead - // we call the response with an empty list - listener.onResponse(Collections.emptyList()); + } }); } catch (Exception e) { - logger.error("unable to retrieve users", e); + logger.error("unable to retrieve users {}", e, Arrays.toString(usernames)); listener.onFailure(e); } } - private UserAndPassword getUserAndPassword(String username) { + private UserAndPassword getUserAndPassword(final String username) { final AtomicReference userRef = new AtomicReference<>(null); final CountDownLatch latch = new CountDownLatch(1); getUserAndPassword(username, new LatchedActionListener<>(new ActionListener() { @@ -247,19 +247,23 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat @Override public void onFailure(Throwable t) { - logger.info("failed to retrieve user", t); + if (t instanceof IndexNotFoundException) { + logger.trace("failed to retrieve user [{}] since security index does not exist", t, username); + } else { + logger.error("failed to retrieve user [{}]", t, username); + } } }, latch)); try { latch.await(30, TimeUnit.SECONDS); } catch (InterruptedException e) { - logger.info("timed out retrieving user"); + logger.error("timed out retrieving user [{}]", username); return null; } return userRef.get(); } - private void getUserAndPassword(String user, final ActionListener listener) { + private void getUserAndPassword(final String user, final ActionListener listener) { try { GetRequest request = client.prepareGet(ShieldTemplateService.SECURITY_INDEX_NAME, USER_DOC_TYPE, user).request(); client.get(request, new ActionListener() { @@ -271,9 +275,9 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat @Override public void onFailure(Throwable t) { if (t instanceof IndexNotFoundException) { - logger.trace("could not retrieve user because security index does not exist", t); + logger.trace("could not retrieve user [{}] because security index does not exist", t, user); } else { - logger.info("failed to retrieve user", t); + logger.error("failed to retrieve user [{}]", t, user); } // We don't invoke the onFailure listener here, instead // we call the response with a null user @@ -281,10 +285,10 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat } }); } catch (IndexNotFoundException infe) { - logger.trace("could not retrieve user because security index does not exist"); + logger.trace("could not retrieve user [{}] because security index does not exist", user); listener.onResponse(null); } catch (Exception e) { - logger.error("unable to retrieve user", e); + logger.error("unable to retrieve user [{}]", e, user); listener.onFailure(e); } } @@ -501,6 +505,22 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat listeners.add(listener); } + private void clearScrollResponse(String scrollId) { + ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(scrollId).request(); + client.clearScroll(clearScrollRequest, new ActionListener() { + @Override + public void onResponse(ClearScrollResponse response) { + // cool, it cleared, we don't really care though... + } + + @Override + public void onFailure(Throwable t) { + // Not really much to do here except for warn about it... + logger.warn("failed to clear scroll [{}]", t, scrollId); + } + }); + } + private void clearRealmCache(String username, ActionListener listener, Response response) { SecurityClient securityClient = new SecurityClient(client); ClearRealmCacheRequest request = securityClient.prepareClearRealmCache() @@ -565,7 +585,7 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat Map metadata = (Map) sourceMap.get(User.Fields.METADATA.getPreferredName()); return new UserAndPassword(new User(username, roles, fullName, email, metadata), password.toCharArray()); } catch (Exception e) { - logger.error("error in the format of get response for user", e); + logger.error("error in the format of data for user [{}]", e, username); return null; } } diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java index 792c7710e4a..98396187539 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java @@ -44,7 +44,6 @@ import org.elasticsearch.shield.action.role.ClearRolesCacheRequest; import org.elasticsearch.shield.action.role.ClearRolesCacheResponse; import org.elasticsearch.shield.action.role.DeleteRoleRequest; import org.elasticsearch.shield.action.role.PutRoleRequest; -import org.elasticsearch.shield.authc.AuthenticationService; import org.elasticsearch.shield.authz.RoleDescriptor; import org.elasticsearch.shield.authz.permission.Role; import org.elasticsearch.shield.authz.store.RolesStore; @@ -52,6 +51,7 @@ import org.elasticsearch.shield.client.SecurityClient; import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -101,8 +101,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, private volatile boolean shieldIndexExists = false; @Inject - public ESNativeRolesStore(Settings settings, Provider clientProvider, - Provider authProvider, ThreadPool threadPool) { + public ESNativeRolesStore(Settings settings, Provider clientProvider, ThreadPool threadPool) { super(settings); this.clientProvider = clientProvider; this.threadPool = threadPool; @@ -192,8 +191,12 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, // This function is MADNESS! But it works, don't think about it too hard... client.search(request, new ActionListener() { + + private SearchResponse lastResponse = null; + @Override public void onResponse(SearchResponse resp) { + lastResponse = resp; boolean hasHits = resp.getHits().getHits().length > 0; if (hasHits) { for (SearchHit hit : resp.getHits().getHits()) { @@ -206,19 +209,9 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, .setScroll(scrollKeepAlive).request(); client.searchScroll(scrollRequest, this); } else { - ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(resp.getScrollId()).request(); - client.clearScroll(clearScrollRequest, new ActionListener() { - @Override - public void onResponse(ClearScrollResponse response) { - // cool, it cleared, we don't really care though... - } - - @Override - public void onFailure(Throwable t) { - // Not really much to do here except for warn about it... - logger.warn("failed to clear scroll after retrieving all roles", t); - } - }); + if (resp.getScrollId() != null) { + clearScollRequest(resp.getScrollId()); + } // Finally, return the list of users listener.onResponse(Collections.unmodifiableList(roles)); } @@ -226,18 +219,22 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, @Override public void onFailure(Throwable t) { + // attempt to clear the scroll request + if (lastResponse != null && lastResponse.getScrollId() != null) { + clearScollRequest(lastResponse.getScrollId()); + } + if (t instanceof IndexNotFoundException) { logger.trace("could not retrieve roles because security index does not exist"); + // since this is expected to happen at times, we just call the listener with an empty list + listener.onResponse(Collections.emptyList()); } else { - logger.info("failed to retrieve roles", t); + listener.onFailure(t); } - // We don't invoke the onFailure listener here, instead - // we call the response with an empty list - listener.onResponse(Collections.emptyList()); } }); } catch (Exception e) { - logger.error("unable to retrieve roles", e); + logger.error("unable to retrieve roles {}", e, Arrays.toString(names)); listener.onFailure(e); } } @@ -283,7 +280,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, public void putRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { if (state() != State.STARTED) { - logger.trace("attempted to put role before service was started"); + logger.trace("attempted to put role [{}] before service was started", request.name()); listener.onResponse(false); } try { @@ -302,12 +299,12 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, @Override public void onFailure(Throwable e) { - logger.error("failed to put role to the index", e); + logger.error("failed to put role [{}]", e, request.name()); listener.onFailure(e); } }); } catch (Exception e) { - logger.error("unable to put role", e); + logger.error("unable to put role [{}]", e, request.name()); listener.onFailure(e); } @@ -336,14 +333,18 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, @Override public void onFailure(Throwable t) { - logger.info("failed to retrieve role", t); + if (t instanceof IndexNotFoundException) { + logger.trace("failed to retrieve role [{}] since security index does not exist", t, roleId); + } else { + logger.error("failed to retrieve role [{}]", t, roleId); + } } }, latch)); try { latch.await(30, TimeUnit.SECONDS); } catch (InterruptedException e) { - logger.info("timed out retrieving role"); + logger.error("timed out retrieving role [{}]", roleId); } GetResponse response = getRef.get(); @@ -371,7 +372,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, GetRequest request = client.prepareGet(ShieldTemplateService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role).request(); client.get(request, listener); } catch (IndexNotFoundException e) { - logger.trace("security index does not exist", e); + logger.trace("unable to retrieve role [{}] since security index does not exist", e, role); listener.onResponse(new GetResponse( new GetResult(ShieldTemplateService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role, -1, false, null, null))); } catch (Exception e) { @@ -380,6 +381,22 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, } } + private void clearScollRequest(final String scrollId) { + ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(scrollId).request(); + client.clearScroll(clearScrollRequest, new ActionListener() { + @Override + public void onResponse(ClearScrollResponse response) { + // cool, it cleared, we don't really care though... + } + + @Override + public void onFailure(Throwable t) { + // Not really much to do here except for warn about it... + logger.warn("failed to clear scroll [{}] after retrieving roles", t, scrollId); + } + }); + } + // FIXME hack for testing public void reset() { final State state = state(); @@ -452,7 +469,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore, try { return RoleDescriptor.parse(name, sourceBytes); } catch (Exception e) { - logger.warn("unable to deserialize role from response", e); + logger.error("error in the format of data for role [{}]", e, name); return null; } } From 1161edca2c24ba74a96825a3557b5af364d91862 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 16 Mar 2016 10:43:14 +0100 Subject: [PATCH 15/27] Monitoring: Do not expose sensitive settings Original commit: elastic/x-pack-elasticsearch@25d81bb7b6fc488762a292063ceb1371459a8386 --- .../20_settings_filter.yaml | 19 ++++ .../elasticsearch/marvel/MarvelSettings.java | 3 +- .../shield/MarvelSettingsFilterTests.java | 98 +++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 elasticsearch/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/20_settings_filter.yaml create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelSettingsFilterTests.java diff --git a/elasticsearch/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/20_settings_filter.yaml b/elasticsearch/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/20_settings_filter.yaml new file mode 100644 index 00000000000..b1b93edc116 --- /dev/null +++ b/elasticsearch/qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/20_settings_filter.yaml @@ -0,0 +1,19 @@ +# Integration tests for smoke testing plugins +# +"Secret settings are correctly filtered": + - do: + cluster.state: {} + + - set: {master_node: master} + + - do: + nodes.info: + metric: [ settings ] + + - is_true: nodes + - is_true: nodes.$master.settings.xpack.monitoring.agent.exporters._http.type + + - is_false: nodes.$master.settings.xpack.monitoring.agent.exporters._http.auth.username + - is_false: nodes.$master.settings.xpack.monitoring.agent.exporters._http.auth.password + - is_false: nodes.$master.settings.xpack.monitoring.agent.exporters._http.ssl.truststore.path + - is_false: nodes.$master.settings.xpack.monitoring.agent.exporters._http.ssl.truststore.password diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java index 8a1759acb67..37cf6ee90d4 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java @@ -134,7 +134,8 @@ public class MarvelSettings extends AbstractComponent { module.registerSetting(ENABLED); module.registerSetting(INDEX_TEMPLATE_VERSION); - module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.auth.password"); + module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.auth.*"); + module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.ssl.*"); } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelSettingsFilterTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelSettingsFilterTests.java new file mode 100644 index 00000000000..cf7c6459fcf --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelSettingsFilterTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.shield; + +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.marvel.MarvelSettings; +import org.elasticsearch.marvel.test.MarvelIntegTestCase; +import org.elasticsearch.shield.authc.support.SecuredString; +import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; +import org.elasticsearch.test.rest.client.http.HttpResponse; +import org.hamcrest.Matchers; +import org.junit.After; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; +import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; +import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.CoreMatchers.nullValue; + +public class MarvelSettingsFilterTests extends MarvelIntegTestCase { + + private CloseableHttpClient httpClient = HttpClients.createDefault(); + + @After + public void cleanup() throws IOException { + httpClient.close(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put(MarvelSettings.INTERVAL.getKey(), "-1") + .put("xpack.monitoring.agent.exporters._http.type", "http") + .put("xpack.monitoring.agent.exporters._http.enabled", false) + .put("xpack.monitoring.agent.exporters._http.auth.username", "_user") + .put("xpack.monitoring.agent.exporters._http.auth.password", "_passwd") + .put("xpack.monitoring.agent.exporters._http.ssl.truststore.path", "/path/to/truststore") + .put("xpack.monitoring.agent.exporters._http.ssl.truststore.password", "_passwd") + .put("xpack.monitoring.agent.exporters._http.ssl.hostname_verification", true) + .build(); + } + + public void testGetSettingsFiltered() throws Exception { + String body = executeRequest("GET", "/_nodes/settings", null, null).getBody(); + Map response = JsonXContent.jsonXContent.createParser(body).map(); + Map nodes = (Map) response.get("nodes"); + for (Object node : nodes.values()) { + Map settings = (Map) ((Map) node).get("settings"); + + assertThat(extractValue("xpack.monitoring.agent.exporters._http.type", settings), Matchers.equalTo("http")); + assertThat(extractValue("xpack.monitoring.agent.exporters._http.enabled", settings), Matchers.equalTo("false")); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.username"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.password"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.path"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.password"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.hostname_verification"); + } + } + + private void assertNullSetting(Map settings, String setting) { + assertThat(extractValue(setting, settings), nullValue()); + } + + protected HttpResponse executeRequest(String method, String path, String body, Map params) throws IOException { + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class, + internalCluster().getMasterName()); + HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient) + .httpTransport(httpServerTransport) + .method(method) + .path(path); + + if (params != null) { + for (Map.Entry entry : params.entrySet()) { + requestBuilder.addParam(entry.getKey(), entry.getValue()); + } + } + if (body != null) { + requestBuilder.body(body); + } + if (shieldEnabled) { + requestBuilder.addHeader(BASIC_AUTH_HEADER, + basicAuthHeaderValue(ShieldSettings.TEST_USERNAME, new SecuredString(ShieldSettings.TEST_PASSWORD.toCharArray()))); + } + return requestBuilder.execute(); + } +} From 9e08579d4f3e118c56e6c7ef8c2ca2822890a313 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 15 Mar 2016 13:10:40 -0400 Subject: [PATCH 16/27] security: file parsing only supports the new format This commit remove the pre-existing file parsing code and replaces it with the updated code in the RoleDescriptor class. This unifies the parsing for the files and API for roles. Closes elastic/elasticsearch#1596 Original commit: elastic/x-pack-elasticsearch@9e0b58fcf1edb72913c30d05c6fc6031309d62bf --- .../qa/shield-reindex-tests/roles.yml | 66 +++-- .../qa/smoke-test-graph-with-shield/roles.yml | 29 +- .../smoke-test-watcher-with-shield/roles.yml | 27 +- .../marvel/test/MarvelIntegTestCase.java | 17 +- .../x-pack/shield/config/xpack/roles.yml | 103 ++++++-- .../shield/authz/RoleDescriptor.java | 93 ++++--- .../shield/authz/permission/Role.java | 6 +- .../shield/authz/store/FileRolesStore.java | 248 ++---------------- .../integration/ClusterPrivilegeTests.java | 7 +- .../DocumentAndFieldLevelSecurityTests.java | 24 +- .../DocumentLevelSecurityRandomTests.java | 6 +- .../DocumentLevelSecurityTests.java | 12 +- .../FieldLevelSecurityRandomTests.java | 26 +- .../integration/FieldLevelSecurityTests.java | 49 ++-- .../integration/IndexPrivilegeTests.java | 51 ++-- ...onsWithAliasesWildcardsAndRegexsTests.java | 20 +- .../integration/LicensingTests.java | 20 +- .../MultipleIndicesPermissionsTests.java | 20 +- .../PermissionPrecedenceTests.java | 8 +- .../SearchGetAndSuggestPermissionsTests.java | 9 +- .../integration/ShieldClearScrollTests.java | 3 +- .../ldap/AbstractAdLdapRealmTestCase.java | 22 +- .../integration/ldap/GroupMappingTests.java | 2 +- .../ldap/MultiGroupMappingTests.java | 5 +- .../shield/authc/AnonymousUserTests.java | 3 +- .../shield/authc/RunAsIntegTests.java | 4 +- .../shield/authz/AnalyzeTests.java | 3 +- .../shield/authz/IndexAliasesTests.java | 21 +- ...cesAndAliasesResolverIntegrationTests.java | 8 +- .../authz/store/FileRolesStoreTests.java | 38 ++- .../test/ShieldSettingsSource.java | 5 +- .../shield/authz/store/default_roles.yml | 128 +++++++-- .../shield/authz/store/invalid_roles.yml | 26 +- .../shield/authz/store/reserved_roles.yml | 20 +- .../shield/authz/store/roles.yml | 60 +++-- .../AbstractWatcherIntegrationTestCase.java | 15 +- 36 files changed, 658 insertions(+), 546 deletions(-) diff --git a/elasticsearch/qa/shield-reindex-tests/roles.yml b/elasticsearch/qa/shield-reindex-tests/roles.yml index cf188427cef..2aaec4b9b6a 100644 --- a/elasticsearch/qa/shield-reindex-tests/roles.yml +++ b/elasticsearch/qa/shield-reindex-tests/roles.yml @@ -1,51 +1,77 @@ admin: - cluster: all + cluster: + - all indices: - '*': - privileges: all - run_as: '*' + - names: '*' + privileges: [ all ] + run_as: + - '*' # Search and write on both source and destination indices. It should work if you could just search on the source and # write to the destination but that isn't how shield works. minimal: indices: - source: - privileges: search, write, create_index, indices:admin/refresh - dest: - privileges: search, write, create_index, indices:admin/refresh + - names: source + privileges: + - search + - write + - create_index + - indices:admin/refresh + - names: dest + privileges: + - search + - write + - create_index + - indices:admin/refresh # Read only operations on indices readonly: indices: - '*': - privileges: search + - names: '*' + privileges: [ search ] # Write operations on destination index, none on source index dest_only: indices: - dest: - privileges: write + - names: dest + privileges: [ write ] # Search and write on both source and destination indices with document level security filtering out some docs. can_not_see_hidden_docs: indices: - source: - privileges: search, write, create_index, indices:admin/refresh + - names: source + privileges: + - search + - write + - create_index + - indices:admin/refresh query: bool: must_not: match: hidden: true - dest: - privileges: search, write, create_index, indices:admin/refresh + - names: dest + privileges: + - search + - write + - create_index + - indices:admin/refresh # Search and write on both source and destination indices with field level security. can_not_see_hidden_fields: indices: - source: - privileges: search, write, create_index, indices:admin/refresh + - names: source + privileges: + - search + - write + - create_index + - indices:admin/refresh fields: - foo - bar - dest: - privileges: search, write, create_index, indices:admin/refresh + - names: dest + privileges: + - search + - write + - create_index + - indices:admin/refresh diff --git a/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml b/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml index e5559b90fe9..69b72aaddd5 100644 --- a/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml +++ b/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml @@ -1,17 +1,30 @@ admin: - cluster: all + cluster: + - all indices: - '*': all + - names: '*' + privileges: + - all graph_explorer: - cluster: cluster:monitor/health + cluster: + - cluster:monitor/health indices: - '*': - privileges: graph, indices:data/write/index, indices:admin/refresh, indices:admin/create + - names: '*' + privileges: + - graph + - indices:data/write/index + - indices:admin/refresh + - indices:admin/create no_graph_explorer: - cluster: cluster:monitor/health + cluster: + - cluster:monitor/health indices: - '*': - privileges: indices:data/read/search, indices:data/write/index, indices:admin/refresh, indices:admin/create + - names: '*' + privileges: + - indices:data/read/search + - indices:data/write/index + - indices:admin/refresh + - indices:admin/create diff --git a/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml b/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml index e5cbe14f6b5..ced242d9d3b 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml +++ b/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml @@ -1,18 +1,31 @@ admin: - cluster: all + cluster: + - all indices: - '*': all + - names: '*' + privileges: + - all watcher_manager: - cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/health + cluster: + - manage_watcher + - cluster:monitor/nodes/info + - cluster:monitor/health indices: - '.watcher-history-*': all - run_as: powerless_user, watcher_manager + - names: '.watcher-history-*' + privileges: + - all + run_as: + - powerless_user + - watcher_manager watcher_monitor: - cluster: monitor_watcher + cluster: + - monitor_watcher indices: - '.watcher-history-*': read + - names: '.watcher-history-*' + privileges: + - read crappy_role: cluster: diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index d062247ecfd..218535a8841 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -436,20 +436,21 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { public static final String ROLES = "test:\n" + // a user for the test infra. - " cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/stats, cluster:monitor/state, " + - "cluster:monitor/health, cluster:monitor/stats, cluster:monitor/task, cluster:admin/settings/update, " + - "cluster:admin/repository/delete, cluster:monitor/nodes/liveness, indices:admin/template/get, " + - "indices:admin/template/put, indices:admin/template/delete\n" + + " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/state', 'cluster:monitor/health', 'cluster:monitor/stats'," + + " 'cluster:admin/settings/update', 'cluster:admin/repository/delete', 'cluster:monitor/nodes/liveness'," + + " 'indices:admin/template/get', 'indices:admin/template/put', 'indices:admin/template/delete'," + + " 'cluster:monitor/task']\n" + " indices:\n" + - " '*': all\n" + + " - names: '*'\n" + + " privileges: [ all ]\n" + "\n" + "admin:\n" + - " cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" + + " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + "transport_client:\n" + - " cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" + + " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + "\n" + "monitor:\n" + - " cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" + " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" ; diff --git a/elasticsearch/x-pack/shield/config/xpack/roles.yml b/elasticsearch/x-pack/shield/config/xpack/roles.yml index 7bfb3b79920..b43fc06ea7b 100644 --- a/elasticsearch/x-pack/shield/config/xpack/roles.yml +++ b/elasticsearch/x-pack/shield/config/xpack/roles.yml @@ -1,22 +1,27 @@ admin: - cluster: all + cluster: + - all indices: - '*': - privileges: all + - names: '*' + privileges: + - all # monitoring cluster privileges # All operations on all indices power_user: - cluster: monitor + cluster: + - monitor indices: - '*': - privileges: all + - names: '*' + privileges: + - all # Read-only operations on indices user: indices: - '*': - privileges: read + - names: '*' + privileges: + - read # Defines the required permissions for transport clients transport_client: @@ -31,10 +36,27 @@ kibana4: - cluster:monitor/nodes/info - cluster:monitor/health indices: - '*': - privileges: indices:admin/mappings/fields/get, indices:admin/validate/query, indices:data/read/search, indices:data/read/msearch, indices:data/read/field_stats, indices:admin/get - '.kibana': - privileges: indices:admin/exists, indices:admin/mapping/put, indices:admin/mappings/fields/get, indices:admin/refresh, indices:admin/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search, indices:data/write/delete, indices:data/write/index, indices:data/write/update + - names: '*' + privileges: + - indices:admin/mappings/fields/get + - indices:admin/validate/query + - indices:data/read/search + - indices:data/read/msearch + - indices:data/read/field_stats + - indices:admin/get + - names: '.kibana' + privileges: + - indices:admin/exists + - indices:admin/mapping/put + - indices:admin/mappings/fields/get + - indices:admin/refresh + - indices:admin/validate/query + - indices:data/read/get + - indices:data/read/mget + - indices:data/read/search + - indices:data/write/delete + - indices:data/write/index + - indices:data/write/update # The required permissions for the kibana 4 server kibana4_server: @@ -42,32 +64,63 @@ kibana4_server: - cluster:monitor/nodes/info - cluster:monitor/health indices: - '.kibana': - privileges: indices:admin/create, indices:admin/exists, indices:admin/mapping/put, indices:admin/mappings/fields/get, indices:admin/refresh, indices:admin/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search, indices:data/write/delete, indices:data/write/index, indices:data/write/update + - names: '.kibana' + privileges: + - indices:admin/create + - indices:admin/exists + - indices:admin/mapping/put + - indices:admin/mappings/fields/get + - indices:admin/refresh + - indices:admin/validate/query + - indices:data/read/get + - indices:data/read/mget + - indices:data/read/search + - indices:data/write/delete + - indices:data/write/index + - indices:data/write/update # The required role for logstash users logstash: - cluster: indices:admin/template/get, indices:admin/template/put + cluster: + - indices:admin/template/get + - indices:admin/template/put indices: - 'logstash-*': - privileges: indices:data/write/bulk, indices:data/write/delete, indices:data/write/update, indices:data/read/search, indices:data/read/scroll, create_index + - names: 'logstash-*' + privileges: + - indices:data/write/bulk + - indices:data/write/delete + - indices:data/write/update + - indices:data/read/search + - indices:data/read/scroll + - create_index # Monitoring user role. Assign to monitoring users. monitoring_user: indices: - '.monitoring-*': - privileges: read - '.kibana': - privileges: indices:admin/exists, indices:admin/mappings/fields/get, indices:admin/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search + - names: '.monitoring-*' + privileges: + - read + - names: '.kibana' + privileges: + - indices:admin/exists + - indices:admin/mappings/fields/get + - indices:admin/validate/query + - indices:data/read/get + - indices:data/read/mget + - indices:data/read/search # Monitoring remote agent role. Assign to the agent user on the remote monitoring cluster # to which the monitoring agent will export all its data remote_monitoring_agent: - cluster: indices:admin/template/put, indices:admin/template/get + cluster: + - indices:admin/template/put + - indices:admin/template/get indices: - '.monitoring-*': - privileges: all + - names: '.monitoring-*' + privileges: + - all # Allows all operations required to manage ingest pipelines ingest_admin: - cluster: manage_pipeline \ No newline at end of file + cluster: + - manage_pipeline \ No newline at end of file diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/RoleDescriptor.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/RoleDescriptor.java index 3e90706cf4f..bb07adac08a 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/RoleDescriptor.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/RoleDescriptor.java @@ -10,15 +10,19 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.shield.support.Validation; import org.elasticsearch.xpack.common.xcontent.XContentUtils; import java.io.IOException; @@ -132,48 +136,69 @@ public class RoleDescriptor implements ToXContent { out.writeStringArray(descriptor.runAs); } - public static RoleDescriptor parse(String name, BytesReference source) throws Exception { + public static RoleDescriptor parse(String name, BytesReference source) throws IOException { assert name != null; try (XContentParser parser = XContentHelper.createParser(source)) { - XContentParser.Token token = parser.nextToken(); // advancing to the START_OBJECT token - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("failed to parse role [{}]. expected an object but found [{}] instead", name, token); - } - String currentFieldName = null; - IndicesPrivileges[] indicesPrivileges = null; - String[] clusterPrivileges = null; - String[] runAsUsers = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.INDICES)) { - indicesPrivileges = parseIndices(name, parser); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.RUN_AS)) { - runAsUsers = XContentUtils.readStringArray(parser, true); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.CLUSTER)) { - clusterPrivileges = XContentUtils.readStringArray(parser, true); - } else { - throw new ElasticsearchParseException("failed to parse role [{}]. unexpected field [{}]", name, currentFieldName); - } - } - return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, runAsUsers); + return parse(name, parser); } } - private static RoleDescriptor.IndicesPrivileges[] parseIndices(String roleName, XContentParser parser) throws Exception { + public static RoleDescriptor parse(String name, XContentParser parser) throws IOException { + // validate name + Validation.Error validationError = Validation.Roles.validateRoleName(name); + if (validationError != null) { + ValidationException ve = new ValidationException(); + ve.addValidationError(validationError.toString()); + throw ve; + } + + // advance to the START_OBJECT token if needed + XContentParser.Token token = parser.currentToken() == null ? parser.nextToken() : parser.currentToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("failed to parse role [{}]. expected an object but found [{}] instead", name, token); + } + String currentFieldName = null; + IndicesPrivileges[] indicesPrivileges = null; + String[] clusterPrivileges = null; + String[] runAsUsers = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.INDICES)) { + indicesPrivileges = parseIndices(name, parser); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.RUN_AS)) { + runAsUsers = readStringArray(name, parser, true); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.CLUSTER)) { + clusterPrivileges = readStringArray(name, parser, true); + } else { + throw new ElasticsearchParseException("failed to parse role [{}]. unexpected field [{}]", name, currentFieldName); + } + } + return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, runAsUsers); + } + + private static String[] readStringArray(String roleName, XContentParser parser, boolean allowNull) throws IOException { + try { + return XContentUtils.readStringArray(parser, allowNull); + } catch (ElasticsearchParseException e) { + // re-wrap in order to add the role name + throw new ElasticsearchParseException("failed to parse role [{}]", e, roleName); + } + } + + private static RoleDescriptor.IndicesPrivileges[] parseIndices(String roleName, XContentParser parser) throws IOException { if (parser.currentToken() != XContentParser.Token.START_ARRAY) { throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. expected field [{}] value " + "to be an array, but found [{}] instead", roleName, parser.currentName(), parser.currentToken()); } List privileges = new ArrayList<>(); - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { privileges.add(parseIndex(roleName, parser)); } return privileges.toArray(new IndicesPrivileges[privileges.size()]); } - private static RoleDescriptor.IndicesPrivileges parseIndex(String roleName, XContentParser parser) throws Exception { + private static RoleDescriptor.IndicesPrivileges parseIndex(String roleName, XContentParser parser) throws IOException { XContentParser.Token token = parser.currentToken(); if (token != XContentParser.Token.START_OBJECT) { throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. expected field [{}] value to " + @@ -191,7 +216,7 @@ public class RoleDescriptor implements ToXContent { if (token == XContentParser.Token.VALUE_STRING) { names = new String[] { parser.text() }; } else if (token == XContentParser.Token.START_ARRAY) { - names = XContentUtils.readStringArray(parser, false); + names = readStringArray(roleName, parser, false); if (names.length == 0) { throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. [{}] cannot be an empty " + "array", roleName, currentFieldName); @@ -201,15 +226,21 @@ public class RoleDescriptor implements ToXContent { "value to be a string or an array of strings, but found [{}] instead", roleName, currentFieldName, token); } } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.QUERY)) { - query = parser.textOrNull(); + if (token == XContentParser.Token.START_OBJECT) { + XContentBuilder builder = JsonXContent.contentBuilder(); + XContentHelper.copyCurrentStructure(builder.generator(), parser); + query = builder.string(); + } else { + query = parser.textOrNull(); + } } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.PRIVILEGES)) { - privileges = XContentUtils.readStringArray(parser, false); + privileges = readStringArray(roleName, parser, true); if (names.length == 0) { throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. [{}] cannot be an empty " + "array", roleName, currentFieldName); } } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.FIELDS)) { - fields = XContentUtils.readStringArray(parser, true); + fields = readStringArray(roleName, parser, true); } else { throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. unexpected field [{}]", roleName, currentFieldName); diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/Role.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/Role.java index 5858a9d9284..5496c04e826 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/Role.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/Role.java @@ -68,7 +68,11 @@ public class Role extends GlobalPermission { private Builder(RoleDescriptor rd) { this.name = rd.getName(); - this.cluster(ClusterPrivilege.get((new Privilege.Name(rd.getClusterPrivileges())))); + if (rd.getClusterPrivileges().length == 0) { + cluster = ClusterPermission.Core.NONE; + } else { + this.cluster(ClusterPrivilege.get((new Privilege.Name(rd.getClusterPrivileges())))); + } for (RoleDescriptor.IndicesPrivileges iGroup : rd.getIndicesPrivileges()) { this.add(iGroup.getFields() == null ? null : Arrays.asList(iGroup.getFields()), iGroup.getQuery(), diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java index a1bcbe73622..db5a00cd00d 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java @@ -7,28 +7,20 @@ package org.elasticsearch.shield.authz.store; import com.fasterxml.jackson.dataformat.yaml.snakeyaml.error.YAMLException; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.env.Environment; import org.elasticsearch.shield.Shield; import org.elasticsearch.shield.SystemUser; import org.elasticsearch.shield.XPackUser; import org.elasticsearch.shield.authc.support.RefreshListener; +import org.elasticsearch.shield.authz.RoleDescriptor; import org.elasticsearch.shield.authz.permission.Role; -import org.elasticsearch.shield.authz.privilege.ClusterPrivilege; -import org.elasticsearch.shield.authz.privilege.GeneralPrivilege; -import org.elasticsearch.shield.authz.privilege.IndexPrivilege; -import org.elasticsearch.shield.authz.privilege.Privilege; import org.elasticsearch.shield.support.NoOpLogger; import org.elasticsearch.shield.support.Validation; import org.elasticsearch.watcher.FileChangesListener; @@ -41,10 +33,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -59,7 +48,6 @@ import static java.util.Collections.unmodifiableMap; */ public class FileRolesStore extends AbstractLifecycleComponent implements RolesStore { - private static final Pattern COMMA_DELIM = Pattern.compile("\\s*,\\s*"); private static final Pattern IN_SEGMENT_LINE = Pattern.compile("^\\s+.+"); private static final Pattern SKIP_LINE = Pattern.compile("(^#.*|^\\s*)"); @@ -174,226 +162,40 @@ public class FileRolesStore extends AbstractLifecycleComponent imple return null; } - Role.Builder role = Role.builder(roleName); if (resolvePermissions == false) { - return role.build(); + return Role.builder(roleName).build(); } token = parser.nextToken(); if (token == XContentParser.Token.START_OBJECT) { - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if ("cluster".equals(currentFieldName)) { - Privilege.Name name = null; - if (token == XContentParser.Token.VALUE_STRING) { - String namesStr = parser.text().trim(); - if (Strings.hasLength(namesStr)) { - String[] names = COMMA_DELIM.split(namesStr); - name = new Privilege.Name(names); - } - } else if (token == XContentParser.Token.START_ARRAY) { - Set names = new HashSet<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - names.add(parser.text()); - } - } - if (!names.isEmpty()) { - name = new Privilege.Name(names); - } - } else if (token == XContentParser.Token.VALUE_NULL) { - continue; - } else { - logger.error("invalid role definition [{}] in roles file [{}]. [cluster] field value can either " + - "be a string or a list of strings, but [{}] was found instead. skipping role...", - roleName, path.toAbsolutePath(), token); - return null; - } - if (name != null) { - try { - role.cluster(ClusterPrivilege.get(name)); - } catch (IllegalArgumentException e) { - logger.error("invalid role definition [{}] in roles file [{}]. could not resolve cluster " + - "privileges [{}]. skipping role...", roleName, path.toAbsolutePath(), name); - return null; - } - } - } else if ("indices".equals(currentFieldName)) { - if (token == XContentParser.Token.START_OBJECT) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (Strings.hasLength(currentFieldName)) { - String[] indices = COMMA_DELIM.split(currentFieldName); - Privilege.Name name = null; - if (token == XContentParser.Token.VALUE_STRING) { - String namesStr = parser.text().trim(); - if (Strings.hasLength(namesStr)) { - String[] names = COMMA_DELIM.split(parser.text()); - name = new Privilege.Name(names); - } - } else if (token == XContentParser.Token.START_ARRAY) { - Set names = new HashSet<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - names.add(parser.text()); - } else { - logger.error("invalid role definition [{}] in roles file [{}]. could not parse " + - "[{}] as index privilege. privilege names must be strings. skipping " + - "role...", roleName, path.toAbsolutePath(), token); - return null; - } - } - if (!names.isEmpty()) { - name = new Privilege.Name(names); - } - } else if (token == XContentParser.Token.START_OBJECT) { - List fields = null; - BytesReference query = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if ("fields".equals(currentFieldName)) { - if (token == XContentParser.Token.START_ARRAY) { - fields = (List) parser.list(); - } else if (token.isValue()) { - String field = parser.text(); - if (field.trim().isEmpty()) { - // The yaml parser doesn't emit null token if the key is empty... - fields = Collections.emptyList(); - } else { - fields = Collections.singletonList(field); - } - } else if (token == XContentParser.Token.VALUE_NULL) { - fields = Collections.emptyList(); - } - } else if ("query".equals(currentFieldName)) { - if (token == XContentParser.Token.START_OBJECT) { - XContentBuilder builder = JsonXContent.contentBuilder(); - XContentHelper.copyCurrentStructure(builder.generator(), parser); - query = builder.bytes(); - } else if (token == XContentParser.Token.VALUE_STRING) { - query = new BytesArray(parser.text()); - } - } else if ("privileges".equals(currentFieldName)) { - if (token == XContentParser.Token.VALUE_STRING) { - String namesStr = parser.text().trim(); - if (Strings.hasLength(namesStr)) { - String[] names = COMMA_DELIM.split(parser.text()); - name = new Privilege.Name(names); - } - } else if (token == XContentParser.Token.START_ARRAY) { - Set names = new HashSet<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - names.add(parser.text()); - } else { - logger.error("invalid role definition [{}] in roles file [{}]. " + - "could not parse [{}] as index privilege. privilege " + - "names must be strings. skipping role...", roleName, - path.toAbsolutePath(), token); - return null; - } - } - if (!names.isEmpty()) { - name = new Privilege.Name(names); - } - } - } - } - if (name != null) { - if ((query != null || (fields != null && fields.isEmpty() == false)) && - Shield.flsDlsEnabled(settings) == false) { - logger.error("invalid role definition [{}] in roles file [{}]. " + - "document and field level security is not enabled. " + - "set [{}] to [true] in the configuration file. skipping role...", - roleName, path.toAbsolutePath(), - XPackPlugin.featureEnabledSetting(Shield.DLS_FLS_FEATURE)); - return null; - } + RoleDescriptor descriptor = RoleDescriptor.parse(roleName, parser); - try { - role.add(fields, query, IndexPrivilege.get(name), indices); - } catch (IllegalArgumentException e) { - logger.error("invalid role definition [{}] in roles file [{}]. could not " + - "resolve indices privileges [{}]. skipping role...", roleName, - path.toAbsolutePath(), name); - return null; - } - } - continue; - } else if (token == XContentParser.Token.VALUE_NULL) { - continue; - } else { - logger.error("invalid role definition [{}] in roles file [{}]. " + - "could not parse [{}] as index privileges. privilege lists must either " + - "be a comma delimited string or an array of strings. skipping role...", roleName, - path.toAbsolutePath(), token); - return null; - } - if (name != null) { - try { - role.add(IndexPrivilege.get(name), indices); - } catch (IllegalArgumentException e) { - logger.error("invalid role definition [{}] in roles file [{}]. could not resolve " + - "indices privileges [{}]. skipping role...", roleName, path.toAbsolutePath(), - name); - return null; - } - } - } - } - } else { - logger.error("invalid role definition [{}] in roles file [{}]. [indices] field value must be an array" + - " of indices-privileges mappings defined as a string" + - " in the form :: , but [{}] was found instead. skipping role...", - roleName, path.toAbsolutePath(), token); - return null; - } - } else if ("run_as".equals(currentFieldName)) { - Set names = new HashSet<>(); - if (token == XContentParser.Token.VALUE_STRING) { - String namesStr = parser.text().trim(); - if (Strings.hasLength(namesStr)) { - String[] namesArr = COMMA_DELIM.split(namesStr); - names.addAll(Arrays.asList(namesArr)); - } - } else if (token == XContentParser.Token.START_ARRAY) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - names.add(parser.text()); - } - } - } else { - logger.error("invalid role definition [{}] in roles file [{}]. [run_as] field value can either " + - "be a string or a list of strings, but [{}] was found instead. skipping role...", - roleName, path.toAbsolutePath(), token); - return null; - } - if (!names.isEmpty()) { - Privilege.Name name = new Privilege.Name(names); - try { - role.runAs(new GeneralPrivilege(new Privilege.Name(names), - names.toArray(new String[names.size()]))); - } catch (IllegalArgumentException e) { - logger.error("invalid role definition [{}] in roles file [{}]. could not resolve run_as " + - "privileges [{}]. skipping role...", roleName, path.toAbsolutePath(), name); - return null; - } - } - } else { - logger.warn("unknown field [{}] found in role definition [{}] in roles file [{}]", currentFieldName, - roleName, path.toAbsolutePath()); + // first check if FLS/DLS is enabled on the role... + for (RoleDescriptor.IndicesPrivileges privilege : descriptor.getIndicesPrivileges()) { + if ((privilege.getQuery() != null || privilege.getFields() != null) + && Shield.flsDlsEnabled(settings) == false) { + logger.error("invalid role definition [{}] in roles file [{}]. document and field level security is not " + + "enabled. set [{}] to [true] in the configuration file. skipping role...", roleName, path + .toAbsolutePath(), XPackPlugin.featureEnabledSetting(Shield.DLS_FLS_FEATURE)); + return null; } } - return role.build(); + + return Role.builder(descriptor).build(); + } else { + logger.error("invalid role definition [{}] in roles file [{}]. skipping role...", roleName, path.toAbsolutePath()); + return null; } - logger.error("invalid role definition [{}] in roles file [{}]. skipping role...", roleName, path.toAbsolutePath()); } } + logger.error("invalid role definition [{}] in roles file [{}]. skipping role...", roleName, path.toAbsolutePath()); + } catch (ElasticsearchParseException e) { + assert roleName != null; + if (logger.isDebugEnabled()) { + logger.debug("parsing exception for role [{}]", e, roleName); + } else { + logger.error(e.getMessage() + ". skipping role..."); + } } catch (YAMLException | IOException e) { if (roleName != null) { logger.error("invalid role definition [{}] in roles file [{}]. skipping role...", e, roleName, path); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java index e30ca6216b8..702fd972447 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java @@ -28,14 +28,15 @@ public class ClusterPrivilegeTests extends AbstractPrivilegeTestCase { public static final String ROLES = "role_a:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + "\n" + "role_b:\n" + - " cluster: monitor\n" + + " cluster: [ monitor ]\n" + "\n" + "role_c:\n" + " indices:\n" + - " 'someindex': all\n"; + " - names: 'someindex'\n" + + " privileges: [ all ]\n"; public static final String USERS = "user_a:" + USERS_PASSWD_HASHED + "\n" + diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 89c6ad8fd49..9bce8e09959 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -52,25 +52,25 @@ public class DocumentAndFieldLevelSecurityTests extends ShieldIntegTestCase { protected String configRoles() { return super.configRoles() + "\nrole1:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: field1\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: [ field1 ]\n" + " query: '{\"term\" : {\"field1\" : \"value1\"}}'\n" + "role2:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: field2\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: [ field2 ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n" + "role3:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: field1\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: [ field1 ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n"; } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java index 5616cd40ba0..3a82a5f8706 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java @@ -60,10 +60,10 @@ public class DocumentLevelSecurityRandomTests extends ShieldIntegTestCase { builder.append('\n'); for (int i = 1; i <= numberOfRoles; i++) { builder.append("role").append(i).append(":\n"); - builder.append(" cluster: all\n"); + builder.append(" cluster: [ all ]\n"); builder.append(" indices:\n"); - builder.append(" '*':\n"); - builder.append(" privileges: ALL\n"); + builder.append(" - names: '*'\n"); + builder.append(" privileges: [ ALL ]\n"); builder.append(" query: \n"); builder.append(" term: \n"); builder.append(" field1: value").append(i).append('\n'); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index fd9c98fee29..0ab2f3e34ae 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -73,18 +73,18 @@ public class DocumentLevelSecurityTests extends ShieldIntegTestCase { protected String configRoles() { return super.configRoles() + "\nrole1:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + " query: \n" + " term: \n" + " field1: value1\n" + "role2:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'"; // <-- query defined as json in a string } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java index adad07b7771..46cb49b1db9 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java @@ -80,30 +80,30 @@ public class FieldLevelSecurityRandomTests extends ShieldIntegTestCase { return super.configRoles() + "\nrole1:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields:\n" + roleFields.toString() + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields:\n" +roleFields.toString() + "role2:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " test:\n" + - " privileges: ALL\n" + + " - names: test\n" + + " privileges: [ ALL ]\n" + " fields:\n" + " - field1\n" + "role3:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " test:\n" + - " privileges: ALL\n" + + " - names: test\n" + + " privileges: [ ALL ]\n" + " fields:\n" + " - field2\n" + "role4:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " test:\n" + - " privileges: ALL\n" + + " - names: test\n" + + " privileges: [ ALL ]\n" + " fields:\n" + " - field3\n"; } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 56c6b0b67b6..d1eb9c7e08e 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -80,41 +80,42 @@ public class FieldLevelSecurityTests extends ShieldIntegTestCase { protected String configRoles() { return super.configRoles() + "\nrole1:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: field1\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: [ field1 ]\n" + "role2:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: field2\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: [ field2 ]\n" + "role3:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: \n" + - " - field1\n" + - " - field2\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields:\n" + + " - field1\n" + + " - field2\n" + "role4:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields:\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: []\n" + "role5:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*': ALL\n" + + " - names: '*'\n" + + " privileges: [ALL]\n" + "role6:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*':\n" + - " privileges: ALL\n" + - " fields: 'field*'\n"; + " - names: '*'\n" + + " privileges: [ ALL ]\n" + + " fields: [ 'field*' ]\n"; } @Override diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java index 168328f4d19..dde8de82277 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java @@ -29,56 +29,71 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { public static final String ROLES = "all_cluster_role:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + "all_indices_role:\n" + " indices:\n" + - " '*': all\n" + + " - names: '*'\n" + + " privileges: [ all ]\n" + "all_a_role:\n" + " indices:\n" + - " 'a': all\n" + + " - names: 'a'\n" + + " privileges: [ all ]\n" + "read_a_role:\n" + " indices:\n" + - " 'a': read\n" + + " - names: 'a'\n" + + " privileges: [ read ]\n" + "write_a_role:\n" + " indices:\n" + - " 'a': write\n" + + " - names: 'a'\n" + + " privileges: [ write ]\n" + "read_ab_role:\n" + " indices:\n" + - " 'a': read\n" + - " 'b': read\n" + + " - names: [ 'a', 'b' ]\n" + + " privileges: [ read ]\n" + "get_b_role:\n" + " indices:\n" + - " 'b': get\n" + + " - names: 'b'\n" + + " privileges: [ get ]\n" + "search_b_role:\n" + " indices:\n" + - " 'b': search\n" + + " - names: 'b'\n" + + " privileges: [ search ]\n" + "all_regex_ab_role:\n" + " indices:\n" + - " '/a|b/': all\n" + + " - names: '/a|b/'\n" + + " privileges: [ all ]\n" + "manage_starts_with_a_role:\n" + " indices:\n" + - " 'a*': manage\n" + + " - names: 'a*'\n" + + " privileges: [ manage ]\n" + "data_access_all_role:\n" + " indices:\n" + - " '*': data_access\n" + + " - names: '*'\n" + + " privileges: [ data_access ]\n" + "create_c_role:\n" + " indices:\n" + - " 'c': create_index\n" + + " - names: 'c'\n" + + " privileges: [ create_index ]\n" + "monitor_b_role:\n" + " indices:\n" + - " 'b': monitor\n" + + " - names: 'b'\n" + + " privileges: [ monitor ]\n" + "crud_a_role:\n" + " indices:\n" + - " 'a': crud\n" + + " - names: 'a'\n" + + " privileges: [ crud ]\n" + "delete_b_role:\n" + " indices:\n" + - " 'b': delete\n" + + " - names: 'b'\n" + + " privileges: [ delete ]\n" + "index_a_role:\n" + " indices:\n" + - " 'a': index\n" + + " - names: 'a'\n" + + " privileges: [ index ]\n" + "search_a_role:\n" + " indices:\n" + - " 'a': search\n" + + " - names: 'a'\n" + + " privileges: [ search ]\n" + "\n"; public static final String USERS = diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java index 7472f30a52a..4134c15bb90 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java @@ -44,17 +44,17 @@ public class IndicesPermissionsWithAliasesWildcardsAndRegexsTests extends Shield protected String configRoles() { return super.configRoles() + "\nrole1:\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " 't*':\n" + - " privileges: ALL\n" + - " fields: field1\n" + - " 'my_alias':\n" + - " privileges: ALL\n" + - " fields: field2\n" + - " '/an_.*/':\n" + - " privileges: ALL\n" + - " fields: field3\n"; + " - names: 't*'\n" + + " privileges: [ALL]\n" + + " fields: [ field1 ]\n" + + " - names: 'my_alias'\n" + + " privileges: [ALL]\n" + + " fields: [field2]\n" + + " - names: '/an_.*/'\n" + + " privileges: [ALL]\n" + + " fields: [field3]\n"; } @Override diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java index 73c5ad7b4ed..84c8784fc01 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java @@ -56,20 +56,26 @@ import static org.hamcrest.Matchers.notNullValue; public class LicensingTests extends ShieldIntegTestCase { public static final String ROLES = ShieldSettingsSource.DEFAULT_ROLE + ":\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*': manage\n" + - " '/.*/': write\n" + - " 'test': read\n" + - " 'test1': read\n" + + " - names: '*'\n" + + " privileges: [manage]\n" + + " - names: '/.*/'\n" + + " privileges: [write]\n" + + " - names: 'test'\n" + + " privileges: [read]\n" + + " - names: 'test1'\n" + + " privileges: [read]\n" + "\n" + "role_a:\n" + " indices:\n" + - " 'a': all\n" + + " - names: 'a'\n" + + " privileges: [all]\n" + "\n" + "role_b:\n" + " indices:\n" + - " 'b': all\n"; + " - names: 'b'\n" + + " privileges: [all]\n"; public static final String USERS = ShieldSettingsSource.CONFIG_STANDARD_USER + diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java index 34f1d58318f..d017c47befd 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java @@ -34,20 +34,26 @@ public class MultipleIndicesPermissionsTests extends ShieldIntegTestCase { @Override protected String configRoles() { return ShieldSettingsSource.DEFAULT_ROLE + ":\n" + - " cluster: all\n" + + " cluster: [ all ]\n" + " indices:\n" + - " '*': manage\n" + - " '/.*/': write\n" + - " 'test': read\n" + - " 'test1': read\n" + + " - names: '*'\n" + + " privileges: [manage]\n" + + " - names: '/.*/'\n" + + " privileges: [write]\n" + + " - names: 'test'\n" + + " privileges: [read]\n" + + " - names: 'test1'\n" + + " privileges: [read]\n" + "\n" + "role_a:\n" + " indices:\n" + - " 'a': all\n" + + " - names: 'a'\n" + + " privileges: [all]\n" + "\n" + "role_b:\n" + " indices:\n" + - " 'b': all\n"; + " - names: 'b'\n" + + " privileges: [all]\n"; } @Override diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java index b5636f511c1..f6be64f2316 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java @@ -38,9 +38,10 @@ public class PermissionPrecedenceTests extends ShieldIntegTestCase { @Override protected String configRoles() { return "admin:\n" + - " cluster: all\n" + + " cluster: [ all ] \n" + " indices:\n" + - " '*': all\n" + + " - names: '*'\n" + + " privileges: [ all ]" + "\n" + "transport_client:\n" + " cluster:\n" + @@ -49,7 +50,8 @@ public class PermissionPrecedenceTests extends ShieldIntegTestCase { "\n" + "user:\n" + " indices:\n" + - " 'test_*': all\n"; + " - names: 'test_*'\n" + + " privileges: [ all ]"; } @Override diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java index 796aab3c3cb..c4208907190 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java @@ -38,15 +38,18 @@ public class SearchGetAndSuggestPermissionsTests extends ShieldIntegTestCase { "\n" + "search_role:\n" + " indices:\n" + - " 'a': search\n" + + " - names: 'a'\n" + + " privileges: [ search ]\n" + "\n" + "get_role:\n" + " indices:\n" + - " 'a': get\n" + + " - names: 'a'\n" + + " privileges: [ get ]\n" + "\n" + "suggest_role:\n" + " indices:\n" + - " 'a': suggest\n"; + " - names: 'a'\n" + + " privileges: [ suggest ]\n"; } @Override diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java index cbd60a46d31..4b8d243fef0 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java @@ -58,7 +58,8 @@ public class ShieldClearScrollTests extends ShieldIntegTestCase { " - cluster:admin/indices/scroll/clear_all \n" + "denied_role:\n" + " indices:\n" + - " '*': ALL\n"; + " - names: '*'" + + " privileges: [ALL]\n"; } @Before diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java index ee339d2ab49..4b6cd586e2a 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java @@ -93,21 +93,25 @@ abstract public class AbstractAdLdapRealmTestCase extends ShieldIntegTestCase { return super.configRoles() + "\n" + "Avengers:\n" + - " cluster: NONE\n" + + " cluster: [ NONE ]\n" + " indices:\n" + - " 'avengers': ALL\n" + + " - names: 'avengers'\n" + + " privileges: [ all ]\n" + "SHIELD:\n" + - " cluster: NONE\n" + - " indices:\n " + - " '" + SHIELD_INDEX + "': ALL\n" + + " cluster: [ NONE ]\n" + + " indices:\n" + + " - names: '" + SHIELD_INDEX + "'\n" + + " privileges: [ all ]\n" + "Gods:\n" + - " cluster: NONE\n" + + " cluster: [ NONE ]\n" + " indices:\n" + - " '" + ASGARDIAN_INDEX + "': ALL\n" + + " - names: '" + ASGARDIAN_INDEX + "'\n" + + " privileges: [ all ]\n" + "Philanthropists:\n" + - " cluster: NONE\n" + + " cluster: [ NONE ]\n" + " indices:\n" + - " '" + PHILANTHROPISTS_INDEX + "': ALL\n"; + " - names: '" + PHILANTHROPISTS_INDEX + "'\n" + + " privileges: [ all ]\n"; } protected void assertAccessAllowed(String user, String index) throws IOException { diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java index fc879b46db8..ddaeb8910aa 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java @@ -13,7 +13,7 @@ import java.io.IOException; * This tests the group to role mappings from LDAP sources provided by the super class - available from super.realmConfig. * The super class will provide appropriate group mappings via configGroupMappings() */ -@Network +//@Network public class GroupMappingTests extends AbstractAdLdapRealmTestCase { public void testAuthcAuthz() throws IOException { String avenger = realmConfig.loginWithCommonName ? "Natasha Romanoff" : "blackwidow"; diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java index d211e6aa3ab..772d965e566 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java @@ -19,9 +19,10 @@ public class MultiGroupMappingTests extends AbstractAdLdapRealmTestCase { return super.configRoles() + "\n" + "MarvelCharacters:\n" + - " cluster: NONE\n" + + " cluster: [ NONE ]\n" + " indices:\n" + - " 'marvel_comics': ALL\n"; + " - names: 'marvel_comics'\n" + + " privileges: [ all ]\n"; } @Override diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java index daa31f54978..fe4c9023427 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java @@ -50,7 +50,8 @@ public class AnonymousUserTests extends ShieldIntegTestCase { return super.configRoles() + "\n" + "anonymous:\n" + " indices:\n" + - " '*': READ"; + " - names: '*'" + + " privileges: [ READ ]\n"; } public void testAnonymousViaHttp() throws Exception { diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java index 9ce717cfe47..f2e899521e6 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/RunAsIntegTests.java @@ -36,9 +36,9 @@ public class RunAsIntegTests extends ShieldIntegTestCase { static final String TRANSPORT_CLIENT_USER = "transport_user"; static final String ROLES = "transport_client:\n" + - " cluster: cluster:monitor/nodes/liveness\n" + + " cluster: [ 'cluster:monitor/nodes/liveness' ]\n" + "run_as_role:\n" + - " run_as: " + ShieldSettingsSource.DEFAULT_USER_NAME + ",idontexist\n"; + " run_as: [ '" + ShieldSettingsSource.DEFAULT_USER_NAME + "', 'idontexist' ]\n"; @Override public Settings nodeSettings(int nodeOrdinal) { diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java index a2604497fd8..f9e46480c22 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java @@ -40,7 +40,8 @@ public class AnalyzeTests extends ShieldIntegTestCase { //role that has analyze indices privileges only "analyze_indices:\n" + " indices:\n" + - " 'test_*': indices:admin/analyze\n" + + " - names: 'test_*'\n" + + " privileges: [ 'indices:admin/analyze' ]\n" + "analyze_cluster:\n" + " cluster:\n" + " - cluster:admin/analyze\n"; diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java index 7ea3866f256..43374091036 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java @@ -58,25 +58,32 @@ public class IndexAliasesTests extends ShieldIntegTestCase { //role that has create index only privileges "create_only:\n" + " indices:\n" + - " '*': create_index\n" + + " - names: '*'\n" + + " privileges: [ create_index ]\n" + //role that has create index and managa aliases on test_*, not enough to manage aliases outside of test_* namespace "create_test_aliases_test:\n" + " indices:\n" + - " 'test_*': create_index,manage_aliases\n" + + " - names: 'test_*'\n" + + " privileges: [ create_index, manage_aliases ]\n" + //role that has create index on test_* and manage aliases on alias_*, can't create aliases pointing to test_* though "create_test_aliases_alias:\n" + " indices:\n" + - " 'test_*': create_index\n" + - " 'alias_*': manage_aliases\n" + + " - names: 'test_*'\n" + + " privileges: [ create_index ]\n" + + " - names: 'alias_*'\n" + + " privileges: [ manage_aliases ]\n" + //role that has create index on test_* and manage_aliases on both alias_* and test_* "create_test_aliases_test_alias:\n" + " indices:\n" + - " 'test_*': create_index\n" + - " 'alias_*,test_*': manage_aliases\n" + + " - names: 'test_*'\n" + + " privileges: [ create_index ]\n" + + " - names: [ 'alias_*', 'test_*' ]\n" + + " privileges: [ manage_aliases ]\n" + //role that has manage_aliases only on both test_* and alias_* "aliases_only:\n" + " indices:\n" + - " 'alias_*,test_*': manage_aliases\n"; + " - names: [ 'alias_*', 'test_*']\n" + + " privileges: [ manage_aliases ]\n"; } @Before diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java index 69b01b9621c..c2de6613531 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java @@ -30,10 +30,12 @@ public class IndicesAndAliasesResolverIntegrationTests extends ShieldIntegTestCa @Override protected String configRoles() { return ShieldSettingsSource.DEFAULT_ROLE + ":\n" + - " cluster: ALL\n" + + " cluster: [ ALL ]\n" + " indices:\n" + - " '*': manage,write\n" + - " '/test.*/': read\n"; + " - names: '*'\n" + + " privileges: [ manage, write ]\n" + + " - names: '/test.*/'\n" + + " privileges: [ read ]\n"; } public void testSearchForAll() { diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java index 31f4727b601..9aac116b137 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java @@ -57,7 +57,7 @@ public class FileRolesStoreTests extends ESTestCase { .put(XPackPlugin.featureEnabledSetting(Shield.DLS_FLS_FEATURE), true) .build()); assertThat(roles, notNullValue()); - assertThat(roles.size(), is(10)); + assertThat(roles.size(), is(9)); Role role = roles.get("role1"); assertThat(role, notNullValue()); @@ -121,12 +121,7 @@ public class FileRolesStoreTests extends ESTestCase { assertThat(group.privilege().isAlias(IndexPrivilege.union(IndexPrivilege.READ, IndexPrivilege.WRITE)), is(true)); role = roles.get("role4"); - assertThat(role, notNullValue()); - assertThat(role.name(), equalTo("role4")); - assertThat(role.cluster(), notNullValue()); - assertThat(role.cluster(), is(ClusterPermission.Core.NONE)); - assertThat(role.indices(), is(IndicesPermission.Core.NONE)); - assertThat(role.runAs(), is(RunAsPermission.Core.NONE)); + assertThat(role, nullValue()); role = roles.get("role_run_as"); assertThat(role, notNullValue()); @@ -214,7 +209,7 @@ public class FileRolesStoreTests extends ESTestCase { .put(XPackPlugin.featureEnabledSetting(Shield.DLS_FLS_FEATURE), false) .build()); assertThat(roles, notNullValue()); - assertThat(roles.size(), is(7)); + assertThat(roles.size(), is(6)); assertThat(roles.get("role_fields"), nullValue()); assertThat(roles.get("role_query"), nullValue()); assertThat(roles.get("role_query_fields"), nullValue()); @@ -236,16 +231,18 @@ public class FileRolesStoreTests extends ESTestCase { Path path = getDataPath("default_roles.yml"); Map roles = FileRolesStore.parseFile(path, logger, Settings.EMPTY); assertThat(roles, notNullValue()); - assertThat(roles.size(), is(8)); + assertThat(roles.size(), is(10)); assertThat(roles, hasKey("admin")); assertThat(roles, hasKey("power_user")); assertThat(roles, hasKey("user")); - assertThat(roles, hasKey("kibana3")); assertThat(roles, hasKey("kibana4")); + assertThat(roles, hasKey("kibana4_server")); assertThat(roles, hasKey("logstash")); assertThat(roles, hasKey("monitoring_user")); - assertThat(roles, hasKey("monitoring_agent")); + assertThat(roles, hasKey("remote_monitoring_agent")); + assertThat(roles, hasKey("ingest_admin")); + assertThat(roles, hasKey("transport_client")); } public void testAutoReload() throws Exception { @@ -288,7 +285,8 @@ public class FileRolesStoreTests extends ESTestCase { writer.newLine(); writer.newLine(); writer.append("role5:").append(System.lineSeparator()); - writer.append(" cluster: 'MONITOR'"); + writer.append(" cluster:").append(System.lineSeparator()); + writer.append(" - 'MONITOR'"); } if (!latch.await(5, TimeUnit.SECONDS)) { @@ -327,24 +325,22 @@ public class FileRolesStoreTests extends ESTestCase { assertThat(role.name(), equalTo("valid_role")); List entries = logger.output(CapturingLogger.Level.ERROR); - assertThat(entries, hasSize(5)); + assertThat(entries, hasSize(6)); assertThat(entries.get(0).text, startsWith("invalid role definition [$dlk39] in roles file [" + path.toAbsolutePath() + "]. invalid role name")); assertThat(entries.get(1).text, startsWith("invalid role definition [role1] in roles file [" + path.toAbsolutePath() + "]")); - assertThat(entries.get(2).text, startsWith("invalid role definition [role2] in roles file [" + path.toAbsolutePath() + - "]. could not resolve cluster privileges [blkjdlkd]")); - assertThat(entries.get(3).text, startsWith("invalid role definition [role3] in roles file [" + path.toAbsolutePath() + - "]. [indices] field value must be an array")); - assertThat(entries.get(4).text, startsWith("invalid role definition [role4] in roles file [" + path.toAbsolutePath() + - "]. could not resolve indices privileges [al;kjdlkj;lkj]")); + assertThat(entries.get(2).text, startsWith("failed to parse role [role2]")); + assertThat(entries.get(3).text, startsWith("failed to parse role [role3]")); + assertThat(entries.get(4).text, startsWith("failed to parse role [role4]")); + assertThat(entries.get(5).text, startsWith("failed to parse indices privileges for role [role5]")); } public void testThatRoleNamesDoesNotResolvePermissions() throws Exception { Path path = getDataPath("invalid_roles.yml"); CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.ERROR); Set roleNames = FileRolesStore.parseFileForRoleNames(path, logger); - assertThat(roleNames.size(), is(5)); - assertThat(roleNames, containsInAnyOrder("valid_role", "role1", "role2", "role3", "role4")); + assertThat(roleNames.size(), is(6)); + assertThat(roleNames, containsInAnyOrder("valid_role", "role1", "role2", "role3", "role4", "role5")); List entries = logger.output(CapturingLogger.Level.ERROR); assertThat(entries, hasSize(1)); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java index 301a705c9f8..66d015f99f2 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java @@ -67,9 +67,10 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ public static final String CONFIG_ROLE_ALLOW_ALL = DEFAULT_ROLE + ":\n" + - " cluster: ALL\n" + + " cluster: [ ALL ]\n" + " indices:\n" + - " '*': ALL\n" + + " - names: '*'\n" + + " privileges: [ ALL ]\n" + DEFAULT_TRANSPORT_CLIENT_ROLE + ":\n" + " cluster:\n" + " - cluster:monitor/nodes/info\n" + diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml index cf6817458e8..3413231018a 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml @@ -1,48 +1,126 @@ admin: - cluster: all + cluster: + - all indices: - '*': all + - names: '*' + privileges: + - all # monitoring cluster privileges # All operations on all indices power_user: - cluster: monitor + cluster: + - monitor indices: - '*': all + - names: '*' + privileges: + - all -# Only operations on indices +# Read-only operations on indices user: indices: - '*': read + - names: '*' + privileges: + - read -# The required role for kibana 3 users -kibana3: - cluster: cluster:monitor/nodes/info - indices: - '*': indices:data/read/search, indices:data/read/get, indices:admin/get - 'kibana-int': indices:data/read/search, indices:data/read/get, indices:data/write/delete, indices:data/write/index, create_index +# Defines the required permissions for transport clients +transport_client: + cluster: + - cluster:monitor/nodes/liveness + #uncomment the following for sniffing + #- cluster:monitor/state -# The required role for kibana 4 users +# The required permissions for kibana 4 users. kibana4: - cluster: cluster:monitor/nodes/info + cluster: + - cluster:monitor/nodes/info + - cluster:monitor/health indices: - '*': indices:data/read/search, indices:data/read/get, indices:admin/get - '.kibana': indices:data/read/search, indices:data/read/get, indices:data/write/delete, indices:data/write/index, create_index + - names: '*' + privileges: + - indices:admin/mappings/fields/get + - indices:admin/validate/query + - indices:data/read/search + - indices:data/read/msearch + - indices:data/read/field_stats + - indices:admin/get + - names: '.kibana' + privileges: + - indices:admin/exists + - indices:admin/mapping/put + - indices:admin/mappings/fields/get + - indices:admin/refresh + - indices:admin/validate/query + - indices:data/read/get + - indices:data/read/mget + - indices:data/read/search + - indices:data/write/delete + - indices:data/write/index + - indices:data/write/update + +# The required permissions for the kibana 4 server +kibana4_server: + cluster: + - cluster:monitor/nodes/info + - cluster:monitor/health + indices: + - names: '.kibana' + privileges: + - indices:admin/create + - indices:admin/exists + - indices:admin/mapping/put + - indices:admin/mappings/fields/get + - indices:admin/refresh + - indices:admin/validate/query + - indices:data/read/get + - indices:data/read/mget + - indices:data/read/search + - indices:data/write/delete + - indices:data/write/index + - indices:data/write/update # The required role for logstash users logstash: - cluster: indices:admin/template/get, indices:admin/template/put + cluster: + - indices:admin/template/get + - indices:admin/template/put indices: - 'logstash-*': indices:data/write/bulk, indices:data/write/delete, indices:data/write/update, create_index + - names: 'logstash-*' + privileges: + - indices:data/write/bulk + - indices:data/write/delete + - indices:data/write/update + - indices:data/read/search + - indices:data/read/scroll + - create_index -# Monitoring role, allowing all operations -# on the monitoring indices +# Monitoring user role. Assign to monitoring users. monitoring_user: indices: - '.monitoring-*': all + - names: '.monitoring-*' + privileges: + - read + - names: '.kibana' + privileges: + - indices:admin/exists + - indices:admin/mappings/fields/get + - indices:admin/validate/query + - indices:data/read/get + - indices:data/read/mget + - indices:data/read/search -# Monitoring Agent users -monitoring_agent: - cluster: indices:admin/template/get, indices:admin/template/put +# Monitoring remote agent role. Assign to the agent user on the remote monitoring cluster +# to which the monitoring agent will export all its data +remote_monitoring_agent: + cluster: + - indices:admin/template/put + - indices:admin/template/get indices: - '.monitoring-*': indices:data/write/bulk, create_index + - names: '.monitoring-*' + privileges: + - all + +# Allows all operations required to manage ingest pipelines +ingest_admin: + cluster: + - manage_pipeline \ No newline at end of file diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml index 796e56df8b6..ddbb0cac447 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/invalid_roles.yml @@ -1,7 +1,10 @@ valid_role: - cluster: ALL + cluster: + - ALL indices: - idx: ALL + - names: idx + privileges: + - ALL "$dlk39": cluster: all @@ -24,4 +27,21 @@ role3: role4: cluster: ALL indices: - '*': al;kjdlkj;lkj \ No newline at end of file + '*': al;kjdlkj;lkj + +#dadfad +# role won't be available since empty privileges... +role5: + cluster: + indices: + - names: + #adfldkkd + - idx2 + privileges: + - names: + - '' + privileges: + - READ + - names: + - 'idx1' + privileges: [] diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml index c7fb7d9e47e..c3bad8a4ee8 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/reserved_roles.yml @@ -1,14 +1,22 @@ admin: - cluster: all + cluster: + - all indices: - '*': all + - names: '*' + privileges: [ all ] __es_system_role: - cluster: all + cluster: + - all indices: - '*' : all + - names: '*' + privileges: + - all __es_internal_role: - cluster: all + cluster: + - all indices: - '*' : all \ No newline at end of file + - names: '*' + privileges: + - all \ No newline at end of file diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml index 280922e741c..766d56487aa 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml @@ -1,31 +1,37 @@ role1: - cluster: ALL + cluster: + - ALL indices: - 'idx1,idx2': READ - idx3: crud + - names: + - idx1 + - idx2 + privileges: + - READ + - names: idx3 + privileges: + - CRUD role1.ab: - cluster: ALL + cluster: + - ALL role2: - cluster: ALL, MONITOR + cluster: + - ALL + - MONITOR role3: indices: - '/.*_.*/': READ, WRITE - -#dadfad -role4: - cluster: - indices: - #adfldkkd - 'idx2': - '': READ - 'idx1': [] + - names: '/.*_.*/' + privileges: + - READ + - WRITE # role with run_as permissions only role_run_as: - run_as: "user1,user2" + run_as: + - user1 + - user2 # role with more than run_as role_run_as1: @@ -33,23 +39,31 @@ role_run_as1: role_fields: indices: - 'field_idx': - privileges: READ + - names: + #23456789ohbh + - 'field_idx' + privileges: + - READ fields: - foo - boo role_query: indices: - 'query_idx': - privileges: READ + - names: + - 'query_idx' + privileges: + - READ query: '{ "match_all": {} }' role_query_fields: indices: - 'query_fields_idx': - privileges: READ - query: '{ "match_all": {} }' + - names: + - 'query_fields_idx' + privileges: + - READ + query: + match_all: fields: - foo - boo \ No newline at end of file diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java index 219974b45c4..b074dd430f0 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -685,19 +685,20 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase public static final String ROLES = "test:\n" + // a user for the test infra. - " cluster: cluster:monitor/nodes/info, cluster:monitor/state, cluster:monitor/health, cluster:monitor/stats, " + - "cluster:admin/settings/update, cluster:admin/repository/delete, cluster:monitor/nodes/liveness, " + - "indices:admin/template/get, indices:admin/template/put, indices:admin/template/delete\n" + + " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/state', 'cluster:monitor/health', 'cluster:monitor/stats'," + + " 'cluster:admin/settings/update', 'cluster:admin/repository/delete', 'cluster:monitor/nodes/liveness'," + + " 'indices:admin/template/get', 'indices:admin/template/put', 'indices:admin/template/delete' ]\n" + " indices:\n" + - " '*': all\n" + + " - names: '*'\n" + + " privileges: [ all ]\n" + "\n" + "admin:\n" + - " cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" + + " cluster: [ 'manage_watcher', 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + "transport_client:\n" + - " cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" + + " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + "\n" + "monitor:\n" + - " cluster: monitor_watcher, cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" + " cluster: [ 'monitor_watcher', 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" ; From 8817d2a3c03154e193dadaede7bc3e7712d7cc06 Mon Sep 17 00:00:00 2001 From: Areek Zillur Date: Wed, 16 Mar 2016 14:01:31 -0400 Subject: [PATCH 17/27] rename license API actions GetLicenseAction: cluster:admin/plugin/license/get --> cluster:monitor/xpack/license/get PutLicenseAction: cluster:admin/plugin/license/put --> cluster:admin/xpack/license/put DeleteLicenseAction: cluster:admin/plugin/license/delete --> cluster:admin/xpack/license/delete closes elastic/elasticsearch#1717 Original commit: elastic/x-pack-elasticsearch@fe3f07cd69e2b30eed100b827074d8a4e586e379 --- .../license/plugin/action/delete/DeleteLicenseAction.java | 2 +- .../license/plugin/action/get/GetLicenseAction.java | 2 +- .../license/plugin/action/put/PutLicenseAction.java | 2 +- .../src/test/resources/org/elasticsearch/transport/actions | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java b/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java index c7c8a675f84..9e0fbb462d0 100644 --- a/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java +++ b/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/delete/DeleteLicenseAction.java @@ -11,7 +11,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class DeleteLicenseAction extends Action { public static final DeleteLicenseAction INSTANCE = new DeleteLicenseAction(); - public static final String NAME = "cluster:admin/plugin/license/delete"; + public static final String NAME = "cluster:admin/xpack/license/delete"; private DeleteLicenseAction() { super(NAME); diff --git a/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java b/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java index d426b1b8aab..8e7de800709 100644 --- a/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java +++ b/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/get/GetLicenseAction.java @@ -11,7 +11,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class GetLicenseAction extends Action { public static final GetLicenseAction INSTANCE = new GetLicenseAction(); - public static final String NAME = "cluster:admin/plugin/license/get"; + public static final String NAME = "cluster:monitor/xpack/license/get"; private GetLicenseAction() { super(NAME); diff --git a/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java b/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java index e90523b9c1d..c4122dfa58d 100644 --- a/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java +++ b/elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/action/put/PutLicenseAction.java @@ -11,7 +11,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class PutLicenseAction extends Action { public static final PutLicenseAction INSTANCE = new PutLicenseAction(); - public static final String NAME = "cluster:admin/plugin/license/put"; + public static final String NAME = "cluster:admin/xpack/license/put"; private PutLicenseAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions index 32bf4a63e4f..579b58738a4 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions @@ -72,9 +72,9 @@ indices:data/write/index indices:data/write/script/delete indices:data/write/script/put indices:data/write/update -cluster:admin/plugin/license/get -cluster:admin/plugin/license/delete -cluster:admin/plugin/license/put +cluster:monitor/xpack/license/get +cluster:admin/xpack/license/delete +cluster:admin/xpack/license/put cluster:admin/shield/realm/cache/clear cluster:admin/shield/roles/cache/clear cluster:admin/shield/user/put From 214b4f269a6a27cc9c68df8e2e8f03eea974aaff Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 17 Mar 2016 14:25:38 +0100 Subject: [PATCH 18/27] Use IndexModule#forceQueryCacheType instead of overriding configrations This is a follow up from elasticsearchelastic/elasticsearch#16799 which prevents setting index level settings on a node level. Original commit: elastic/x-pack-elasticsearch@80d1819ab3fc93ea1059dd0708660c9346aab4e0 --- .../marvel/test/MarvelIntegTestCase.java | 9 +---- .../java/org/elasticsearch/shield/Shield.java | 33 +++---------------- .../org/elasticsearch/shield/ShieldF.java | 2 -- .../audit/index/IndexAuditTrailTests.java | 9 ----- .../test/ShieldSettingsSource.java | 5 --- .../AbstractWatcherIntegrationTestCase.java | 5 --- 6 files changed, 6 insertions(+), 57 deletions(-) diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index 218535a8841..1f404a59ec5 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.MonitoredSystem; @@ -25,7 +24,6 @@ import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.shield.Shield; import org.elasticsearch.shield.authc.esusers.ESUsersRealm; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; @@ -463,8 +461,6 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { Path folder = createTempDir().resolve("marvel_shield"); Files.createDirectories(folder); - builder.remove("index.queries.cache.type"); - builder.put("shield.enabled", true) .put("shield.authc.realms.esusers.type", ESUsersRealm.TYPE) .put("shield.authc.realms.esusers.order", 0) @@ -473,10 +469,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", ROLES)) .put("shield.system_key.file", writeFile(folder, "system_key.yml", systemKey)) .put("shield.authc.sign_user_header", false) - .put("shield.audit.enabled", auditLogsEnabled) - // Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the - // validation in ShieldPlugin fail. Shield can only run with this query cache impl - .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), Shield.OPT_OUT_QUERY_CACHE); + .put("shield.audit.enabled", auditLogsEnabled); } catch (IOException ex) { throw new RuntimeException("failed to build settings for shield", ex); } diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java index 57cc08d3be4..4c0ce5a3c8a 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java @@ -109,7 +109,6 @@ public class Shield { this.transportClientMode = XPackPlugin.transportClientMode(settings); this.enabled = XPackPlugin.featureEnabled(settings, NAME, true); if (enabled && !transportClientMode) { - failIfShieldQueryCacheIsNotActive(settings, true); validateAutoCreateIndex(settings); } } @@ -172,7 +171,6 @@ public class Shield { settingsBuilder.put(NetworkModule.HTTP_TYPE_SETTING.getKey(), Shield.NAME); addUserSettings(settingsBuilder); addTribeSettings(settingsBuilder); - addQueryCacheSettings(settingsBuilder); return settingsBuilder.build(); } @@ -235,7 +233,11 @@ public class Shield { } if (transportClientMode == false) { module.registerQueryCache(Shield.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new); - failIfShieldQueryCacheIsNotActive(module.getSettings(), false); + /* We need to forcefully overwrite the query cache implementation to use Shield's opt out query cache implementation. + * This impl. disabled the query cache if field level security is used for a particular request. If we wouldn't do + * forcefully overwrite the query cache implementation then we leave the system vulnerable to leakages of data to + * unauthorized users. */ + module.forceQueryCacheType(Shield.OPT_OUT_QUERY_CACHE); } } @@ -386,16 +388,6 @@ public class Shield { } } - /** - * We need to forcefully overwrite the query cache implementation to use Shield's opt out query cache implementation. - * This impl. disabled the query cache if field level security is used for a particular request. If we wouldn't do - * forcefully overwrite the query cache implementation then we leave the system vulnerable to leakages of data to - * unauthorized users. - */ - private void addQueryCacheSettings(Settings.Builder settingsBuilder) { - settingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), OPT_OUT_QUERY_CACHE); - } - public static boolean enabled(Settings settings) { return XPackPlugin.featureEnabled(settings, NAME, true); } @@ -404,21 +396,6 @@ public class Shield { return XPackPlugin.featureEnabled(settings, DLS_FLS_FEATURE, true); } - private void failIfShieldQueryCacheIsNotActive(Settings settings, boolean nodeSettings) { - String queryCacheImplementation; - if (nodeSettings) { - // in case this are node settings then the plugin additional settings have not been applied yet, - // so we use 'opt_out_cache' as default. So in that case we only fail if the node settings contain - // another cache impl than 'opt_out_cache'. - queryCacheImplementation = settings.get(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), OPT_OUT_QUERY_CACHE); - } else { - queryCacheImplementation = settings.get(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey()); - } - if (OPT_OUT_QUERY_CACHE.equals(queryCacheImplementation) == false) { - throw new IllegalStateException("shield does not support a user specified query cache. remove the setting [" + IndexModule - .INDEX_QUERY_CACHE_TYPE_SETTING.getKey() + "] with value [" + queryCacheImplementation + "]"); - } - } static void validateAutoCreateIndex(Settings settings) { String value = settings.get("action.auto_create_index"); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldF.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldF.java index 92c199083ab..c4561a98406 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldF.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldF.java @@ -10,7 +10,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexModule; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.shield.authc.esnative.ESNativeRealm; @@ -43,7 +42,6 @@ public class ShieldF { settings.put("xpack.shield.enabled", "true"); // Disable Monitoring to prevent cluster activity settings.put("xpack.monitoring.enabled", "false"); - settings.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), Shield.OPT_OUT_QUERY_CACHE); settings.put("cluster.name", ShieldF.class.getSimpleName()); String homeDir = System.getProperty("es.path.home"); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java index cfccbc7805a..6d226741692 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.SearchHit; @@ -168,14 +167,6 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase { Settings.Builder builder = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(XPackPlugin.featureEnabledSetting(Shield.NAME), useShield); - - // For tests we forcefully configure Shield's custom query cache because the test framework - // randomizes the query cache impl but if shield is disabled then we don't need to forcefully - // set the query cache - if (useShield == false) { - builder.remove(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey()); - } - return builder.build(); } }; diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java index 66d015f99f2..ceca8a36646 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java @@ -9,10 +9,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexModule; import org.elasticsearch.marvel.Marvel; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.shield.Shield; import org.elasticsearch.shield.authc.esusers.ESUsersRealm; import org.elasticsearch.shield.authc.esnative.ESNativeRealm; import org.elasticsearch.shield.authc.support.Hasher; @@ -136,9 +134,6 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ .put("shield.authc.realms.index.type", ESNativeRealm.TYPE) .put("shield.authc.realms.index.order", "1") .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRoles())) - // Test framework sometimes randomly selects the 'index' or 'none' cache and that makes the - // validation in ShieldPlugin fail. - .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), Shield.OPT_OUT_QUERY_CACHE) .put(getNodeSSLSettings()); return builder.build(); diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java index b074dd430f0..2859e9ad664 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -21,14 +21,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Callback; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.marvel.Marvel; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockMustacheScriptEngine; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.shield.Shield; import org.elasticsearch.shield.authc.esusers.ESUsersRealm; import org.elasticsearch.shield.authc.support.Hasher; import org.elasticsearch.shield.authc.support.SecuredString; @@ -719,9 +717,6 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase .put("shield.system_key.file", writeFile(folder, "system_key.yml", systemKey)) .put("shield.authc.sign_user_header", false) .put("shield.audit.enabled", auditLogsEnabled) - // Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the - // validation in ShieldPlugin fail. Shield can only run with this query cache impl - .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), Shield.OPT_OUT_QUERY_CACHE) .build(); } catch (IOException ex) { throw new RuntimeException("failed to build settings for shield", ex); From 03f8452d705a6ac846cc80f35c0a6b7065b70f09 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Thu, 17 Mar 2016 10:27:44 -0700 Subject: [PATCH 19/27] Update API spec for Shield Original commit: elastic/x-pack-elasticsearch@6dcdfebc0aab8811caa4302135f2691464ef52a1 --- .../src/test/resources/rest-api-spec/api/shield.get_role.json | 2 +- .../src/test/resources/rest-api-spec/api/shield.get_user.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_role.json b/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_role.json index f5ac8c9cecc..2dc241795b5 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_role.json +++ b/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_role.json @@ -4,7 +4,7 @@ "methods": [ "GET" ], "url": { "path": "/_shield/role/{name}", - "paths": [ "/_shield/role/{name}" ], + "paths": [ "/_shield/role/{name}", "/_shield/role" ], "parts": { "name": { "type" : "string", diff --git a/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_user.json b/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_user.json index 8bb75ec0a1c..da821e5e67f 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_user.json +++ b/elasticsearch/x-pack/shield/src/test/resources/rest-api-spec/api/shield.get_user.json @@ -4,7 +4,7 @@ "methods": [ "GET" ], "url": { "path": "/_shield/user/{username}", - "paths": [ "/_shield/user/{username}" ], + "paths": [ "/_shield/user/{username}", "/_shield/user" ], "parts": { "username": { "type" : "list", From a22539aca02620ec01faf50ff50d2246ff90d857 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 8 Mar 2016 14:20:54 -0500 Subject: [PATCH 20/27] shield: add support for new privilege naming This commit adds support for the privilege naming defined in elastic/elasticsearch#1342 and removes the support for the privileges that were deprecated in 2.3. This change also includes updates to the documentation to account for the new roles format. Original commit: elastic/x-pack-elasticsearch@98e9afd40990e3f6fa9796e627417c82e8d162b3 --- .../messy/tests/ShieldCachePermissionIT.java | 3 +- .../qa/shield-reindex-tests/roles.yml | 14 +- .../x-pack/shield/config/xpack/roles.yml | 92 ++------ .../authz/privilege/ClusterPrivilege.java | 44 +++- .../authz/privilege/IndexPrivilege.java | 80 ++++--- .../shield/support/Automatons.java | 2 + .../DocumentLevelSecurityRandomTests.java | 3 +- .../DocumentLevelSecurityTests.java | 6 +- .../FieldLevelSecurityRandomTests.java | 6 +- .../SearchGetAndSuggestPermissionsTests.java | 199 ------------------ .../shield/authz/IndexAliasesTests.java | 22 +- .../InternalAuthorizationServiceTests.java | 2 +- .../authz/permission/PermissionTests.java | 3 +- .../authz/privilege/PrivilegeTests.java | 28 --- .../authz/store/FileRolesStoreTests.java | 9 +- .../test/ShieldSettingsSource.java | 3 +- .../shield/authz/store/default_roles.yml | 92 ++------ .../shield/authz/store/roles.yml | 3 +- 18 files changed, 167 insertions(+), 444 deletions(-) delete mode 100644 elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/ShieldCachePermissionIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/ShieldCachePermissionIT.java index 4e5c2ab6f35..ba3471e61c5 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/ShieldCachePermissionIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/ShieldCachePermissionIT.java @@ -53,7 +53,8 @@ public class ShieldCachePermissionIT extends ShieldIntegTestCase { return super.configRoles() + "\nread_one_idx:\n" + " indices:\n" - + " 'data': READ\n"; + + " 'data':\n" + + " - read\n"; } @Override diff --git a/elasticsearch/qa/shield-reindex-tests/roles.yml b/elasticsearch/qa/shield-reindex-tests/roles.yml index 2aaec4b9b6a..059156cd5a6 100644 --- a/elasticsearch/qa/shield-reindex-tests/roles.yml +++ b/elasticsearch/qa/shield-reindex-tests/roles.yml @@ -13,13 +13,13 @@ minimal: indices: - names: source privileges: - - search + - read - write - create_index - indices:admin/refresh - names: dest privileges: - - search + - read - write - create_index - indices:admin/refresh @@ -28,7 +28,7 @@ minimal: readonly: indices: - names: '*' - privileges: [ search ] + privileges: [ read ] # Write operations on destination index, none on source index dest_only: @@ -41,7 +41,7 @@ can_not_see_hidden_docs: indices: - names: source privileges: - - search + - read - write - create_index - indices:admin/refresh @@ -52,7 +52,7 @@ can_not_see_hidden_docs: hidden: true - names: dest privileges: - - search + - read - write - create_index - indices:admin/refresh @@ -62,7 +62,7 @@ can_not_see_hidden_fields: indices: - names: source privileges: - - search + - read - write - create_index - indices:admin/refresh @@ -71,7 +71,7 @@ can_not_see_hidden_fields: - bar - names: dest privileges: - - search + - read - write - create_index - indices:admin/refresh diff --git a/elasticsearch/x-pack/shield/config/xpack/roles.yml b/elasticsearch/x-pack/shield/config/xpack/roles.yml index b43fc06ea7b..1c36fd5ab07 100644 --- a/elasticsearch/x-pack/shield/config/xpack/roles.yml +++ b/elasticsearch/x-pack/shield/config/xpack/roles.yml @@ -26,101 +26,51 @@ user: # Defines the required permissions for transport clients transport_client: cluster: - - cluster:monitor/nodes/liveness - #uncomment the following for sniffing - #- cluster:monitor/state - -# The required permissions for kibana 4 users. -kibana4: - cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health - indices: - - names: '*' - privileges: - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/search - - indices:data/read/msearch - - indices:data/read/field_stats - - indices:admin/get - - names: '.kibana' - privileges: - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update + - transport_client # The required permissions for the kibana 4 server kibana4_server: cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health + - monitor indices: - names: '.kibana' privileges: - - indices:admin/create - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update + - all # The required role for logstash users logstash: cluster: - - indices:admin/template/get - - indices:admin/template/put + - manage_index_templates indices: - names: 'logstash-*' privileges: - - indices:data/write/bulk - - indices:data/write/delete - - indices:data/write/update - - indices:data/read/search - - indices:data/read/scroll + - write + - read - create_index -# Monitoring user role. Assign to monitoring users. +# Marvel user role. Assign to marvel users. monitoring_user: indices: - - names: '.monitoring-*' - privileges: - - read + - names: + - '.marvel-es-*' + - '.monitoring-*' + privileges: [ "read" ] - names: '.kibana' privileges: - - indices:admin/exists - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search + - view_index_metadata + - read -# Monitoring remote agent role. Assign to the agent user on the remote monitoring cluster -# to which the monitoring agent will export all its data +# Marvel remote agent role. Assign to the agent user on the remote marvel cluster +# to which the marvel agent will export all its data remote_monitoring_agent: - cluster: - - indices:admin/template/put - - indices:admin/template/get + cluster: [ "manage_index_templates" ] indices: - - names: '.monitoring-*' - privileges: - - all + - names: + - '.marvel-es-*' + - '.monitoring-*' + privileges: [ "all" ] # Allows all operations required to manage ingest pipelines ingest_admin: cluster: - - manage_pipeline \ No newline at end of file + - manage_pipeline diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java index 82ae689c414..9dc00dae90b 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java @@ -6,8 +6,10 @@ package org.elasticsearch.shield.authz.privilege; import dk.brics.automaton.Automaton; -import dk.brics.automaton.BasicAutomata; import org.elasticsearch.common.Strings; +import org.elasticsearch.shield.action.realm.ClearRealmCacheAction; +import org.elasticsearch.shield.action.role.ClearRolesCacheAction; +import org.elasticsearch.shield.support.Automatons; import java.util.Locale; import java.util.Set; @@ -15,16 +17,35 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.function.Predicate; +import static org.elasticsearch.shield.support.Automatons.minusAndDeterminize; +import static org.elasticsearch.shield.support.Automatons.patterns; + /** * */ public class ClusterPrivilege extends AbstractAutomatonPrivilege { - public static final ClusterPrivilege NONE = new ClusterPrivilege(Name.NONE, BasicAutomata.makeEmpty()); - public static final ClusterPrivilege ALL = new ClusterPrivilege(Name.ALL, "cluster:*", "indices:admin/template/*"); - public static final ClusterPrivilege MONITOR = new ClusterPrivilege("monitor", "cluster:monitor/*"); - public static final ClusterPrivilege MANAGE_SHIELD = new ClusterPrivilege("manage_shield", "cluster:admin/shield/*"); - public static final ClusterPrivilege MANAGE_PIPELINE = new ClusterPrivilege("manage_pipeline", "cluster:admin/ingest/pipeline/*"); + // shared automatons + private static final Automaton MANAGE_USER_AUTOMATON = patterns("cluster:admin/shield/user/*", ClearRolesCacheAction.NAME); + private static final Automaton MANAGE_ROLE_AUTOMATON = patterns("cluster:admin/shield/role/*", ClearRealmCacheAction.NAME); + private static final Automaton MANAGE_SECURITY_AUTOMATON = patterns("cluster:admin/shield/*"); + private static final Automaton MONITOR_AUTOMATON = patterns("cluster:monitor/*"); + private static final Automaton ALL_CLUSTER_AUTOMATON = patterns("cluster:*", "indices:admin/template/*"); + private static final Automaton MANAGE_AUTOMATON = minusAndDeterminize(ALL_CLUSTER_AUTOMATON, MANAGE_SECURITY_AUTOMATON); + private static final Automaton TRANSPORT_CLIENT_AUTOMATON = patterns("cluster:monitor/nodes/liveness", "cluster:monitor/state"); + private static final Automaton MANAGE_IDX_TEMPLATE_AUTOMATON = patterns("indices:admin/template/*"); + + public static final ClusterPrivilege NONE = new ClusterPrivilege(Name.NONE, Automatons.EMPTY); + public static final ClusterPrivilege ALL = new ClusterPrivilege(Name.ALL, ALL_CLUSTER_AUTOMATON); + public static final ClusterPrivilege MONITOR = new ClusterPrivilege("monitor", MONITOR_AUTOMATON); + public static final ClusterPrivilege MANAGE = new ClusterPrivilege("manage", MANAGE_AUTOMATON); + public static final ClusterPrivilege MANAGE_IDX_TEMPLATES = + new ClusterPrivilege("manage_index_templates", MANAGE_IDX_TEMPLATE_AUTOMATON); + public static final ClusterPrivilege TRANSPORT_CLIENT = new ClusterPrivilege("transport_client", TRANSPORT_CLIENT_AUTOMATON); + public static final ClusterPrivilege MANAGE_USERS = new ClusterPrivilege("manage_users", MANAGE_USER_AUTOMATON); + public static final ClusterPrivilege MANAGE_ROLES = new ClusterPrivilege("manage_roles", MANAGE_ROLE_AUTOMATON); + public static final ClusterPrivilege MANAGE_SECURITY = new ClusterPrivilege("manage_security", MANAGE_SECURITY_AUTOMATON); + public static final ClusterPrivilege MANAGE_PIPELINE = new ClusterPrivilege("manage_pipeline", "cluster:admin/ingest/pipeline/*"); public final static Predicate ACTION_MATCHER = ClusterPrivilege.ALL.predicate(); @@ -34,7 +55,12 @@ public class ClusterPrivilege extends AbstractAutomatonPrivilege { - public static final IndexPrivilege NONE = new IndexPrivilege(Name.NONE, BasicAutomata.makeEmpty()); - public static final IndexPrivilege ALL = new IndexPrivilege(Name.ALL, "indices:*"); - public static final IndexPrivilege MANAGE = new IndexPrivilege("manage", "indices:monitor/*", "indices:admin/*"); - public static final IndexPrivilege CREATE_INDEX = new IndexPrivilege("create_index", CreateIndexAction.NAME); - public static final IndexPrivilege MANAGE_ALIASES = new IndexPrivilege("manage_aliases", "indices:admin/aliases*"); - public static final IndexPrivilege MONITOR = new IndexPrivilege("monitor", "indices:monitor/*"); - public static final IndexPrivilege DATA_ACCESS = new IndexPrivilege("data_access", "indices:data/*", "indices:admin/mapping/put"); - public static final IndexPrivilege CRUD = - new IndexPrivilege("crud", "indices:data/write/*", "indices:data/read/*", "indices:admin/mapping/put"); - public static final IndexPrivilege READ = new IndexPrivilege("read", "indices:data/read/*"); - public static final IndexPrivilege SEARCH = - new IndexPrivilege("search", SearchAction.NAME + "*", MultiSearchAction.NAME + "*", SuggestAction.NAME + "*"); - public static final IndexPrivilege GET = new IndexPrivilege("get", GetAction.NAME + "*", MultiGetAction.NAME + "*"); - public static final IndexPrivilege SUGGEST = new IndexPrivilege("suggest", SuggestAction.NAME + "*"); - public static final IndexPrivilege INDEX = - new IndexPrivilege("index", "indices:data/write/index*", "indices:data/write/update*", "indices:admin/mapping/put"); - public static final IndexPrivilege DELETE = new IndexPrivilege("delete", "indices:data/write/delete*"); - public static final IndexPrivilege WRITE = new IndexPrivilege("write", "indices:data/write/*", "indices:admin/mapping/put"); + private static final Automaton ALL_AUTOMATON = patterns("indices:*"); + private static final Automaton READ_AUTOMATON = patterns("indices:data/read/*"); + private static final Automaton CREATE_AUTOMATON = patterns("indices:data/write/index*", PutMappingAction.NAME); + private static final Automaton INDEX_AUTOMATON = + patterns("indices:data/write/index*", "indices:data/write/update*", PutMappingAction.NAME); + private static final Automaton DELETE_AUTOMATON = patterns("indices:data/write/delete*"); + private static final Automaton WRITE_AUTOMATON = patterns("indices:data/write/*", PutMappingAction.NAME); + private static final Automaton MONITOR_AUTOMATON = patterns("indices:monitor/*"); + private static final Automaton MANAGE_AUTOMATON = unionAndDeterminize(MONITOR_AUTOMATON, patterns("indices:admin/*")); + private static final Automaton CREATE_INDEX_AUTOMATON = patterns(CreateIndexAction.NAME); + private static final Automaton DELETE_INDEX_AUTOMATON = patterns(DeleteIndexAction.NAME); + private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME, AliasesExistAction.NAME, + GetIndexAction.NAME, IndicesExistsAction.NAME, GetFieldMappingsAction.NAME, GetMappingsAction.NAME, + ClusterSearchShardsAction.NAME, TypesExistsAction.NAME, ValidateQueryAction.NAME, GetSettingsAction.NAME); + + public static final IndexPrivilege NONE = new IndexPrivilege(Name.NONE, Automatons.EMPTY); + public static final IndexPrivilege ALL = new IndexPrivilege(Name.ALL, ALL_AUTOMATON); + public static final IndexPrivilege READ = new IndexPrivilege("read", READ_AUTOMATON); + public static final IndexPrivilege CREATE = new IndexPrivilege("create", CREATE_AUTOMATON); + public static final IndexPrivilege INDEX = new IndexPrivilege("index", INDEX_AUTOMATON); + public static final IndexPrivilege DELETE = new IndexPrivilege("delete", DELETE_AUTOMATON); + public static final IndexPrivilege WRITE = new IndexPrivilege("write", WRITE_AUTOMATON); + public static final IndexPrivilege MONITOR = new IndexPrivilege("monitor", MONITOR_AUTOMATON); + public static final IndexPrivilege MANAGE = new IndexPrivilege("manage", MANAGE_AUTOMATON); + public static final IndexPrivilege DELETE_INDEX = new IndexPrivilege("delete_index", DELETE_INDEX_AUTOMATON); + public static final IndexPrivilege CREATE_INDEX = new IndexPrivilege("create_index", CREATE_INDEX_AUTOMATON); + public static final IndexPrivilege VIEW_METADATA = new IndexPrivilege("view_index_metadata", VIEW_METADATA_AUTOMATON); private static final Set values = new CopyOnWriteArraySet<>(); @@ -52,17 +71,14 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { values.add(ALL); values.add(MANAGE); values.add(CREATE_INDEX); - values.add(MANAGE_ALIASES); values.add(MONITOR); - values.add(DATA_ACCESS); - values.add(CRUD); values.add(READ); - values.add(SEARCH); - values.add(GET); - values.add(SUGGEST); values.add(INDEX); values.add(DELETE); values.add(WRITE); + values.add(CREATE); + values.add(DELETE_INDEX); + values.add(VIEW_METADATA); } public static final Predicate ACTION_MATCHER = ALL.predicate(); @@ -78,8 +94,8 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { super(name, patterns); } - private IndexPrivilege(Name name, String... patterns) { - super(name, patterns); + private IndexPrivilege(String name, Automaton automaton) { + super(new Name(name), automaton); } private IndexPrivilege(Name name, Automaton automaton) { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java index 03ae9e41011..48280b2c599 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/support/Automatons.java @@ -24,6 +24,8 @@ import static dk.brics.automaton.MinimizationOperations.minimize; */ public final class Automatons { + public static final Automaton EMPTY = BasicAutomata.makeEmpty(); + static final char WILDCARD_STRING = '*'; // String equality with support for wildcards static final char WILDCARD_CHAR = '?'; // Char equality with support for wildcards static final char WILDCARD_ESCAPE = '\\'; // Escape character diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java index 3a82a5f8706..633c3d9c77c 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java @@ -63,7 +63,8 @@ public class DocumentLevelSecurityRandomTests extends ShieldIntegTestCase { builder.append(" cluster: [ all ]\n"); builder.append(" indices:\n"); builder.append(" - names: '*'\n"); - builder.append(" privileges: [ ALL ]\n"); + builder.append(" privileges:\n"); + builder.append(" - all\n"); builder.append(" query: \n"); builder.append(" term: \n"); builder.append(" field1: value").append(i).append('\n'); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 0ab2f3e34ae..48678fc9e63 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -73,10 +73,12 @@ public class DocumentLevelSecurityTests extends ShieldIntegTestCase { protected String configRoles() { return super.configRoles() + "\nrole1:\n" + - " cluster: [ all ]\n" + + " cluster:\n" + + " - all\n" + " indices:\n" + " - names: '*'\n" + - " privileges: [ ALL ]\n" + + " privileges:\n" + + " - all\n" + " query: \n" + " term: \n" + " field1: value1\n" + diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java index 46cb49b1db9..f4c48692e6b 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java @@ -86,10 +86,12 @@ public class FieldLevelSecurityRandomTests extends ShieldIntegTestCase { " privileges: [ ALL ]\n" + " fields:\n" +roleFields.toString() + "role2:\n" + - " cluster: [ all ]\n" + + " cluster:\n" + + " - all\n" + " indices:\n" + " - names: test\n" + - " privileges: [ ALL ]\n" + + " privileges:\n" + + " - all\n" + " fields:\n" + " - field1\n" + "role3:\n" + diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java deleted file mode 100644 index c4208907190..00000000000 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.integration; - -import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.suggest.SuggestResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.search.suggest.SuggestBuilders; -import org.elasticsearch.shield.authc.support.Hasher; -import org.elasticsearch.shield.authc.support.SecuredString; -import org.elasticsearch.shield.authc.support.SecuredStringTests; -import org.elasticsearch.shield.authc.support.UsernamePasswordToken; -import org.elasticsearch.test.ShieldIntegTestCase; - -import java.util.Map; - -import static java.util.Collections.singletonMap; -import static org.elasticsearch.client.Requests.searchRequest; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ShieldTestsUtils.assertAuthorizationException; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -public class SearchGetAndSuggestPermissionsTests extends ShieldIntegTestCase { - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray()))); - - @Override - protected String configRoles() { - return super.configRoles() + "\n" + - "\n" + - "search_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ search ]\n" + - "\n" + - "get_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ get ]\n" + - "\n" + - "suggest_role:\n" + - " indices:\n" + - " - names: 'a'\n" + - " privileges: [ suggest ]\n"; - } - - @Override - protected String configUsers() { - return super.configUsers() + - "search_user:" + USERS_PASSWD_HASHED + "\n" + - "get_user:" + USERS_PASSWD_HASHED + "\n" + - "suggest_user:" + USERS_PASSWD_HASHED + "\n"; - - } - - @Override - protected String configUsersRoles() { - return super.configUsersRoles() + - "search_role:search_user\n" + - "get_role:get_user\n" + - "suggest_role:suggest_user\n"; - } - - /** - * testing both "search" and "suggest" privileges can execute the suggest API - */ - public void testSuggestAPI() throws Exception { - IndexResponse indexResponse = index("a", "type", jsonBuilder() - .startObject() - .field("name", "value") - .endObject()); - assertThat(indexResponse.isCreated(), is(true)); - - refresh(); - - Client client = internalCluster().transportClient(); - - Map headers = singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("suggest_user", "passwd")); - SuggestResponse suggestResponse = client.filterWithHeader(headers) - .prepareSuggest("a") - .addSuggestion(randomAsciiOfLengthBetween(3,7), SuggestBuilders.termSuggestion("name").text("val")).get(); - assertNoFailures(suggestResponse); - assertThat(suggestResponse.getSuggest().size(), is(1)); - - suggestResponse = client - .filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("search_user", "passwd"))) - .prepareSuggest("a") - .addSuggestion(randomAsciiOfLengthBetween(3, 7), SuggestBuilders.termSuggestion("name").text("val")).get(); - assertNoFailures(suggestResponse); - assertThat(suggestResponse.getSuggest().size(), is(1)); - - try { - client.filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("suggest_user", "passwd"))) - .prepareSearch("a") - .get(); - fail("a user with only a suggest privilege cannot execute search"); - } catch (ElasticsearchSecurityException e) { - logger.error("failed to search", e); - // expected - } - } - - /** - * testing that "search" privilege cannot execute the get API - */ - public void testGetAPI() throws Exception { - IndexResponse indexResponse = index("a", "type", jsonBuilder() - .startObject() - .field("name", "value") - .endObject()); - assertThat(indexResponse.isCreated(), is(true)); - - refresh(); - - Client client = internalCluster().transportClient(); - - try { - client.filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("search_user", "passwd"))) - .prepareGet("a", "type", indexResponse.getId()) - .get(); - fail("a user with only search privilege should not be authorized for a get request"); - } catch (ElasticsearchSecurityException e) { - // expected - assertAuthorizationException(e); - logger.error("could not get document", e); - } - } - - /** - * testing that "get" privilege can execute the mget API, and "search" privilege cannot execute mget - */ - public void testMultiGetAPI() throws Exception { - IndexResponse indexResponse = index("a", "type", jsonBuilder() - .startObject() - .field("name", "value") - .endObject()); - assertThat(indexResponse.isCreated(), is(true)); - - refresh(); - - Client client = internalCluster().transportClient(); - - MultiGetResponse response = client - .filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("get_user", "passwd"))) - .prepareMultiGet().add("a", "type", indexResponse.getId()) - .get(); - assertNotNull(response); - assertThat(response.getResponses().length, is(1)); - assertThat(response.getResponses()[0].getId(), equalTo(indexResponse.getId())); - - try { - client.filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("search_user", "passwd"))) - .prepareMultiGet().add("a", "type", indexResponse.getId()) - .get(); - fail("a user with only a search privilege should not be able to execute the mget API"); - } catch (ElasticsearchSecurityException e) { - // expected - assertAuthorizationException(e); - logger.error("could not mget documents", e); - } - } - - /** - * testing that "search" privilege can execute the msearch API - */ - public void testMultiSearchAPI() throws Exception { - IndexResponse indexResponse = index("a", "type", jsonBuilder() - .startObject() - .field("name", "value") - .endObject()); - assertThat(indexResponse.isCreated(), is(true)); - - refresh(); - - Client client = internalCluster().transportClient(); - - MultiSearchResponse response = client - .filterWithHeader(singletonMap(UsernamePasswordToken.BASIC_AUTH_HEADER, userHeader("search_user", "passwd"))) - .prepareMultiSearch().add(searchRequest("a").types("type")) - .get(); - assertNotNull(response); - assertThat(response.getResponses().length, is(1)); - SearchResponse first = response.getResponses()[0].getResponse(); - assertNotNull(first); - assertNoFailures(first); - } - - private static String userHeader(String username, String password) { - return UsernamePasswordToken.basicAuthHeaderValue(username, SecuredStringTests.build(password)); - } -} diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java index 43374091036..14564560d53 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java @@ -60,30 +60,30 @@ public class IndexAliasesTests extends ShieldIntegTestCase { " indices:\n" + " - names: '*'\n" + " privileges: [ create_index ]\n" + - //role that has create index and managa aliases on test_*, not enough to manage aliases outside of test_* namespace + //role that has create index and manage_aliases on test_*, not enough to manage_aliases aliases outside of test_* namespace "create_test_aliases_test:\n" + " indices:\n" + " - names: 'test_*'\n" + - " privileges: [ create_index, manage_aliases ]\n" + - //role that has create index on test_* and manage aliases on alias_*, can't create aliases pointing to test_* though + " privileges: [ create_index, 'indices:admin/aliases*' ]\n" + + //role that has create index on test_* and manage_aliases on alias_*, can't create aliases pointing to test_* though "create_test_aliases_alias:\n" + " indices:\n" + " - names: 'test_*'\n" + " privileges: [ create_index ]\n" + " - names: 'alias_*'\n" + - " privileges: [ manage_aliases ]\n" + + " privileges: [ 'indices:admin/aliases*' ]\n" + //role that has create index on test_* and manage_aliases on both alias_* and test_* "create_test_aliases_test_alias:\n" + " indices:\n" + " - names: 'test_*'\n" + " privileges: [ create_index ]\n" + " - names: [ 'alias_*', 'test_*' ]\n" + - " privileges: [ manage_aliases ]\n" + + " privileges: [ 'indices:admin/aliases*' ]\n" + //role that has manage_aliases only on both test_* and alias_* "aliases_only:\n" + " indices:\n" + " - names: [ 'alias_*', 'test_*']\n" + - " privileges: [ manage_aliases ]\n"; + " privileges: [ 'indices:admin/aliases*' ]\n"; } @Before @@ -368,7 +368,7 @@ public class IndexAliasesTests extends ShieldIntegTestCase { assertAcked(client.admin().indices().prepareCreate("test_1")); try { - //fails: user doesn't have manage aliases on test_1 + //fails: user doesn't have manage_aliases aliases on test_1 client.admin().indices().prepareAliases().addAlias("test_1", "test_alias").get(); fail("add alias should have failed due to missing manage_aliases privileges on test_alias and test_1"); } catch(ElasticsearchSecurityException e) { @@ -377,7 +377,7 @@ public class IndexAliasesTests extends ShieldIntegTestCase { } try { - //fails: user doesn't have manage aliases on test_1 + //fails: user doesn't have manage_aliases aliases on test_1 client.admin().indices().prepareAliases().addAlias("test_1", "alias_1").get(); fail("add alias should have failed due to missing manage_aliases privileges on test_1"); } catch(ElasticsearchSecurityException e) { @@ -386,7 +386,7 @@ public class IndexAliasesTests extends ShieldIntegTestCase { } try { - //fails: user doesn't have manage aliases on test_*, no matching indices to replace wildcards + //fails: user doesn't have manage_aliases aliases on test_*, no matching indices to replace wildcards client.admin().indices().prepareAliases().addAlias("test_*", "alias_1").get(); fail("add alias should have failed due to missing manage_aliases privileges on test_1"); } catch(IndexNotFoundException e) { @@ -465,7 +465,7 @@ public class IndexAliasesTests extends ShieldIntegTestCase { assertAcked(client.admin().indices().prepareCreate("test_1")); try { - //fails: user doesn't have manage aliases on test_1, nor test_alias + //fails: user doesn't have manage_aliases aliases on test_1, nor test_alias client.admin().indices().prepareGetAliases().setAliases("test_alias").setIndices("test_1").get(); fail("get alias should have failed due to missing manage_aliases privileges on test_alias and test_1"); } catch(ElasticsearchSecurityException e) { @@ -474,7 +474,7 @@ public class IndexAliasesTests extends ShieldIntegTestCase { } try { - //fails: user doesn't have manage aliases on test_*, no matching indices to replace wildcards + //fails: user doesn't have manage_aliases aliases on test_*, no matching indices to replace wildcards client.admin().indices().prepareGetAliases().setIndices("test_*").setAliases("test_alias").get(); fail("get alias should have failed due to missing manage_aliases privileges on test_*"); } catch(IndexNotFoundException e) { diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java index 3e11e9dd00e..4863d157686 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java @@ -302,7 +302,7 @@ public class InternalAuthorizationServiceTests extends ESTestCase { User user = new User("test user", "a_star", "b"); ClusterState state = mock(ClusterState.class); when(rolesStore.role("a_star")).thenReturn(Role.builder("a_star").add(IndexPrivilege.ALL, "a*").build()); - when(rolesStore.role("b")).thenReturn(Role.builder("a_star").add(IndexPrivilege.SEARCH, "b").build()); + when(rolesStore.role("b")).thenReturn(Role.builder("a_star").add(IndexPrivilege.READ, "b").build()); when(clusterService.state()).thenReturn(state); Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); when(state.metaData()).thenReturn(MetaData.builder() diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/PermissionTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/PermissionTests.java index 255b8f78644..b3ae5bb80df 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/PermissionTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/PermissionTests.java @@ -18,7 +18,6 @@ import java.util.function.Predicate; import static org.elasticsearch.shield.authz.privilege.IndexPrivilege.MONITOR; import static org.elasticsearch.shield.authz.privilege.IndexPrivilege.READ; -import static org.elasticsearch.shield.authz.privilege.IndexPrivilege.SEARCH; import static org.elasticsearch.shield.authz.privilege.IndexPrivilege.union; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -33,7 +32,7 @@ public class PermissionTests extends ESTestCase { @Before public void init() { Role.Builder builder = Role.builder("test"); - builder.add(union(SEARCH, MONITOR), "test_*", "/foo.*/"); + builder.add(union(MONITOR), "test_*", "/foo.*/"); builder.add(union(READ), "baz_*foo", "/fool.*bar/"); builder.add(union(MONITOR), "/bar.*/"); permission = builder.build(); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java index 5be16799a12..11fb580ea5b 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java @@ -5,15 +5,10 @@ */ package org.elasticsearch.shield.authz.privilege; -import org.elasticsearch.action.get.GetAction; -import org.elasticsearch.action.get.MultiGetAction; import org.elasticsearch.action.ingest.DeletePipelineAction; import org.elasticsearch.action.ingest.GetPipelineAction; import org.elasticsearch.action.ingest.PutPipelineAction; import org.elasticsearch.action.ingest.SimulatePipelineAction; -import org.elasticsearch.action.search.MultiSearchAction; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.suggest.SuggestAction; import org.elasticsearch.shield.support.AutomatonPredicate; import org.elasticsearch.shield.support.Automatons; import org.elasticsearch.test.ESTestCase; @@ -262,27 +257,4 @@ public class PrivilegeTests extends ESTestCase { assertThat(predicate.test("indices:admin/mapping/put"), is(false)); assertThat(predicate.test("indices:admin/mapping/whatever"), is(false)); } - - public void testSearchPrivilege() throws Exception { - Predicate predicate = IndexPrivilege.SEARCH.predicate(); - assertThat(predicate.test(SearchAction.NAME), is(true)); - assertThat(predicate.test(SearchAction.NAME + "/whatever"), is(true)); - assertThat(predicate.test(MultiSearchAction.NAME), is(true)); - assertThat(predicate.test(MultiSearchAction.NAME + "/whatever"), is(true)); - assertThat(predicate.test(SuggestAction.NAME), is(true)); - assertThat(predicate.test(SuggestAction.NAME + "/whatever"), is(true)); - - assertThat(predicate.test(GetAction.NAME), is(false)); - assertThat(predicate.test(GetAction.NAME + "/whatever"), is(false)); - assertThat(predicate.test(MultiGetAction.NAME), is(false)); - assertThat(predicate.test(MultiGetAction.NAME + "/whatever"), is(false)); - } - - public void testGetPrivilege() throws Exception { - Predicate predicate = IndexPrivilege.GET.predicate(); - assertThat(predicate.test(GetAction.NAME), is(true)); - assertThat(predicate.test(GetAction.NAME + "/whatever"), is(true)); - assertThat(predicate.test(MultiGetAction.NAME), is(true)); - assertThat(predicate.test(MultiGetAction.NAME + "/whatever"), is(true)); - } } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java index 9aac116b137..ff724398c63 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java @@ -82,7 +82,8 @@ public class FileRolesStoreTests extends ESTestCase { assertThat(group.indices().length, is(1)); assertThat(group.indices()[0], equalTo("idx3")); assertThat(group.privilege(), notNullValue()); - assertThat(group.privilege(), is(IndexPrivilege.CRUD)); + assertThat(group.privilege().implies(IndexPrivilege.READ), is(true)); + assertThat(group.privilege().implies(IndexPrivilege.WRITE),is(true)); role = roles.get("role1.ab"); assertThat(role, notNullValue()); @@ -228,21 +229,21 @@ public class FileRolesStoreTests extends ESTestCase { * This test is mainly to make sure we can read the default roles.yml config */ public void testDefaultRolesFile() throws Exception { + // TODO we should add the config dir to the resources so we don't copy this stuff around... Path path = getDataPath("default_roles.yml"); Map roles = FileRolesStore.parseFile(path, logger, Settings.EMPTY); assertThat(roles, notNullValue()); - assertThat(roles.size(), is(10)); + assertThat(roles.size(), is(9)); assertThat(roles, hasKey("admin")); assertThat(roles, hasKey("power_user")); assertThat(roles, hasKey("user")); - assertThat(roles, hasKey("kibana4")); + assertThat(roles, hasKey("transport_client")); assertThat(roles, hasKey("kibana4_server")); assertThat(roles, hasKey("logstash")); assertThat(roles, hasKey("monitoring_user")); assertThat(roles, hasKey("remote_monitoring_agent")); assertThat(roles, hasKey("ingest_admin")); - assertThat(roles, hasKey("transport_client")); } public void testAutoReload() throws Exception { diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java index ceca8a36646..356faf4a430 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java @@ -71,8 +71,7 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ " privileges: [ ALL ]\n" + DEFAULT_TRANSPORT_CLIENT_ROLE + ":\n" + " cluster:\n" + - " - cluster:monitor/nodes/info\n" + - " - cluster:monitor/state"; + " - transport_client"; private final Path parentFolder; private final String subfolderPrefix; diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml index 3413231018a..1c36fd5ab07 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml @@ -26,101 +26,51 @@ user: # Defines the required permissions for transport clients transport_client: cluster: - - cluster:monitor/nodes/liveness - #uncomment the following for sniffing - #- cluster:monitor/state - -# The required permissions for kibana 4 users. -kibana4: - cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health - indices: - - names: '*' - privileges: - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/search - - indices:data/read/msearch - - indices:data/read/field_stats - - indices:admin/get - - names: '.kibana' - privileges: - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update + - transport_client # The required permissions for the kibana 4 server kibana4_server: cluster: - - cluster:monitor/nodes/info - - cluster:monitor/health + - monitor indices: - names: '.kibana' privileges: - - indices:admin/create - - indices:admin/exists - - indices:admin/mapping/put - - indices:admin/mappings/fields/get - - indices:admin/refresh - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search - - indices:data/write/delete - - indices:data/write/index - - indices:data/write/update + - all # The required role for logstash users logstash: cluster: - - indices:admin/template/get - - indices:admin/template/put + - manage_index_templates indices: - names: 'logstash-*' privileges: - - indices:data/write/bulk - - indices:data/write/delete - - indices:data/write/update - - indices:data/read/search - - indices:data/read/scroll + - write + - read - create_index -# Monitoring user role. Assign to monitoring users. +# Marvel user role. Assign to marvel users. monitoring_user: indices: - - names: '.monitoring-*' - privileges: - - read + - names: + - '.marvel-es-*' + - '.monitoring-*' + privileges: [ "read" ] - names: '.kibana' privileges: - - indices:admin/exists - - indices:admin/mappings/fields/get - - indices:admin/validate/query - - indices:data/read/get - - indices:data/read/mget - - indices:data/read/search + - view_index_metadata + - read -# Monitoring remote agent role. Assign to the agent user on the remote monitoring cluster -# to which the monitoring agent will export all its data +# Marvel remote agent role. Assign to the agent user on the remote marvel cluster +# to which the marvel agent will export all its data remote_monitoring_agent: - cluster: - - indices:admin/template/put - - indices:admin/template/get + cluster: [ "manage_index_templates" ] indices: - - names: '.monitoring-*' - privileges: - - all + - names: + - '.marvel-es-*' + - '.monitoring-*' + privileges: [ "all" ] # Allows all operations required to manage ingest pipelines ingest_admin: cluster: - - manage_pipeline \ No newline at end of file + - manage_pipeline diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml index 766d56487aa..47eda4b6dae 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/roles.yml @@ -9,7 +9,8 @@ role1: - READ - names: idx3 privileges: - - CRUD + - READ + - WRITE role1.ab: cluster: From 5bc3c0c1f88fcc26042690a40b0b4d736c72f961 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 16 Mar 2016 10:28:56 -0400 Subject: [PATCH 21/27] security: rename actions to not use shield This commit renames the security actions to not use shield in their action names. This also includes updating the privileges as well. Original commit: elastic/x-pack-elasticsearch@10460dffdbbc7c8abf1d6d642af247d6d05a8e92 --- .../action/realm/ClearRealmCacheAction.java | 2 +- .../action/role/ClearRolesCacheAction.java | 2 +- .../shield/action/role/DeleteRoleAction.java | 2 +- .../shield/action/role/GetRolesAction.java | 2 +- .../shield/action/role/PutRoleAction.java | 2 +- .../shield/action/user/DeleteUserAction.java | 2 +- .../shield/action/user/GetUsersAction.java | 2 +- .../shield/action/user/PutUserAction.java | 2 +- .../authz/privilege/ClusterPrivilege.java | 6 +++--- .../org/elasticsearch/transport/actions | 16 +++++++-------- .../org/elasticsearch/transport/handlers | 20 +++++++++---------- 11 files changed, 29 insertions(+), 29 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheAction.java index 218d9e84fa0..07bd31943fb 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class ClearRealmCacheAction extends Action { public static final ClearRealmCacheAction INSTANCE = new ClearRealmCacheAction(); - public static final String NAME = "cluster:admin/shield/realm/cache/clear"; + public static final String NAME = "cluster:admin/xpack/security/realm/cache/clear"; protected ClearRealmCacheAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheAction.java index 237e174450e..f7d75cfd662 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class ClearRolesCacheAction extends Action { public static final ClearRolesCacheAction INSTANCE = new ClearRolesCacheAction(); - public static final String NAME = "cluster:admin/shield/roles/cache/clear"; + public static final String NAME = "cluster:admin/xpack/security/roles/cache/clear"; protected ClearRolesCacheAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleAction.java index 67ad9af74e2..3d302912815 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class DeleteRoleAction extends Action { public static final DeleteRoleAction INSTANCE = new DeleteRoleAction(); - public static final String NAME = "cluster:admin/shield/role/delete"; + public static final String NAME = "cluster:admin/xpack/security/role/delete"; protected DeleteRoleAction() { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesAction.java index b1c36136466..e84236bf987 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class GetRolesAction extends Action { public static final GetRolesAction INSTANCE = new GetRolesAction(); - public static final String NAME = "cluster:admin/shield/role/get"; + public static final String NAME = "cluster:admin/xpack/security/role/get"; protected GetRolesAction() { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/PutRoleAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/PutRoleAction.java index 9e0a844ff4b..3e8d9fdf1af 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/PutRoleAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/PutRoleAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class PutRoleAction extends Action { public static final PutRoleAction INSTANCE = new PutRoleAction(); - public static final String NAME = "cluster:admin/shield/role/put"; + public static final String NAME = "cluster:admin/xpack/security/role/put"; protected PutRoleAction() { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserAction.java index b4f3e90950a..89c91c9c148 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class DeleteUserAction extends Action { public static final DeleteUserAction INSTANCE = new DeleteUserAction(); - public static final String NAME = "cluster:admin/shield/user/delete"; + public static final String NAME = "cluster:admin/xpack/security/user/delete"; protected DeleteUserAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersAction.java index a2849d4d027..fff2a3569d5 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class GetUsersAction extends Action { public static final GetUsersAction INSTANCE = new GetUsersAction(); - public static final String NAME = "cluster:admin/shield/user/get"; + public static final String NAME = "cluster:admin/xpack/security/user/get"; protected GetUsersAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/PutUserAction.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/PutUserAction.java index a466f38dfd0..33ec4aa6668 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/PutUserAction.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/PutUserAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class PutUserAction extends Action { public static final PutUserAction INSTANCE = new PutUserAction(); - public static final String NAME = "cluster:admin/shield/user/put"; + public static final String NAME = "cluster:admin/xpack/security/user/put"; protected PutUserAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java index 9dc00dae90b..637c7db61f1 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java @@ -26,9 +26,9 @@ import static org.elasticsearch.shield.support.Automatons.patterns; public class ClusterPrivilege extends AbstractAutomatonPrivilege { // shared automatons - private static final Automaton MANAGE_USER_AUTOMATON = patterns("cluster:admin/shield/user/*", ClearRolesCacheAction.NAME); - private static final Automaton MANAGE_ROLE_AUTOMATON = patterns("cluster:admin/shield/role/*", ClearRealmCacheAction.NAME); - private static final Automaton MANAGE_SECURITY_AUTOMATON = patterns("cluster:admin/shield/*"); + private static final Automaton MANAGE_USER_AUTOMATON = patterns("cluster:admin/xpack/security/user/*", ClearRealmCacheAction.NAME); + private static final Automaton MANAGE_ROLE_AUTOMATON = patterns("cluster:admin/xpack/security/role/*", ClearRolesCacheAction.NAME); + private static final Automaton MANAGE_SECURITY_AUTOMATON = patterns("cluster:admin/xpack/security/*"); private static final Automaton MONITOR_AUTOMATON = patterns("cluster:monitor/*"); private static final Automaton ALL_CLUSTER_AUTOMATON = patterns("cluster:*", "indices:admin/template/*"); private static final Automaton MANAGE_AUTOMATON = minusAndDeterminize(ALL_CLUSTER_AUTOMATON, MANAGE_SECURITY_AUTOMATON); diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions index 579b58738a4..da8ea8cc62b 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions @@ -75,14 +75,14 @@ indices:data/write/update cluster:monitor/xpack/license/get cluster:admin/xpack/license/delete cluster:admin/xpack/license/put -cluster:admin/shield/realm/cache/clear -cluster:admin/shield/roles/cache/clear -cluster:admin/shield/user/put -cluster:admin/shield/user/delete -cluster:admin/shield/user/get -cluster:admin/shield/role/put -cluster:admin/shield/role/delete -cluster:admin/shield/role/get +cluster:admin/xpack/security/realm/cache/clear +cluster:admin/xpack/security/roles/cache/clear +cluster:admin/xpack/security/user/put +cluster:admin/xpack/security/user/delete +cluster:admin/xpack/security/user/get +cluster:admin/xpack/security/role/put +cluster:admin/xpack/security/role/delete +cluster:admin/xpack/security/role/get internal:indices/admin/upgrade cluster:admin/ingest/pipeline/delete cluster:admin/ingest/pipeline/get diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers index cd59ee501b7..f98c8371fd0 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers @@ -8,16 +8,16 @@ cluster:monitor/nodes/liveness cluster:monitor/nodes/stats[n] cluster:monitor/stats[n] cluster:monitor/tasks/lists[n] -cluster:admin/shield/realm/cache/clear -cluster:admin/shield/realm/cache/clear[n] -cluster:admin/shield/roles/cache/clear -cluster:admin/shield/roles/cache/clear[n] -cluster:admin/shield/role/put -cluster:admin/shield/role/delete -cluster:admin/shield/role/get -cluster:admin/shield/user/put -cluster:admin/shield/user/delete -cluster:admin/shield/user/get +cluster:admin/xpack/security/realm/cache/clear +cluster:admin/xpack/security/realm/cache/clear[n] +cluster:admin/xpack/security/roles/cache/clear +cluster:admin/xpack/security/roles/cache/clear[n] +cluster:admin/xpack/security/role/put +cluster:admin/xpack/security/role/delete +cluster:admin/xpack/security/role/get +cluster:admin/xpack/security/user/put +cluster:admin/xpack/security/user/delete +cluster:admin/xpack/security/user/get indices:admin/analyze[s] indices:admin/cache/clear[n] indices:admin/forcemerge[n] From 135742823e42953203d9dd20426fbfa15c93bd45 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 17 Mar 2016 09:41:29 -0400 Subject: [PATCH 22/27] rename watcher actions Original commit: elastic/x-pack-elasticsearch@5c6734459506d51ef68492e7cdc72afa041a923b --- .../java/org/elasticsearch/watcher/Watcher.java | 4 ++-- .../transport/actions/ack/AckWatchAction.java | 2 +- .../actions/activate/ActivateWatchAction.java | 2 +- .../actions/delete/DeleteWatchAction.java | 2 +- .../actions/execute/ExecuteWatchAction.java | 2 +- .../transport/actions/get/GetWatchAction.java | 2 +- .../transport/actions/put/PutWatchAction.java | 2 +- .../actions/service/WatcherServiceAction.java | 2 +- .../actions/stats/WatcherStatsAction.java | 2 +- .../watcher/shield/BasicShieldTests.java | 14 +++++++------- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java index 1d9a203f54c..28dbd3d328b 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java @@ -123,8 +123,8 @@ public class Watcher { // adding the watcher privileges to shield if (Shield.enabled(settings)) { - Shield.registerClusterPrivilege("manage_watcher", "cluster:admin/watcher/*", "cluster:monitor/watcher/*"); - Shield.registerClusterPrivilege("monitor_watcher", "cluster:monitor/watcher/*"); + Shield.registerClusterPrivilege("manage_watcher", "cluster:admin/xpack/watcher/*", "cluster:monitor/xpack/watcher/*"); + Shield.registerClusterPrivilege("monitor_watcher", "cluster:monitor/xpack/watcher/*"); } } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java index 010872e11ee..14c983095fa 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/ack/AckWatchAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class AckWatchAction extends Action { public static final AckWatchAction INSTANCE = new AckWatchAction(); - public static final String NAME = "cluster:admin/watcher/watch/ack"; + public static final String NAME = "cluster:admin/xpack/watcher/watch/ack"; private AckWatchAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java index 5ea0c3833ee..b8e5dbb408b 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/activate/ActivateWatchAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class ActivateWatchAction extends Action { public static final ActivateWatchAction INSTANCE = new ActivateWatchAction(); - public static final String NAME = "cluster:admin/watcher/watch/activate"; + public static final String NAME = "cluster:admin/xpack/watcher/watch/activate"; private ActivateWatchAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java index ebe89e9b42f..2c795d9074d 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/delete/DeleteWatchAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class DeleteWatchAction extends Action { public static final DeleteWatchAction INSTANCE = new DeleteWatchAction(); - public static final String NAME = "cluster:admin/watcher/watch/delete"; + public static final String NAME = "cluster:admin/xpack/watcher/watch/delete"; private DeleteWatchAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java index 7652db22c38..6f0b2e03740 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/execute/ExecuteWatchAction.java @@ -15,7 +15,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class ExecuteWatchAction extends Action { public static final ExecuteWatchAction INSTANCE = new ExecuteWatchAction(); - public static final String NAME = "cluster:admin/watcher/watch/execute"; + public static final String NAME = "cluster:admin/xpack/watcher/watch/execute"; private ExecuteWatchAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java index 710dbbc1eff..3b081293bab 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/get/GetWatchAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class GetWatchAction extends org.elasticsearch.action.Action { public static final GetWatchAction INSTANCE = new GetWatchAction(); - public static final String NAME = "cluster:monitor/watcher/watch/get"; + public static final String NAME = "cluster:monitor/xpack/watcher/watch/get"; private GetWatchAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java index 2e99270a8ac..9025f6d7953 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/put/PutWatchAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class PutWatchAction extends Action { public static final PutWatchAction INSTANCE = new PutWatchAction(); - public static final String NAME = "cluster:admin/watcher/watch/put"; + public static final String NAME = "cluster:admin/xpack/watcher/watch/put"; private PutWatchAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java index 18db6c848e4..064d3379da4 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/service/WatcherServiceAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class WatcherServiceAction extends Action { public static final WatcherServiceAction INSTANCE = new WatcherServiceAction(); - public static final String NAME = "cluster:admin/watcher/service"; + public static final String NAME = "cluster:admin/xpack/watcher/service"; private WatcherServiceAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java index e5339e873ca..6400f429adc 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/transport/actions/stats/WatcherStatsAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ElasticsearchClient; public class WatcherStatsAction extends Action { public static final WatcherStatsAction INSTANCE = new WatcherStatsAction(); - public static final String NAME = "cluster:monitor/watcher/stats"; + public static final String NAME = "cluster:monitor/xpack/watcher/stats"; private WatcherStatsAction() { super(NAME); diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java index 098cc58a92c..72920eceb8c 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/shield/BasicShieldTests.java @@ -50,7 +50,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { fail("authentication failure should have occurred"); } catch (Exception e) { // transport_client is the default user - assertThat(e.getMessage(), equalTo("action [cluster:monitor/watcher/stats] is unauthorized for user [transport_client]")); + assertThat(e.getMessage(), equalTo("action [cluster:monitor/xpack/watcher/stats] is unauthorized for user [transport_client]")); } } @@ -62,7 +62,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { .get(); fail("authentication failure should have occurred"); } catch (Exception e) { - assertThat(e.getMessage(), equalTo("action [cluster:monitor/watcher/stats] is unauthorized for user [test]")); + assertThat(e.getMessage(), equalTo("action [cluster:monitor/xpack/watcher/stats] is unauthorized for user [test]")); } try { @@ -70,7 +70,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { .get(); fail("authentication failure should have occurred"); } catch (Exception e) { - assertThat(e.getMessage(), equalTo("action [cluster:monitor/watcher/watch/get] is unauthorized for user [test]")); + assertThat(e.getMessage(), equalTo("action [cluster:monitor/xpack/watcher/watch/get] is unauthorized for user [test]")); } // stats and get watch are allowed by role monitor: @@ -89,7 +89,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { .get(); fail("authentication failure should have occurred"); } catch (Exception e) { - assertThat(e.getMessage(), equalTo("action [cluster:admin/watcher/watch/put] is unauthorized for user [monitor]")); + assertThat(e.getMessage(), equalTo("action [cluster:admin/xpack/watcher/watch/put] is unauthorized for user [monitor]")); } } @@ -102,7 +102,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { .get(); fail("authentication failure should have occurred"); } catch (Exception e) { - assertThat(e.getMessage(), equalTo("action [cluster:admin/watcher/watch/put] is unauthorized for user [test]")); + assertThat(e.getMessage(), equalTo("action [cluster:admin/xpack/watcher/watch/put] is unauthorized for user [test]")); } TriggerEvent triggerEvent = new ScheduleTriggerEvent(new DateTime(UTC), new DateTime(UTC)); @@ -112,7 +112,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { .get(); fail("authentication failure should have occurred"); } catch (Exception e) { - assertThat(e.getMessage(), equalTo("action [cluster:admin/watcher/watch/execute] is unauthorized for user [test]")); + assertThat(e.getMessage(), equalTo("action [cluster:admin/xpack/watcher/watch/execute] is unauthorized for user [test]")); } try { @@ -120,7 +120,7 @@ public class BasicShieldTests extends AbstractWatcherIntegrationTestCase { .get(); fail("authentication failure should have occurred"); } catch (Exception e) { - assertThat(e.getMessage(), equalTo("action [cluster:admin/watcher/watch/delete] is unauthorized for user [test]")); + assertThat(e.getMessage(), equalTo("action [cluster:admin/xpack/watcher/watch/delete] is unauthorized for user [test]")); } // put, execute and delete watch apis are allowed by role admin: From cf0fd986e117ff4b7660b197e5176eea5466b98a Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 17 Mar 2016 09:46:05 -0400 Subject: [PATCH 23/27] rename graph actions Original commit: elastic/x-pack-elasticsearch@70a71d6bd6239d8b644eb62dc5b1d19b2ee73ea8 --- .../smoketest/GraphWithShieldInsufficientRoleIT.java | 2 +- .../java/org/elasticsearch/graph/action/GraphExploreAction.java | 2 +- .../src/test/resources/org/elasticsearch/transport/actions | 2 +- .../src/test/resources/org/elasticsearch/transport/handlers | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/elasticsearch/qa/smoke-test-graph-with-shield/src/test/java/org/elasticsearch/smoketest/GraphWithShieldInsufficientRoleIT.java b/elasticsearch/qa/smoke-test-graph-with-shield/src/test/java/org/elasticsearch/smoketest/GraphWithShieldInsufficientRoleIT.java index 795f98edbda..b85dd676a5c 100644 --- a/elasticsearch/qa/smoke-test-graph-with-shield/src/test/java/org/elasticsearch/smoketest/GraphWithShieldInsufficientRoleIT.java +++ b/elasticsearch/qa/smoke-test-graph-with-shield/src/test/java/org/elasticsearch/smoketest/GraphWithShieldInsufficientRoleIT.java @@ -24,7 +24,7 @@ public class GraphWithShieldInsufficientRoleIT extends GraphWithShieldIT { super.test(); fail(); } catch(AssertionError ae) { - assertThat(ae.getMessage(), containsString("action [indices:data/read/graph/explore")); + assertThat(ae.getMessage(), containsString("action [indices:data/read/xpack/graph/explore")); assertThat(ae.getMessage(), containsString("returned [403 Forbidden]")); assertThat(ae.getMessage(), containsString("is unauthorized for user [no_graph_explorer]")); } diff --git a/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/GraphExploreAction.java b/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/GraphExploreAction.java index 33687ac98ec..1d3537c80e6 100644 --- a/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/GraphExploreAction.java +++ b/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/GraphExploreAction.java @@ -12,7 +12,7 @@ public class GraphExploreAction extends Action { public static final GraphExploreAction INSTANCE = new GraphExploreAction(); - public static final String NAME = "indices:data/read/graph/explore"; + public static final String NAME = "indices:data/read/xpack/graph/explore"; private GraphExploreAction() { super(NAME); diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions index da8ea8cc62b..0075c73310e 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions @@ -54,7 +54,7 @@ indices:monitor/upgrade indices:data/read/explain indices:data/read/field_stats indices:data/read/get -indices:data/read/graph/explore +indices:data/read/xpack/graph/explore indices:data/read/mget indices:data/read/mpercolate indices:data/read/msearch diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers index f98c8371fd0..94f93e379cb 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/handlers @@ -36,7 +36,7 @@ indices:admin/validate/query[s] indices:data/read/explain[s] indices:data/read/field_stats[s] indices:data/read/get[s] -indices:data/read/graph/explore +indices:data/read/xpack/graph/explore indices:data/read/mget[shard][s] indices:data/read/mpercolate[shard][s] indices:data/read/mtv[shard][s] From 833bf726e659e5495859a9e98c42ad6ce9e1dabe Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 17 Mar 2016 09:58:33 -0400 Subject: [PATCH 24/27] define graph and watcher privileges statically These privileges no longer need to be defined as a custom privilege since the code is now consolidated into a single plugin. This also changes the manage cluster privilege to be an alias to the all privilege. Original commit: elastic/x-pack-elasticsearch@a7f444c8984dee907dd054eba22f80f90b701ca9 --- .../java/org/elasticsearch/graph/Graph.java | 8 ------- .../java/org/elasticsearch/shield/Shield.java | 24 ------------------- .../authz/privilege/ClusterPrivilege.java | 11 ++++++--- .../authz/privilege/IndexPrivilege.java | 7 ++++++ .../org/elasticsearch/watcher/Watcher.java | 7 ------ 5 files changed, 15 insertions(+), 42 deletions(-) diff --git a/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/Graph.java b/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/Graph.java index 28ebf053702..a44c990ab80 100644 --- a/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/Graph.java +++ b/elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/Graph.java @@ -10,7 +10,6 @@ import java.util.Collection; import java.util.Collections; import org.elasticsearch.action.ActionModule; -import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.network.NetworkModule; @@ -23,8 +22,6 @@ import org.elasticsearch.graph.license.GraphLicensee; import org.elasticsearch.graph.license.GraphModule; import org.elasticsearch.graph.rest.action.RestGraphAction; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.action.SearchTransportService; -import org.elasticsearch.shield.Shield; import org.elasticsearch.xpack.XPackPlugin; public class Graph extends Plugin { @@ -37,11 +34,6 @@ public class Graph extends Plugin { public Graph(Settings settings) { this.transportClientMode = XPackPlugin.transportClientMode(settings); enabled = enabled(settings); - // adding the graph privileges to shield - if (Shield.enabled(settings)) { - Shield.registerIndexPrivilege( "graph", GraphExploreAction.NAME, SearchTransportService.QUERY_ACTION_NAME, - SearchAction.NAME, SearchTransportService.QUERY_FETCH_ACTION_NAME); - } } @Override diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java index 4c0ce5a3c8a..f5b41c6742b 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java @@ -290,30 +290,6 @@ public class Shield { } } - public static void registerClusterPrivilege(String name, String... patterns) { - try { - ClusterPrivilege.addCustom(name, patterns); - } catch (Exception se) { - logger.warn("could not register cluster privilege [{}]", name); - - // we need to prevent bubbling the shield exception here for the tests. In the tests - // we create multiple nodes in the same jvm and since the custom cluster is a static binding - // multiple nodes will try to add the same privileges multiple times. - } - } - - public static void registerIndexPrivilege(String name, String... patterns) { - try { - IndexPrivilege.addCustom(name, patterns); - } catch (Exception se) { - logger.warn("could not register index privilege [{}]", name); - - // we need to prevent bubbling the shield exception here for the tests. In the tests - // we create multiple nodes in the same jvm and since the custom cluster is a static binding - // multiple nodes will try to add the same privileges multiple times. - } - } - private void addUserSettings(Settings.Builder settingsBuilder) { String authHeaderSettingName = ThreadContext.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER; if (settings.get(authHeaderSettingName) != null) { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java index 637c7db61f1..f683ca76d20 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java @@ -7,6 +7,7 @@ package org.elasticsearch.shield.authz.privilege; import dk.brics.automaton.Automaton; import org.elasticsearch.common.Strings; +import org.elasticsearch.license.plugin.action.get.GetLicenseAction; import org.elasticsearch.shield.action.realm.ClearRealmCacheAction; import org.elasticsearch.shield.action.role.ClearRolesCacheAction; import org.elasticsearch.shield.support.Automatons; @@ -17,7 +18,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.function.Predicate; -import static org.elasticsearch.shield.support.Automatons.minusAndDeterminize; import static org.elasticsearch.shield.support.Automatons.patterns; /** @@ -29,16 +29,17 @@ public class ClusterPrivilege extends AbstractAutomatonPrivilege ACTION_MATCHER = ClusterPrivilege.ALL.predicate(); @@ -62,6 +65,8 @@ public class ClusterPrivilege extends AbstractAutomatonPrivilege values() { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java index 603be891a4e..96ebbb95478 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java @@ -19,7 +19,10 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; +import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.common.Strings; +import org.elasticsearch.graph.action.GraphExploreAction; +import org.elasticsearch.search.action.SearchTransportService; import org.elasticsearch.shield.support.Automatons; import java.util.Locale; @@ -50,6 +53,8 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME, AliasesExistAction.NAME, GetIndexAction.NAME, IndicesExistsAction.NAME, GetFieldMappingsAction.NAME, GetMappingsAction.NAME, ClusterSearchShardsAction.NAME, TypesExistsAction.NAME, ValidateQueryAction.NAME, GetSettingsAction.NAME); + private static final Automaton GRAPH_AUTOMATON = patterns(GraphExploreAction.NAME, SearchTransportService.QUERY_ACTION_NAME, + SearchAction.NAME, SearchTransportService.QUERY_FETCH_ACTION_NAME); public static final IndexPrivilege NONE = new IndexPrivilege(Name.NONE, Automatons.EMPTY); public static final IndexPrivilege ALL = new IndexPrivilege(Name.ALL, ALL_AUTOMATON); @@ -63,6 +68,7 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { public static final IndexPrivilege DELETE_INDEX = new IndexPrivilege("delete_index", DELETE_INDEX_AUTOMATON); public static final IndexPrivilege CREATE_INDEX = new IndexPrivilege("create_index", CREATE_INDEX_AUTOMATON); public static final IndexPrivilege VIEW_METADATA = new IndexPrivilege("view_index_metadata", VIEW_METADATA_AUTOMATON); + public static final IndexPrivilege GRAPH = new IndexPrivilege("graph", GRAPH_AUTOMATON); private static final Set values = new CopyOnWriteArraySet<>(); @@ -79,6 +85,7 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { values.add(CREATE); values.add(DELETE_INDEX); values.add(VIEW_METADATA); + values.add(GRAPH); } public static final Predicate ACTION_MATCHER = ALL.predicate(); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java index 28dbd3d328b..350abe41746 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/Watcher.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.script.ScriptModule; -import org.elasticsearch.shield.Shield; import org.elasticsearch.watcher.actions.WatcherActionModule; import org.elasticsearch.watcher.actions.email.service.EmailService; import org.elasticsearch.watcher.actions.email.service.InternalEmailService; @@ -120,12 +119,6 @@ public class Watcher { transportClient = "transport".equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())); enabled = enabled(settings); validAutoCreateIndex(settings); - - // adding the watcher privileges to shield - if (Shield.enabled(settings)) { - Shield.registerClusterPrivilege("manage_watcher", "cluster:admin/xpack/watcher/*", "cluster:monitor/xpack/watcher/*"); - Shield.registerClusterPrivilege("monitor_watcher", "cluster:monitor/xpack/watcher/*"); - } } public Collection nodeModules() { From 2872acd742f00d11d998eb40487e53e534501ae6 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 17 Mar 2016 12:51:03 -0400 Subject: [PATCH 25/27] remove watcher and graph privileges. manage does not include security Original commit: elastic/x-pack-elasticsearch@da250ed84224653f6fd8e0035ba6f4edeaf82d54 --- .../watcher-with-shield-roles.yml | 4 ++-- .../qa/smoke-test-graph-with-shield/roles.yml | 4 ++-- .../smoke-test-watcher-with-shield/roles.yml | 6 ++---- .../x-pack/shield/config/xpack/roles.yml | 5 ----- .../authz/privilege/ClusterPrivilege.java | 21 +++---------------- .../authz/privilege/IndexPrivilege.java | 7 ------- .../authz/privilege/PrivilegeTests.java | 10 --------- .../authz/store/FileRolesStoreTests.java | 3 +-- .../shield/authz/store/default_roles.yml | 5 ----- .../AbstractWatcherIntegrationTestCase.java | 6 +++--- 10 files changed, 13 insertions(+), 58 deletions(-) diff --git a/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml index a0c1878f919..74a86b35eb5 100644 --- a/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml +++ b/elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml @@ -4,12 +4,12 @@ admin: '*': all watcher_manager: - cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/health + cluster: manage indices: '.watcher-history-*': all watcher_monitor: - cluster: monitor_watcher + cluster: monitor indices: '.watcher-history-*': read diff --git a/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml b/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml index 69b72aaddd5..6a5b4183287 100644 --- a/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml +++ b/elasticsearch/qa/smoke-test-graph-with-shield/roles.yml @@ -12,8 +12,8 @@ graph_explorer: indices: - names: '*' privileges: - - graph - - indices:data/write/index + - read + - write - indices:admin/refresh - indices:admin/create diff --git a/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml b/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml index ced242d9d3b..ba29cd6d7ae 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml +++ b/elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml @@ -8,9 +8,7 @@ admin: watcher_manager: cluster: - - manage_watcher - - cluster:monitor/nodes/info - - cluster:monitor/health + - manage indices: - names: '.watcher-history-*' privileges: @@ -21,7 +19,7 @@ watcher_manager: watcher_monitor: cluster: - - monitor_watcher + - monitor indices: - names: '.watcher-history-*' privileges: diff --git a/elasticsearch/x-pack/shield/config/xpack/roles.yml b/elasticsearch/x-pack/shield/config/xpack/roles.yml index 1c36fd5ab07..f998b18b427 100644 --- a/elasticsearch/x-pack/shield/config/xpack/roles.yml +++ b/elasticsearch/x-pack/shield/config/xpack/roles.yml @@ -69,8 +69,3 @@ remote_monitoring_agent: - '.marvel-es-*' - '.monitoring-*' privileges: [ "all" ] - -# Allows all operations required to manage ingest pipelines -ingest_admin: - cluster: - - manage_pipeline diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java index f683ca76d20..1df9151bb43 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/ClusterPrivilege.java @@ -7,9 +7,6 @@ package org.elasticsearch.shield.authz.privilege; import dk.brics.automaton.Automaton; import org.elasticsearch.common.Strings; -import org.elasticsearch.license.plugin.action.get.GetLicenseAction; -import org.elasticsearch.shield.action.realm.ClearRealmCacheAction; -import org.elasticsearch.shield.action.role.ClearRolesCacheAction; import org.elasticsearch.shield.support.Automatons; import java.util.Locale; @@ -18,6 +15,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.function.Predicate; +import static org.elasticsearch.shield.support.Automatons.minusAndDeterminize; import static org.elasticsearch.shield.support.Automatons.patterns; /** @@ -26,29 +24,21 @@ import static org.elasticsearch.shield.support.Automatons.patterns; public class ClusterPrivilege extends AbstractAutomatonPrivilege { // shared automatons - private static final Automaton MANAGE_USER_AUTOMATON = patterns("cluster:admin/xpack/security/user/*", ClearRealmCacheAction.NAME); - private static final Automaton MANAGE_ROLE_AUTOMATON = patterns("cluster:admin/xpack/security/role/*", ClearRolesCacheAction.NAME); private static final Automaton MANAGE_SECURITY_AUTOMATON = patterns("cluster:admin/xpack/security/*"); - private static final Automaton MANAGE_WATCHER_AUTOMATON = patterns("cluster:admin/xpack/watcher/*", "cluster:monitor/xpack/watcher/*"); - private static final Automaton MONITOR_WATCHER_AUTOMATON = patterns("cluster:monitor/xpack/watcher/*"); private static final Automaton MONITOR_AUTOMATON = patterns("cluster:monitor/*"); private static final Automaton ALL_CLUSTER_AUTOMATON = patterns("cluster:*", "indices:admin/template/*"); + private static final Automaton MANAGE_AUTOMATON = minusAndDeterminize(ALL_CLUSTER_AUTOMATON, MANAGE_SECURITY_AUTOMATON); private static final Automaton TRANSPORT_CLIENT_AUTOMATON = patterns("cluster:monitor/nodes/liveness", "cluster:monitor/state"); private static final Automaton MANAGE_IDX_TEMPLATE_AUTOMATON = patterns("indices:admin/template/*"); public static final ClusterPrivilege NONE = new ClusterPrivilege(Name.NONE, Automatons.EMPTY); public static final ClusterPrivilege ALL = new ClusterPrivilege(Name.ALL, ALL_CLUSTER_AUTOMATON); public static final ClusterPrivilege MONITOR = new ClusterPrivilege("monitor", MONITOR_AUTOMATON); - public static final ClusterPrivilege MANAGE = new ClusterPrivilege("manage", ALL_CLUSTER_AUTOMATON); + public static final ClusterPrivilege MANAGE = new ClusterPrivilege("manage", MANAGE_AUTOMATON); public static final ClusterPrivilege MANAGE_IDX_TEMPLATES = new ClusterPrivilege("manage_index_templates", MANAGE_IDX_TEMPLATE_AUTOMATON); public static final ClusterPrivilege TRANSPORT_CLIENT = new ClusterPrivilege("transport_client", TRANSPORT_CLIENT_AUTOMATON); - public static final ClusterPrivilege MANAGE_USERS = new ClusterPrivilege("manage_users", MANAGE_USER_AUTOMATON); - public static final ClusterPrivilege MANAGE_ROLES = new ClusterPrivilege("manage_roles", MANAGE_ROLE_AUTOMATON); public static final ClusterPrivilege MANAGE_SECURITY = new ClusterPrivilege("manage_security", MANAGE_SECURITY_AUTOMATON); - public static final ClusterPrivilege MANAGE_PIPELINE = new ClusterPrivilege("manage_pipeline", "cluster:admin/ingest/pipeline/*"); - public static final ClusterPrivilege MONITOR_WATCHER = new ClusterPrivilege("monitor_watcher", MONITOR_WATCHER_AUTOMATON); - public static final ClusterPrivilege MANAGE_WATCHER = new ClusterPrivilege("manage_watcher", MANAGE_WATCHER_AUTOMATON); public final static Predicate ACTION_MATCHER = ClusterPrivilege.ALL.predicate(); @@ -61,12 +51,7 @@ public class ClusterPrivilege extends AbstractAutomatonPrivilege values() { diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java index 96ebbb95478..603be891a4e 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java @@ -19,10 +19,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; -import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.common.Strings; -import org.elasticsearch.graph.action.GraphExploreAction; -import org.elasticsearch.search.action.SearchTransportService; import org.elasticsearch.shield.support.Automatons; import java.util.Locale; @@ -53,8 +50,6 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME, AliasesExistAction.NAME, GetIndexAction.NAME, IndicesExistsAction.NAME, GetFieldMappingsAction.NAME, GetMappingsAction.NAME, ClusterSearchShardsAction.NAME, TypesExistsAction.NAME, ValidateQueryAction.NAME, GetSettingsAction.NAME); - private static final Automaton GRAPH_AUTOMATON = patterns(GraphExploreAction.NAME, SearchTransportService.QUERY_ACTION_NAME, - SearchAction.NAME, SearchTransportService.QUERY_FETCH_ACTION_NAME); public static final IndexPrivilege NONE = new IndexPrivilege(Name.NONE, Automatons.EMPTY); public static final IndexPrivilege ALL = new IndexPrivilege(Name.ALL, ALL_AUTOMATON); @@ -68,7 +63,6 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { public static final IndexPrivilege DELETE_INDEX = new IndexPrivilege("delete_index", DELETE_INDEX_AUTOMATON); public static final IndexPrivilege CREATE_INDEX = new IndexPrivilege("create_index", CREATE_INDEX_AUTOMATON); public static final IndexPrivilege VIEW_METADATA = new IndexPrivilege("view_index_metadata", VIEW_METADATA_AUTOMATON); - public static final IndexPrivilege GRAPH = new IndexPrivilege("graph", GRAPH_AUTOMATON); private static final Set values = new CopyOnWriteArraySet<>(); @@ -85,7 +79,6 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege { values.add(CREATE); values.add(DELETE_INDEX); values.add(VIEW_METADATA); - values.add(GRAPH); } public static final Predicate ACTION_MATCHER = ALL.predicate(); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java index 11fb580ea5b..65f5a4e796f 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java @@ -72,16 +72,6 @@ public class PrivilegeTests extends ESTestCase { assertThat(cluster, is(cluster2)); } - public void testIngestPrivilege() throws Exception { - Privilege.Name name = new Privilege.Name("manage_pipeline"); - ClusterPrivilege cluster = ClusterPrivilege.get(name); - assertThat(cluster, is(ClusterPrivilege.MANAGE_PIPELINE)); - assertThat(cluster.predicate().test(PutPipelineAction.NAME), is(true)); - assertThat(cluster.predicate().test(DeletePipelineAction.NAME), is(true)); - assertThat(cluster.predicate().test(GetPipelineAction.NAME), is(true)); - assertThat(cluster.predicate().test(SimulatePipelineAction.NAME), is(true)); - } - public void testClusterTemplateActions() throws Exception { Privilege.Name name = new Privilege.Name("indices:admin/template/delete"); ClusterPrivilege cluster = ClusterPrivilege.get(name); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java index ff724398c63..7a3e498b624 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java @@ -233,7 +233,7 @@ public class FileRolesStoreTests extends ESTestCase { Path path = getDataPath("default_roles.yml"); Map roles = FileRolesStore.parseFile(path, logger, Settings.EMPTY); assertThat(roles, notNullValue()); - assertThat(roles.size(), is(9)); + assertThat(roles.size(), is(8)); assertThat(roles, hasKey("admin")); assertThat(roles, hasKey("power_user")); @@ -243,7 +243,6 @@ public class FileRolesStoreTests extends ESTestCase { assertThat(roles, hasKey("logstash")); assertThat(roles, hasKey("monitoring_user")); assertThat(roles, hasKey("remote_monitoring_agent")); - assertThat(roles, hasKey("ingest_admin")); } public void testAutoReload() throws Exception { diff --git a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml index 1c36fd5ab07..f998b18b427 100644 --- a/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml +++ b/elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/shield/authz/store/default_roles.yml @@ -69,8 +69,3 @@ remote_monitoring_agent: - '.marvel-es-*' - '.monitoring-*' privileges: [ "all" ] - -# Allows all operations required to manage ingest pipelines -ingest_admin: - cluster: - - manage_pipeline diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java index 2859e9ad664..0c958f27ec8 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -691,12 +691,12 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase " privileges: [ all ]\n" + "\n" + "admin:\n" + - " cluster: [ 'manage_watcher', 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + + " cluster: [ 'manage' ]\n" + "transport_client:\n" + - " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + + " cluster: [ 'transport_client' ]\n" + "\n" + "monitor:\n" + - " cluster: [ 'monitor_watcher', 'cluster:monitor/nodes/info', 'cluster:monitor/nodes/liveness' ]\n" + " cluster: [ 'monitor' ]\n" ; From 67c6cf405546da528024647156cef4692ac6b84c Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 18 Mar 2016 07:33:22 -0400 Subject: [PATCH 26/27] test: add debug logging and cleanup stopping of servers Adds debug logging to try to get more information about random failures in these tests. Also cleans up some potential issues with the code that handled the stopping of random ldap servers to test failure cases. See elastic/elasticsearch#1542 Original commit: elastic/x-pack-elasticsearch@573b4161dd6ac6951dea66c0acd36203d353302f --- .../SessionFactoryLoadBalancingTests.java | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryLoadBalancingTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryLoadBalancingTests.java index 843d6bd457f..423edb20dc7 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryLoadBalancingTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/SessionFactoryLoadBalancingTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.authc.ldap.support; +import com.unboundid.ldap.listener.InMemoryDirectoryServer; import com.unboundid.ldap.sdk.LDAPConnection; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.shield.authc.RealmConfig; @@ -12,8 +13,10 @@ import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.shield.ssl.ClientSSLService; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -43,6 +46,7 @@ public class SessionFactoryLoadBalancingTests extends LdapTestCase { public void testRoundRobinWithFailures() throws Exception { assumeTrue("at least one ldap server should be present for this test", ldapServers.length > 1); + logger.debug("using [{}] ldap servers, urls {}", ldapServers.length, ldapUrls()); TestSessionFactory testSessionFactory = createSessionFactory(LdapLoadBalancing.ROUND_ROBIN); // create a list of ports @@ -50,19 +54,31 @@ public class SessionFactoryLoadBalancingTests extends LdapTestCase { for (int i = 0; i < ldapServers.length; i++) { ports.add(ldapServers[i].getListenPort()); } + logger.debug("list of all ports {}", ports); - int numberToKill = randomIntBetween(1, numberOfLdapServers - 1); - for (int i = 0; i < numberToKill; i++) { - int index = randomIntBetween(0, numberOfLdapServers - 1); - ports.remove(Integer.valueOf(ldapServers[index].getListenPort())); + final int numberToKill = randomIntBetween(1, numberOfLdapServers - 1); + logger.debug("killing [{}] servers", numberToKill); + + // get a subset to kil + final List ldapServersToKill = randomSubsetOf(numberToKill, ldapServers); + final List ldapServersList = Arrays.asList(ldapServers); + for (InMemoryDirectoryServer ldapServerToKill : ldapServersToKill) { + final int index = ldapServersList.indexOf(ldapServerToKill); + assertThat(index, greaterThanOrEqualTo(0)); + final Integer port = Integer.valueOf(ldapServers[index].getListenPort()); + logger.debug("shutting down server index [{}] listening on [{}]", index, port); + assertTrue(ports.remove(port)); ldapServers[index].shutDown(true); + assertThat(ldapServers[index].getListenPort(), is(-1)); } final int numberOfIterations = randomIntBetween(1, 5); for (int iteration = 0; iteration < numberOfIterations; iteration++) { + logger.debug("iteration [{}]", iteration); for (Integer port : ports) { LDAPConnection connection = null; try { + logger.debug("attempting connection with expected port [{}]", port); connection = testSessionFactory.getServerSet().getConnection(); assertThat(connection.getConnectedPort(), is(port)); } finally { @@ -76,6 +92,7 @@ public class SessionFactoryLoadBalancingTests extends LdapTestCase { public void testFailover() throws Exception { assumeTrue("at least one ldap server should be present for this test", ldapServers.length > 1); + logger.debug("using [{}] ldap servers, urls {}", ldapServers.length, ldapUrls()); TestSessionFactory testSessionFactory = createSessionFactory(LdapLoadBalancing.FAILOVER); // first test that there is no round robin stuff going on @@ -92,32 +109,46 @@ public class SessionFactoryLoadBalancingTests extends LdapTestCase { } } - List stoppedServers = new ArrayList<>(); - // now we should kill some servers including the first one - int numberToKill = randomIntBetween(1, numberOfLdapServers - 1); - // always kill the first one, but don't add to the list + logger.debug("shutting down server index [0] listening on [{}]", ldapServers[0].getListenPort()); + // always kill the first one ldapServers[0].shutDown(true); - stoppedServers.add(0); - for (int i = 0; i < numberToKill - 1; i++) { - int index = randomIntBetween(1, numberOfLdapServers - 1); - ldapServers[index].shutDown(true); - stoppedServers.add(index); + assertThat(ldapServers[0].getListenPort(), is(-1)); + + // now randomly shutdown some others + if (ldapServers.length > 2) { + // kill at least one other server, but we need at least one good one. Hence the upper bound is number - 2 since we need at least + // one server to use! + final int numberToKill = randomIntBetween(1, numberOfLdapServers - 2); + InMemoryDirectoryServer[] allButFirstServer = Arrays.copyOfRange(ldapServers, 1, ldapServers.length); + // get a subset to kil + final List ldapServersToKill = randomSubsetOf(numberToKill, allButFirstServer); + final List ldapServersList = Arrays.asList(ldapServers); + for (InMemoryDirectoryServer ldapServerToKill : ldapServersToKill) { + final int index = ldapServersList.indexOf(ldapServerToKill); + assertThat(index, greaterThanOrEqualTo(1)); + final Integer port = Integer.valueOf(ldapServers[index].getListenPort()); + logger.debug("shutting down server index [{}] listening on [{}]", index, port); + ldapServers[index].shutDown(true); + assertThat(ldapServers[index].getListenPort(), is(-1)); + } } int firstNonStoppedPort = -1; // now we find the first that isn't stopped for (int i = 0; i < numberOfLdapServers; i++) { - if (stoppedServers.contains(i) == false) { + if (ldapServers[i].getListenPort() != -1) { firstNonStoppedPort = ldapServers[i].getListenPort(); break; } } + logger.debug("first non stopped port [{}]", firstNonStoppedPort); assertThat(firstNonStoppedPort, not(-1)); final int numberOfIterations = randomIntBetween(1, 5); for (int iteration = 0; iteration < numberOfIterations; iteration++) { LDAPConnection connection = null; try { + logger.debug("attempting connection with expected port [{}] iteration [{}]", firstNonStoppedPort, iteration); connection = testSessionFactory.getServerSet().getConnection(); assertThat(connection.getConnectedPort(), is(firstNonStoppedPort)); } finally { From fc99174b48c2ac79c7aa4c5aaf590791a9a0a962 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Fri, 18 Mar 2016 15:28:50 +0100 Subject: [PATCH 27/27] Testing: Removing AwaitsFix annotation as tests pass again Tests had been muted in elastic/x-pack@0ba4d7ead880abe19b27b4d5e8f9701461916e60, but several local runs passed. Original commit: elastic/x-pack-elasticsearch@6bf0dfa4e390d14d5cb40cbce772579509d6fdaa --- .../watcher/actions/email/EmailAttachmentTests.java | 2 -- .../watcher/history/HistoryTemplateEmailMappingsTests.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailAttachmentTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailAttachmentTests.java index 7f767656c75..cef29b085bb 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailAttachmentTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/actions/email/EmailAttachmentTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.watcher.actions.email; import com.squareup.okhttp.mockwebserver.MockResponse; import com.squareup.okhttp.mockwebserver.MockWebServer; import com.squareup.okhttp.mockwebserver.QueueDispatcher; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; @@ -61,7 +60,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.startsWith; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/17065") public class EmailAttachmentTests extends AbstractWatcherIntegrationTestCase { static final String USERNAME = "_user"; diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java index f623fc843f1..92e4a5ec733 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/history/HistoryTemplateEmailMappingsTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.watcher.history; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.aggregations.Aggregations; @@ -32,7 +31,6 @@ import static org.hamcrest.Matchers.notNullValue; * This test makes sure that the email address fields in the watch_record action result are * not analyzed so they can be used in aggregations */ -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/17065") public class HistoryTemplateEmailMappingsTests extends AbstractWatcherIntegrationTestCase { static final String USERNAME = "_user"; static final String PASSWORD = "_passwd";